调用 OnActionReceived 或 RequestDecision 来让 Unity ML-Agent 轮流执行，减少观察 (0)

Question

我使用 unity ML-Agents 制作了一款回合制 RPG 游戏。我的 Unity ML-Agents 在这里作为敌人，我希望我的 Unity ML-Agents 根据他们收到的奖励进行轮流，但我仍然无法掌握其中的逻辑。

这是我的代码

EnemyAgentAI.cs

using System;
using System.Collections;
using System.Collections.Generic;
using Unity.MLAgents;
using Unity.MLAgents.Actuators;
using Unity.MLAgents.Sensors;
using Unity.MLAgents.Sensors.Reflection;
using UnityEngine;

public class EnemyAIAgent : Agent
{
    public float attackPower = 10f;
    public float weaknessMultiplier = 2f;
    public float resistanceMultiplier = 0.5f;
    public float blockMultiplier = 0.2f;

    private int NUM_ELEMENT;
    private float reward = 0f;

    public override void OnEpisodeBegin()
    {
        
    }

    public override void CollectObservations(VectorSensor sensor)
    {
        // Observe the attack element
        AttackScript attackScript = GetComponent<AttackScript>();
        NUM_ELEMENT = (int)AttackScript.magicElement.LastElement;


        if (attackScript != null)
        {
            sensor.AddObservation(NUM_ELEMENT);
        }
    }

    public void AgentAttack(ActionSegment<int> act)
    {
        AttackScript attackScript = GetComponent<AttackScript>();
        FighterStats fighterStats = GetComponent<FighterStats>();
        FighterAction fighterAction = GetComponent<FighterAction>();
        var dirToGo = Vector3.zero;
        var rotateDir = Vector3.zero;
        var physicalAttack = act[0];
        var fireAttack = act[1];

        if(physicalAttack == 1)
        {
            attackScript = GameObject.Find("EMeleePrefab").GetComponent<AttackScript>();
            fighterAction.SelectAttack("melee");
            Debug.Log("Melee attack");
        } else if (fireAttack == 1)
        {
            attackScript = GameObject.Find("ERangePrefab").GetComponent<AttackScript>();
            fighterAction.SelectAttack("range");
            Debug.Log("Range attack");
        } 

        // If the agent is trying to attack

        if (physicalAttack == 1 || fireAttack == 1)
        {
            if (attackScript.IsBlockingAttack)
            {
                // reward = -attackPower * blockMultiplier;
                reward = -1f;
            } else if (attackScript.IsResistingAttack)
            {
                // reward = -attackPower * resistanceMultiplier;
                reward = -0.5f;
            } else if (attackScript.IsWeakToAttack)
            {
                // reward = attackPower * weaknessMultiplier;
                reward = 1f;

            } else
            {
                // reward = attackPower;
                reward = 0.5f;
            }
        }

    }


    public override void OnActionReceived(ActionBuffers actions)
    {

        AgentAttack(actions.DiscreteActions);

        // Apply the reward
        AddReward(reward);
        Debug.Log("Reward: " + reward);

        EndEpisode();
    }
}

现在，我有三个问题：

1. 我希望我的特工使用攻击获得最好的奖励。我尝试在我的

   GameController.cs

   处调用OnActionReceived，但这会让我的代理做静态的事情（不选择或尝试其他攻击，只是进行物理攻击）。

using System.Collections;
using System.Collections.Generic;
using UnityEngine;
using UnityEngine.UI;
using System.Transactions;
using UnityEngine.SocialPlatforms;
using Unity.MLAgents;
using Unity.MLAgents.Actuators;
using Unity.MLAgents.Sensors;

public class GameController : MonoBehaviour
{
    private List<FighterStats> fighterStats;

    private GameObject battleMenu;

    public Text battleText;

    private void Awake()
    {
        battleMenu = GameObject.Find("ActionMenu");
    }
    void Start()
    {
        fighterStats = new List<FighterStats>();
        GameObject hero = GameObject.FindGameObjectWithTag("Hero");
        FighterStats currentFighterStats = hero.GetComponent<FighterStats>();
        currentFighterStats.CalculateNextTurn(0);
        fighterStats.Add(currentFighterStats);

        GameObject enemy = GameObject.FindGameObjectWithTag("Enemy");
        FighterStats currentEnemyStats = enemy.GetComponent<FighterStats>();
        currentEnemyStats.CalculateNextTurn(0);
        fighterStats.Add(currentEnemyStats);

        fighterStats.Sort();
        

        NextTurn();
    }

    public void NextTurn()
    {
        battleText.gameObject.SetActive(false);
        FighterStats currentFighterStats = fighterStats[0];
        fighterStats.Remove(currentFighterStats);
        if (!currentFighterStats.GetDead())
        {
            GameObject currentUnit = currentFighterStats.gameObject;
            currentFighterStats.CalculateNextTurn(currentFighterStats.nextActTurn);
            fighterStats.Add(currentFighterStats);
            fighterStats.Sort();
            if (currentUnit.CompareTag("Hero"))
            {
                battleMenu.SetActive(true);
                Debug.Log("Hero's turn");
            }
            else
            {
                battleMenu.SetActive(false);
                Debug.Log("Enemy's turn");

                // Scripted AI
                // string attackType = Random.Range(0, 2) == 1 ? "melee" : "range";
                // currentUnit.GetComponent<FighterAction>().SelectAttack(attackType);

                // ML-Agents AI
                float[] actions = new float[] { 0, 1, 0, 0, 0, 0, 0 };
                ActionBuffers actionBuffers = ActionBuffers.FromDiscreteActions(actions);
                currentUnit.GetComponent<EnemyAIAgent>().OnActionReceived(actionBuffers);
            }
        } else
        {
            NextTurn();
        }
    }
}

2. 当我在

   GameController.cs

   上使用 OnActionReceived 时，会出现警告：进行的观测 (0) 少于矢量观测大小 (1)。观察结果将被填充。

3. 然后我尝试使用 RequestDecision，但代理轮流不执行任何操作。

                ...
                // ML-Agents AI
                currentUnit.GetComponent<EnemyAIAgent>().RequestDecision();
                ...

Answer 1

我也遇到了第三个问题。现在解决了吗