如何在 CNTK (C#) 中为循环网络创建输入数据？

Question

我需要为形状为 (5(input_vector), 50(sequence_length), 100(amount_of_input_sequences)) 的小批量创建数据，以便在 1 次模型运行中运行循环神经网络的 100 个输入数据实例.评估.

目前我无法创建输入数据，即使是 1 次运行。我的代码生成神经网络 50 次运行的输出，而不是将数据用作单个序列。 我没有找到使用 Value 对象为 RNN 创建输入数据的示例。我也没有找到创建具有三个轴的输入张量的示例。

如果有人告诉我如何做到这一点，我会很高兴。

`var cpu_device = DeviceDescriptor.CPUDevice;
var gpu_device = DeviceDescriptor.GPUDevice(0);
var model_device = gpu_device;

int inputDim = 5;
int cellDim = 5;
int outputDim = 3;
int sequenceLength = 50;
int inputSequences = 100;

NDShape inputShape = NDShape.CreateNDShape(new int[] { inputDim });
NDShape outputShape = NDShape.CreateNDShape(new int[] { outputDim });

var inputVariable = Variable.InputVariable(inputShape, DataType.Float);
var outputVariable = Variable.InputVariable(outputShape, DataType.Float);

var lstmLayer = CntkWrapper.Layers.LSTM<float>(cellDim, inputVariable, model_device);
var model = CntkWrapper.Layers.Dense<float>(outputDim, lstmLayer, CNTKLib.Sigmoid, model_device);

Random random = new Random();
var inputs = new float[inputSequences][][];
for(int i = 0; i < inputs.Length; i++)
{
    inputs[i] = new float[sequenceLength][];
    for(int k = 0; k < inputs[i].Length; k++)
    {
        inputs[i][k] = new float[inputDim];
        for(int p = 0; p < inputs[i][k].Length; p++)
        {
            inputs[i][k][p] = (float)random.NextDouble();
        }
    }
}

NDArrayView[] sequenceNDArrayView = inputs[0].Select(o => new NDArrayView(inputShape, o, cpu_device)).ToArray();
var gpuInputsValue = Value.Create(inputShape, sequenceNDArrayView, gpu_device);

var inputDataMap = new Dictionary<Variable, Value>() { { inputVariable, gpuInputsValue } };
var outputDataMap = new Dictionary<Variable, Value>() { { model.Output, null } };
model.Evaluate(inputDataMap, outputDataMap, gpu_device);
var outputValue = outputDataMap[model.Output];
IList<IList<float>> actualLabelSoftMax = outputValue.GetDenseData<float>(model.Output);`

`public static class Layers
{
    public static uint Seed = 1;
    public static double NormalMean = 0.0;
    public static double StandartDev = 0.25;

    public static Function Dense<ElementType>(int dim, Variable previousLayer, Func<Variable, Function> activationFunction, DeviceDescriptor device)
    {
        int inputDimension = previousLayer.Shape.Dimensions[0];
        var weights = new Parameter(NDArrayView.RandomNormal<ElementType>(new[] { dim, inputDimension }, NormalMean, StandartDev, Seed++, device));
        var bias = new Parameter(NDArrayView.RandomNormal<ElementType>(new[] { dim }, NormalMean, StandartDev, Seed++, device));
        var layer = activationFunction(CNTKLib.Plus(bias, CNTKLib.Times(weights, previousLayer)));
        return layer;
    }
    public static Function LSTM<ElementType>(int cellDim, Variable previousLayer, DeviceDescriptor device)
    {
        Func<Variable, Function> pastValueRecurrenceHook = (x) => CNTKLib.PastValue(x);
        var dh = Variable.PlaceholderVariable(new int[] { cellDim }, previousLayer.DynamicAxes);
        var dc = Variable.PlaceholderVariable(new int[] { cellDim }, previousLayer.DynamicAxes);

        var prevOutput = dh;
        var prevCellState = dc;
        int outputDim = prevOutput.Shape[0];

        bool isFloatType = typeof(ElementType).Equals(typeof(float));
        DataType dataType = isFloatType ? DataType.Float : DataType.Double;

        Func<int, Parameter> createBiasParam;
        if (isFloatType)
            createBiasParam = (dim) => new Parameter(new int[] { dim }, 0.01f, device, "");
        else
            createBiasParam = (dim) => new Parameter(new int[] { dim }, 0.01, device, "");

        uint seed2 = 1;
        Func<int, Parameter> createProjectionParam = (oDim) => new Parameter(new int[] { oDim, NDShape.InferredDimension }, dataType, CNTKLib.GlorotUniformInitializer(1.0, 1, 0, seed2++), device);

        Func<int, Parameter> createDiagWeightParam = (dim) => new Parameter(new int[] { dim }, dataType, CNTKLib.GlorotUniformInitializer(1.0, 1, 0, seed2++), device);

        Func<Variable> projectInput = () => createBiasParam(cellDim) + (createProjectionParam(cellDim) * previousLayer);

        // Input gate
        Function it =
            CNTKLib.Sigmoid((Variable)(projectInput() + (createProjectionParam(cellDim) * prevOutput)) + CNTKLib.ElementTimes(createDiagWeightParam(cellDim), prevCellState));
        Function bit = CNTKLib.ElementTimes(it, CNTKLib.Tanh(projectInput() + (createProjectionParam(cellDim) * prevOutput)));

        // Forget-me-not gate
        Function ft = CNTKLib.Sigmoid((Variable)(projectInput() + (createProjectionParam(cellDim) * prevOutput)) + CNTKLib.ElementTimes(createDiagWeightParam(cellDim), prevCellState));
        Function bft = CNTKLib.ElementTimes(ft, prevCellState);

        Function ct = (Variable)bft + bit;

        // Output gate
        Function ot = CNTKLib.Sigmoid((Variable)(projectInput() + (createProjectionParam(cellDim) * prevOutput)) + CNTKLib.ElementTimes(createDiagWeightParam(cellDim), ct));
        Function ht = CNTKLib.ElementTimes(ot, CNTKLib.Tanh(ct));

        Function c = ct;
        Function h = (outputDim != cellDim) ? (createProjectionParam(outputDim) * ht) : ht;

        var LSTMCell = new Tuple<Function, Function>(h, c);

        var actualDh = pastValueRecurrenceHook(LSTMCell.Item1);
        var actualDc = pastValueRecurrenceHook(LSTMCell.Item2);

        (LSTMCell.Item1).ReplacePlaceholders(new Dictionary<Variable, Variable> { { dh, actualDh }, { dc, actualDc } });

        Function LSTMFunction = LSTMCell.Item1;

        Function layer = CNTKLib.SequenceLast(LSTMFunction);
        return layer;
    }
}`

Answer 1

我发现这个例子对我帮助很大：https://github.com/albertalrisa/cntk-csharp-rnn/blob/master/playground/CNTKRefresh/CharRNNCNTK/Program.cs

输入序列存储为输入向量序列，例如要存储10个长度为5的输入向量的输入序列，需要将所有输入向量组合成一个长度为50个元素的向量。

Value.CreateSequence

用于创建单个输入序列，

Value.CreateBatchOfSequences

用于创建包含多个序列的输入。

我以为

Value.Create

只接受

IEnumerable<NDArrayView>

，而构造函数中的

NDArrayView

接受一维

float[]

数组，也就是说，不可能向其中传递几个序列。为此，您需要使用

List<List<float>>

并将其传递给

Value.CreateBatchOfSequences

。

此外，序列长度轴是动态轴，因为它可以具有可变长度。在示例中，它创建为：

var axis = new Axis("inputAxis");
var inputVariable = Variable.InputVariable(inputShape, DataType.Float, "inputVariable", new List<Axis> { axis, Axis.DefaultBatchAxis() });
var outputVariable = Variable.InputVariable(outputShape, DataType.Float, "outputVariable", new List<Axis> { axis, Axis.DefaultBatchAxis() });

但没有它也能工作，我不知道为什么。

因此，使用随机值创建多个输入序列并通过循环神经网络运行它们将如下所示：

var cpu_device = DeviceDescriptor.CPUDevice;
var gpu_device = DeviceDescriptor.GPUDevice(0);
var model_device = gpu_device;

int inputDim = 5;
int cellDim = 5;
int outputDim = 3;
int sequenceLength = 50;
int sequencesCount = 100;

NDShape inputShape = NDShape.CreateNDShape(new int[] { inputDim });
NDShape outputShape = NDShape.CreateNDShape(new int[] { outputDim });

var axis = new Axis("inputAxis");
var inputVariable = Variable.InputVariable(inputShape, DataType.Float, "inputVariable", new List<Axis> { axis, Axis.DefaultBatchAxis() });
var outputVariable = Variable.InputVariable(outputShape, DataType.Float, "outputVariable", new List<Axis> { axis, Axis.DefaultBatchAxis() });

var lstmLayer = CntkWrapper.Layers.LSTM<float>(cellDim, inputVariable, model_device);
var model = CntkWrapper.Layers.Dense<float>(outputDim, lstmLayer, CNTKLib.Sigmoid, model_device);

Random random = new Random();
List<List<float>> inputSequences = new List<List<float>>();
for (int i = 0; i < sequencesCount; i++)
{
    List<float> sequence = new List<float>();
    for (int k = 0; k < sequenceLength; k++)
    {
        float[] input_vector = new float[inputDim];
        for (int n = 0; n < input_vector.Length; n++)
        {
            input_vector[n] = (float)random.NextDouble();
        }
        sequence.AddRange(input_vector);
    }
    inputSequences.Add(sequence);
}

var gpuInputSequences = Value.CreateBatchOfSequences(inputShape, inputSequences, gpu_device);

var inputDataMap = new Dictionary<Variable, Value>() { { inputVariable, gpuInputSequences } };
var outputDataMap = new Dictionary<Variable, Value>() { { model.Output, null } };
model.Evaluate(inputDataMap, outputDataMap, gpu_device);
var outputValue = outputDataMap[model.Output];
IList<IList<float>> actualLabelSoftMax = outputValue.GetDenseData<float>(model.Output);