我试图以分布式方式训练分类模型。我正在使用雅虎开发的TensorflowOnSpark库。我使用github link的例子
我正在使用除了mnist之外的数据集，它在github链接中提到的示例中使用。我正在使用的这个数据集在预处理（260000,28047）之后具有如下尺寸，并且类（标签）的范围从0:13。
>>> import os >>> import tensorflow as tf >>> from tensorflow.python import keras >>> from tensorflow.python.keras import backend as K >>> from tensorflow.python.keras.models import Sequential, load_model, save_model >>> from tensorflow.python.keras.layers import Dense, Dropout >>> from tensorflow.python.keras.callbacks import LambdaCallback, TensorBoard >>> from tensorflow.python.saved_model import builder as saved_model_builder >>> from tensorflow.python.saved_model import tag_constants >>> from tensorflow.python.saved_model.signature_def_utils_impl import predict_signature_def >>> from tensorflowonspark import TFNode >>> from pyspark.context import SparkContext >>> from pyspark.conf import SparkConf >>> from tensorflowonspark import TFCluster >>> import numpy >>> >>> ... >>> >>> data = sc.textFile('ReducedFeatures.tsv') >>> >>> data = data.map(lambda l: l.encode("UTF8", "ignore").split('\t')) >>> >>> labels = data.map(lambda x: x[1]) >>> data = data.map(lambda x: x[19:28066]) >>> >>> header = data.first() >>> data = data.filter(lambda line: line != header) >>> label_header = labels.first() >>> labels = labels.filter(lambda line: line != label_header) >>> >>> #convert values to float ... convertToFloat = lambda data: [float(str(x)) for x in data] >>> dataset = data.map(convertToFloat) >>> labels = labels.map(lambda x:float(x)) >>> >>> >>> labels = labels.map(lambda x: keras.utils.to_categorical(x, num_classes=14)) >>> >>> >>> # zip the data as tuple ... dataRDD = dataset.zip(labels) >>> >>> sampledata = dataRDD.take(20) >>> >>> model = Sequential() >>> model.add(Dense(512, activation='relu', input_shape=(28047,))) >>> model.add(Dropout(0.2)) >>> model.add(Dense(512, activation='relu')) >>> model.add(Dropout(0.2)) >>> model.add(Dense(14, activation='softmax')) >>> >>> model.summary() _________________________________________________________________ Layer (type) Output Shape Param # ================================================================= dense (Dense) (None, 512) 14360576 _________________________________________________________________ dropout (Dropout) (None, 512) 0 _________________________________________________________________ dense_1 (Dense) (None, 512) 262656 _________________________________________________________________ dropout_1 (Dropout) (None, 512) 0 _________________________________________________________________ dense_2 (Dense) (None, 14) 7182 ================================================================= Total params: 14,630,414 Trainable params: 14,630,414 Non-trainable params: 0 _________________________________________________________________ >>> >>> model.compile(loss='sparse_categorical_crossentropy', ... optimizer=tf.train.RMSPropOptimizer(learning_rate=0.001), ... metrics=['accuracy']) >>> >>> print("model.inputs: {}".format(model.inputs)) model.inputs: [<tf.Tensor 'dense_input:0' shape=(?, 28047) dtype=float32>] >>> print("model.outputs: {}".format(model.outputs)) model.outputs: [<tf.Tensor 'dense_2/Softmax:0' shape=(?, 14) dtype=float32>] >>> >>> def generate_rdd_data(dataRDD): ... while True: ... feature_vector = [] ... lbls = [] ... for item in dataRDD: ... #record = item[0] ... feature_vector.append(item[0]) ... lbls.append(item[1]) ... features = numpy.array(feature_vector).astype('float32') ... labels = numpy.stack(lbls).astype('float32') ... return (features, labels) ... >>> >>> feat, lbls = generate_rdd_data(sampledata) >>> lbls array([[0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 1., 0.], [1., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.], [0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 1., 0., 0.], [1., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.], [0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 1., 0.], [1., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.], [1., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.], [0., 0., 0., 1., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.], [0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 1., 0., 0., 0.], [0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 1., 0.], [0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 1., 0., 0., 0.], [0., 0., 1., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.], [1., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.], [0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 1., 0., 0., 0.], [0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 1., 0., 0.], [1., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.], [1., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.], [0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 1., 0., 0., 0.], [1., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.], [0., 0., 0., 1., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.]], dtype=float32) >>> >>> >>> x_train = tf.placeholder(tf.float32, [None, 28047], name="x_train") >>> y_train = tf.placeholder(tf.float32, [None, 14], name="y_train") >>> ... >>> model.fit_generator(generator=generate_rdd_data(sampledata),steps_per_epoch=200,epochs=5,verbose=1,validation_data=(x_train, y_train))
另外请看下面的回溯，我为标签做了OneHotEncoding，同时在添加输出图层时它是14，你可以在代码和模型摘要中看到它。
> Traceback (most recent call last):
> File "<stdin>", line 1, in <module>
> File "/usr/lib/python2.7/site-packages/tensorflow/python/keras/engine/training.py",
> line 2177, in fit_generator
> initial_epoch=initial_epoch)
> File "/usr/lib/python2.7/site-packages/tensorflow/python/keras/engine/training_generator.py",
> line 104, in fit_generator
> val_x, val_y, val_sample_weights)
> File "/usr/lib/python2.7/site-packages/tensorflow/python/keras/engine/training.py",
> line 992, in _standardize_user_data
> class_weight, batch_size)
> File "/usr/lib/python2.7/site-packages/tensorflow/python/keras/engine/training.py",
> line 1154, in _standardize_weights
> exception_prefix='target')
> File "/usr/lib/python2.7/site-packages/tensorflow/python/keras/engine/training_utils.py",
> line 332, in standardize_input_data
> ' but got array with shape ' + str(data_shape))
> ValueError: Error when checking target: expected dense_2 to have shape (1,) but got array with shape (14,)
但是当我开始使用生成器方法的输出进行训练时。我得到了类的以下维度错误。请帮忙
正如@Matias在评论中指出的那样，你使用了错误的丢失函数
当输出为0,1,2,3，...等整数时，使用稀疏交叉熵。但是你的输出是onehot编码的[0,0，... 1,0]。
所以使用分类交叉熵。
model.compile(loss='categorical_crossentropy',optimizer=tf.train.RMSPropOptimizer(learning_rate=0.001),metrics=['accuracy'])
链接以便更好地了解这些损失
https://jovianlin.io/cat-crossentropy-vs-sparse-cat-crossentropy/
更新：
如果你想使用稀疏交叉熵，你的模型的最后一层应该是
Dense(1, activation="Softmax")
y标签的格式应为0,1,2，... 13