最近，我正在尝试使用TFRecords训练tf.keras模型。由于对于TensorFlow 2，最有效的方法是使用tf.data API，因此我尝试使用它来训练我的MTCNN Keras模型。但是，这让我感到困惑：
根据原始论文，不同的样本（正负，负，局部脸部，界标）参加了培训的不同部分。每种样本在每个微型批次中都有特定的比率，即对于一个微型批次，pos，neg，面部和界标样本的比率应为1：3：1：2。
例如，当我制作了四个TFRecords样本后，我将需要从pos tfrecords中提取128个样本，从neg中提取384个样本，从面部提取128个样本，从地标tfrecords提取256个样本，然后将它们混合成一个mini -批量。然后，我需要在训练之前先对这批产品进行洗牌。
我真的不知道在使用TFRecords和tf.data API时如何进行此操作。现在，我只能通过逐批读取图像和标签来实现所有这些步骤，但是培训太慢了。所以我想知道是否有任何有效的方法来做到这一点。
任何建议都值得赞赏。
更新2020.04.04 10:40感谢@AAudibert，我认为他/她的答案有效，而且我也找到了实现此问题的一种方法。这是供参考的代码：
raw_pos_dataset = tf.data.TFRecordDataset(POS_TFRECORDS_PATH_LIST)
raw_neg_dataset = tf.data.TFRecordDataset(NEG_TFRECORDS_PATH_LIST)
raw_part_dataset = tf.data.TFRecordDataset(PART_TFRECORDS_PATH_LIST)
raw_landmark_dataset = tf.data.TFRecordDataset(LANDMARK_TFRECORDS_PATH_LIST)
image_feature_description = {
'height': tf.io.FixedLenFeature([], tf.int64),
'width': tf.io.FixedLenFeature([], tf.int64),
'depth': tf.io.FixedLenFeature([], tf.int64),
'info': tf.io.FixedLenFeature([17], tf.float32),
'image_raw': tf.io.FixedLenFeature([], tf.string),
}
def _read_tfrecord(serialized_example):
example = tf.io.parse_single_example(serialized_example, image_feature_description)
img = tf.image.decode_jpeg(example['image_raw'], channels = 3) # RGB rather than BGR!!!
img = (tf.cast(img, tf.float32) - 127.5) / 128.
img_shape = [example['height'], example['width'], example['depth']]
img = tf.reshape(img, img_shape)
info = example['info']
return img, info
parsed_pos_dataset = raw_pos_dataset.map(_read_tfrecord)
parsed_neg_dataset = raw_neg_dataset.map(_read_tfrecord)
parsed_part_dataset = raw_part_dataset.map(_read_tfrecord)
parsed_landmark_dataset = raw_landmark_dataset.map(_read_tfrecord)
parsed_image_dataset = tf.data.Dataset.zip((parsed_pos_dataset.repeat().shuffle(16384).batch(int(BATCH_SIZE * DATA_COMPOSE_RATIO[0])),
parsed_neg_dataset.repeat().shuffle(16384).batch(int(BATCH_SIZE * DATA_COMPOSE_RATIO[1])),
parsed_part_dataset.repeat().shuffle(16384).batch(int(BATCH_SIZE * DATA_COMPOSE_RATIO[2])),
parsed_landmark_dataset.repeat().shuffle(16384).batch(int(BATCH_SIZE * DATA_COMPOSE_RATIO[3]))))
def concatenate(pos_info, neg_info, part_info, landmark_info):
img_tensor = tf.zeros((0, IMG_SIZE, IMG_SIZE, 3), dtype = tf.float32)
label_tensor = tf.zeros((0, 17), dtype = tf.float32)
pos_img = pos_info[0]
neg_img = neg_info[0]
part_img = part_info[0]
landmark_img = landmark_info[0]
pos_info = pos_info[1]
neg_info = neg_info[1]
part_info = part_info[1]
landmark_info = landmark_info[1]
img_tensor = tf.concat([img_tensor, pos_img, neg_img, part_img, landmark_img], axis = 0)
info_tensor = tf.concat([label_tensor, pos_info, neg_info, part_info, landmark_info], axis = 0)
return img_tensor, info_tensor
ds = parsed_image_dataset.map(concatenate)
@ AAudibert的解决方案和我的问题都会发生，如果数据集不平衡（或者可能是由其他原因引起的），则损失变为'nan'。这是我给自己写的风俗习惯损失，在我将其投入培训之前已得到逐步验证：
def custom_loss(y_true, y_pred, loss_weights = loss_weights):
zero_index = K.zeros_like(y_true[:, 0])
ones_index = K.ones_like(y_true[:, 0])
# Classifier
labels = y_true[:, 0]
class_preds = y_pred[:, 0]
bi_crossentropy_loss = -labels * K.log(class_preds) - (1 - labels) * K.log(1 - class_preds)
classify_valid_index = tf.where(K.less(y_true[:, 0], 0), zero_index, ones_index)
classify_keep_num = K.cast(tf.cast(tf.reduce_sum(classify_valid_index), tf.float32) * SAMPLE_KEEP_RATIO, dtype = tf.int32)
# For classification problem, only pick 70% of the valid samples.
classify_loss_sum = bi_crossentropy_loss * tf.cast(classify_valid_index, bi_crossentropy_loss.dtype)
classify_loss_sum_filtered, _ = tf.nn.top_k(classify_loss_sum, k = classify_keep_num)
if classify_loss_sum_filtered.shape == []:
classify_loss = tf.constant(0, dtype = tf.float32)
else:
classify_loss = K.mean(classify_loss_sum_filtered)
# Bounding box regressor
rois = y_true[:, 1: 5]
roi_preds = y_pred[:, 1: 5]
roi_raw_mean_square_error = K.sum(K.square(rois - roi_preds), axis = 1) # mse
roi_valid_index = tf.where(K.equal(K.abs(y_true[:, 0]), 1), ones_index, zero_index)
roi_keep_num = K.cast(tf.reduce_sum(roi_valid_index), dtype = tf.int32)
roi_valid_mean_square_error = roi_raw_mean_square_error * tf.cast(roi_valid_index, roi_raw_mean_square_error.dtype)
roi_filtered_mean_square_error, _ = tf.nn.top_k(roi_valid_mean_square_error, k = roi_keep_num)
if roi_filtered_mean_square_error.shape == []:
roi_loss = tf.constant(0, dtype = tf.float32)
else:
roi_loss = K.mean(roi_filtered_mean_square_error)
# Landmark regressor
pts = y_true[:, 5: 17]
pt_preds = y_pred[:, 5: 17]
pts_raw_mean_square_error = K.sum(K.square(pts - pt_preds), axis = 1) # mse
pts_valid_index = tf.where(K.equal(y_true[:, 0], -2), ones_index, zero_index)
pts_keep_num = K.cast(tf.reduce_sum(pts_valid_index), dtype = tf.int32)
pts_valid_mean_square_error = pts_raw_mean_square_error * tf.cast(pts_valid_index, tf.float32)
pts_filtered_mean_square_error, _ = tf.nn.top_k(pts_valid_mean_square_error, k = pts_keep_num)
if len(pts_filtered_mean_square_error.shape) == 0:
pts_loss = tf.constant(0, dtype = tf.float32)
else:
pts_loss = K.mean(pts_filtered_mean_square_error)
loss = classify_loss * loss_weights[0] + roi_loss * loss_weights[1] + pts_loss * loss_weights[2]
return loss
或者如果我可以继续检查训练期间的中间变量变化，也许可以解决。
您可以使用sample_from_datasets根据指定的比例进行采样>
请参见this colab为例。我还复制了下面的代码。
import tensorflow as tf
batch_size = 8
pos = tf.data.Dataset.range(0, 100)
neg = tf.data.Dataset.range(100, 200)
part_face = tf.data.Dataset.range(200, 300)
landmark = tf.data.Dataset.range(300, 400)
dataset = tf.data.experimental.sample_from_datasets(
[pos, neg, part_face, landmark], [1/7, 3/7, 1/7, 2/7])
dataset = dataset.batch(batch_size)
# Optionally shuffle data further. Samples will already be interspersed between datasets.
# dataset = dataset.map(lambda batch: tf.random.shuffle(batch))
for elem in dataset:
print(elem.numpy())
# Prints
[200 300 100 201 301 302 101 303]
[ 0 304 202 102 203 103 305 104]
[306 307 105 204 308 205 206 1]
[207 309 106 107 310 108 311 312]
[208 209 210 2 109 211 110 212]
...