以下生成器抛出异常。
如何解决这个问题？
import os
import numpy as np
import pandas as pd
import tensorflow as tf
from sklearn.model_selection import train_test_split
import autokeras as ak
DATA_COLUMN_COUNT = 12
LABEL_COLUMN_INDEX = 2
FEATURE_START_INDEX = 3
NUM_OF_FEATURES = DATA_COLUMN_COUNT - FEATURE_START_INDEX
NUM_OF_CLASSES = 3
BATCH_SIZE = 64
EPOCHS = 111
MAX_TRIALS = 10
STEPS_PER_EPOCH = 200
VALIDATION_STEPS = 50
def load_data(file_path):
data = pd.read_csv(file_path, header=None, sep='\s+', usecols=range(0, DATA_COLUMN_COUNT))
print("Sample data with usecols and correct separator:")
print(data.head())
label_conversion = {'H': 0, 'E': 1, 'C': 2}
y = data.iloc[:, LABEL_COLUMN_INDEX].map(label_conversion)
y = tf.keras.utils.to_categorical(y, num_classes=len(label_conversion))
x = data.iloc[:, FEATURE_START_INDEX:].to_numpy()
return x, y
def data_generator(file_paths, batch_size, num_classes):
while True:
for file_path in file_paths:
x, y = load_data(file_path)
# Remove the redundant to_categorical call
# y = tf.keras.utils.to_categorical(y, num_classes)
for i in range(0, len(x), batch_size):
end = i + batch_size
batch_x = x[i:end]
batch_y = y[i:end]
if batch_x.shape[0] < batch_size:
padding = batch_size - batch_x.shape[0]
batch_x = np.pad(batch_x, ((0, padding), (0, 0)), mode='constant', constant_values=0)
# Ensure batch_y has the correct number of dimensions before padding
batch_y = np.pad(batch_y, ((0, padding), (0, 0)), mode='constant', constant_values=0)
yield batch_x, batch_y
def create_generators(folder_path, batch_size, num_classes, validation_split=0.2):
files = [os.path.join(folder_path, f) for f in os.listdir(folder_path) if f.endswith('.dat')]
train_files, validation_files = train_test_split(files, test_size=validation_split)
train_gen = data_generator(train_files, batch_size, num_classes)
validation_gen = data_generator(validation_files, batch_size, num_classes)
return train_gen, validation_gen
if __name__ == "__main__":
data_folder = r'/home/my_username/my_project_v2_original_data'
train_gen, val_gen = create_generators(data_folder, BATCH_SIZE, NUM_OF_CLASSES)
train_dataset = tf.data.Dataset.from_generator(
lambda: train_gen,
output_signature=(
tf.TensorSpec(shape=(None, NUM_OF_FEATURES), dtype=tf.float32),
tf.TensorSpec(shape=(None, NUM_OF_CLASSES), dtype=tf.float32)
)
)
validation_dataset = tf.data.Dataset.from_generator(
lambda: val_gen,
output_signature=(
tf.TensorSpec(shape=(None, NUM_OF_FEATURES), dtype=tf.float32),
tf.TensorSpec(shape=(None, NUM_OF_CLASSES), dtype=tf.float32)
)
)
clf = ak.StructuredDataClassifier(max_trials=MAX_TRIALS, overwrite=True, num_classes=NUM_OF_CLASSES)
# Fit the model
clf.fit(train_dataset, epochs=EPOCHS, steps_per_epoch=STEPS_PER_EPOCH, validation_data=validation_dataset, validation_steps=VALIDATION_STEPS)
best_hps = clf.tuner.get_best_hyperparameters()[0]
print(best_hps)
model = clf.export_model()
# Save the model
model.save("cnn_autokeras_by_chunk_without_ohe")
# Evaluate the model
evaluation = clf.evaluate(validation_dataset)
print(evaluation)
C:\Users\pc\AppData\Local\Programs\Python\Python311\python.exe C:/git/my_project_v2/my_project_v2/my_project_v2_autokeras/cnn_autokeras_by_original_by_chunk.py
Using TensorFlow backend
2024-02-12 23:14:56.551477: I tensorflow/core/platform/cpu_feature_guard.cc:182] This TensorFlow binary is optimized to use available CPU instructions in performance-critical operations.
To enable the following instructions: SSE SSE2 SSE3 SSE4.1 SSE4.2 AVX, in other operations, rebuild TensorFlow with the appropriate compiler flags.
Sample data with usecols and correct separator:
0 1 2 3 4 5 6 7 8 9 10 11
0 0 SER C 0.000 0.000 0.000 1 1 1 1 1 0
1 1 ASN C 5.651 8.805 0.000 1 1 1 1 1 0
2 2 ALA C 5.582 5.602 9.099 0 0 0 1 1 0
3 3 MET C 5.756 7.791 7.902 0 0 0 0 0 0
4 4 ILE E 6.853 10.406 11.253 0 0 0 0 0 0
Sample data with usecols and correct separator:
0 1 2 3 4 5 6 7 8 9 10 11
0 0 PRO C 0.000 0.000 0.000 1 1 1 1 1 0
1 1 ILE E 6.388 10.133 0.000 1 1 1 1 1 0
2 2 MET E 7.090 9.464 12.018 0 0 0 0 1 0
3 3 LEU E 6.281 9.841 12.931 0 0 0 0 0 0
4 4 ARG E 6.628 5.599 9.306 0 0 0 0 0 0
2024-02-12 23:14:58.258253: W tensorflow/core/framework/op_kernel.cc:1827] INVALID_ARGUMENT: ValueError: generator already executing
Traceback (most recent call last):
File "C:\Users\pc\AppData\Roaming\Python\Python311\site-packages\tensorflow\python\ops\script_ops.py", line 270, in __call__
ret = func(*args)
^^^^^^^^^^^
File "C:\Users\pc\AppData\Roaming\Python\Python311\site-packages\tensorflow\python\autograph\impl\api.py", line 643, in wrapper
return func(*args, **kwargs)
^^^^^^^^^^^^^^^^^^^^^
File "C:\Users\pc\AppData\Roaming\Python\Python311\site-packages\tensorflow\python\data\ops\from_generator_op.py", line 198, in generator_py_func
values = next(generator_state.get_iterator(iterator_id))
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
ValueError: generator already executing
Traceback (most recent call last):
File "C:\git\my_project_v2\my_project_v2\my_project_v2_autokeras\cnn_autokeras_by_original_by_chunk.py", line 97, in <module>
2024-02-12 23:14:58.260762: W tensorflow/core/framework/op_kernel.cc:1827] INVALID_ARGUMENT: ValueError: generator already executing
Tr aclf.fit(train_dataset, epochs=EPOCHS, steps_per_epoch=STEPS_PER_EPOCH, validation_data=validation_dataset, validation_steps=VALIDATION_STEPS)ce
back (most recent call last):
File " File "C:\Users\pc\AppData\Local\Programs\Python\Python311\Lib\site-packages\autokeras\tasks\structured_data.py", line 326, in fit
C:\Users\pc\AppData\Roaming\Python\Python311\site-packages\tensorflow\python\ops\script_ops.py", line 270, in __call__
ret = func(*args)
^^^^^^^^^^^
File "C:\Users\pc\AppData\Roaming\Python\Python311\site-packages\tensorflow\python\autograph\impl\api.py", line 643, in wrapper
return func(*args, **kwargs)
^^^^^^^^^^^^^^^^^^^^^
File "C:\Users\pc\AppData\Roaming\Python\Python311\site-packages\tensorflow\python\data\ops\from_generator_op.py", line 198, in generator_py_func
values = next(generator_state.get_iterator(iterator_id))
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
ValueError: generator already executing
2024-02-12 23:14:58.263112: W tensorflow/core/framework/op_kernel.cc:1827] INVALID_ARGUMENT: ValueError: generator already executing
Traceback (most recent call last):
File "C:\Users\pc\AppData\Roaming\Python\Python311\site-packages\tensorflow\python\ops\script_ops.py", line 270, in __call__
ret = func(*args)
^^^^^^^^^^^
File "C:\Users\pc\AppData\Roaming\Python\Python311\site-packages\tensorflow\python\autograph\impl\api.py", line 643, in wrapper
return func(*args, **kwargs)
^^^^^^^^^^^^^^^^^^^^^
File "C:\Users\pc\AppData\Roaming\Python\Python311\site-packages\tensorflow\python\data\ops\from_generator_op.py", line 198, in generator_py_func
values = next(generator_state.get_iterator(iterator_id))
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
ValueError: generator already executing
2024-02-12 23:14:58.265071: W tensorflow/core/framework/op_kernel.cc:1827] INVALID_ARGUMENT: ValueError: generator already executing
Traceback (most recent call last):
File "C:\Users\pc\AppData\Roaming\Python\Python311\site-packages\tensorflow\python\ops\script_ops.py", line 270, in __call__
ret = func(*args)
^^^^^^^^^^^
File "C:\Users\pc\AppData\Roaming\Python\Python311\site-packages\tensorflow\python\autograph\impl\api.py", line 643, in wrapper
return func(*args, **kwargs)
^^^^^^^^^^^^^^^^^^^^^
File "C:\Users\pc\AppData\Roaming\Python\Python311\site-packages\tensorflow\python\data\ops\from_generator_op.py", line 198, in generator_py_func
values = next(generator_state.get_iterator(iterator_id))
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
ValueError: generator already executing
2024-02-12 23:14:58.266913: W tensorflow/core/framework/op_kernel.cc:1827] INVALID_ARGUMENT: ValueError: generator already executing
Traceback (most recent call last):
File "C:\Users\pc\AppData\Roaming\Python\Python311\site-packages\tensorflow\python\ops\script_ops.py", line 270, in __call__
ret = func(*args)
^^^^^^^^^^^
File "C:\Users\pc\AppData\Roaming\Python\Python311\site-packages\tensorflow\python\autograph\impl\api.py", line 643, in wrapper
return func(*args, **kwargs)
^^^^^^^^^^^^^^^^^^^^^
File "C:\Users\pc\AppData\Roaming\Python\Python311\site-packages\tensorflow\python\data\ops\from_generator_op.py", line 198, in generator_py_func
values = next(generator_state.get_iterator(iterator_id))
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
ValueError: generator already executing
2024-02-12 23:14:58.271889: W tensorflow/core/framework/op_kernel.cc:1827] INVALID_ARGUMENT: ValueError: generator already executing
Traceback (most recent call last):
File "C:\Users\pc\AppData\Roaming\Python\Python311\site-packages\tensorflow\python\ops\script_ops.py", line 270, in __call__
ret = func(*args)
^^^^^^^^^^^
File "C:\Users\pc\AppData\Roaming\Python\Python311\site-packages\tensorflow\python\autograph\impl\api.py", line 643, in wrapper
return func(*args, **kwargs)
^^^^^^^^^^^^^^^^^^^^^
File "C:\Users\pc\AppData\Roaming\Python\Python311\site-packages\tensorflow\python\data\ops\from_generator_op.py", line 198, in generator_py_func
values = next(generator_state.get_iterator(iterator_id))
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
ValueError: generator already executing
2024-02-12 23:14:58.274866: W tensorflow/core/framework/op_kernel.cc:1827] INVALID_ARGUMENT: ValueError: generator already executing
Traceback (most recent call last):
File "C:\Users\pc\AppData\Roaming\Python\Python311\site-packages\tensorflow\python\ops\script_ops.py", line 270, in __call__
ret = func(*args)
^^^^^^^^^^^
File "C:\Users\pc\AppData\Roaming\Python\Python311\site-packages\tensorflow\python\autograph\impl\api.py", line 643, in wrapper
return func(*args, **kwargs)
^^^^^^^^^^^^^^^^^^^^^
File "C:\Users\pc\AppData\Roaming\Python\Python311\site-packages\tensorflow\python\data\ops\from_generator_op.py", line 198, in generator_py_func
values = next(generator_state.get_iterator(iterator_id))
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
ValueError: generator already executing
history = super().fit(
^^^^^^^^^^^^
File "C:\Users\pc\AppData\Local\Programs\Python\Python311\Lib\site-packages\autokeras\tasks\structured_data.py", line 139, in fit
history = super().fit(
^^^^^^^^^^^^
File "C:\Users\pc\AppData\Local\Programs\Python\Python311\Lib\site-packages\autokeras\auto_model.py", line 283, in fit
self._analyze_data(dataset)
File "C:\Users\pc\AppData\Local\Programs\Python\Python311\Lib\site-packages\autokeras\auto_model.py", line 369, in _analyze_data
for x, y in dataset:
File "C:\Users\pc\AppData\Roaming\Python\Python311\site-packages\tensorflow\python\data\ops\iterator_ops.py", line 809, in __next__
return self._next_internal()
^^^^^^^^^^^^^^^^^^^^^
File "C:\Users\pc\AppData\Roaming\Python\Python311\site-packages\tensorflow\python\data\ops\iterator_ops.py", line 772, in _next_internal
ret = gen_dataset_ops.iterator_get_next(
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "C:\Users\pc\AppData\Roaming\Python\Python311\site-packages\tensorflow\python\ops\gen_dataset_ops.py", line 3055, in iterator_get_next
Sample data with usecols and correct separator:
0 1 2 3 4 5 6 7 8 9 10 11
0 0 MET C 0.000 0.000 0.000 1 1 1 1 1 0
1 1 THR C 7.281 10.424 0.000 1 1 1 1 1 0
2 2 PHE C 6.626 8.966 12.663 0 0 0 0 1 0
3 3 GLU C 6.220 9.739 12.658 0 0 0 0 0 0
4 4 LEU E 6.748 10.167 12.863 0 0 0 0 0 0
_ops.raise_from_not_ok_status(e, name)
File "C:\Users\pc\AppData\Roaming\Python\Python311\site-packages\tensorflow\python\framework\ops.py", line 5888, in raise_from_not_ok_status
raise core._status_to_exception(e) from None # pylint: disable=protected-access
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
tensorflow.python.framework.errors_impl.InvalidArgumentError: {{function_node __wrapped__IteratorGetNext_output_types_2_device_/job:localhost/replica:0/task:0/device:CPU:0}} ValueError: generator already executing
Traceback (most recent call last):
File "C:\Users\pc\AppData\Roaming\Python\Python311\site-packages\tensorflow\python\ops\script_ops.py", line 270, in __call__
ret = func(*args)
^^^^^^^^^^^
File "C:\Users\pc\AppData\Roaming\Python\Python311\site-packages\tensorflow\python\autograph\impl\api.py", line 643, in wrapper
return func(*args, **kwargs)
^^^^^^^^^^^^^^^^^^^^^
File "C:\Users\pc\AppData\Roaming\Python\Python311\site-packages\tensorflow\python\data\ops\from_generator_op.py", line 198, in generator_py_func
values = next(generator_state.get_iterator(iterator_id))
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
ValueError: generator already executing
[[{{node PyFunc}}]] [Op:IteratorGetNext] name:
2024-02-12 23:14:58.675970: W tensorflow/core/kernels/data/generator_dataset_op.cc:108] Error occurred when finalizing GeneratorDataset iterator: FAILED_PRECONDITION: Python interpreter state is not initialized. The process may be terminated.
[[{{node PyFunc}}]]
2024-02-12 23:14:58.676507: W tensorflow/core/kernels/data/generator_dataset_op.cc:108] Error occurred when finalizing GeneratorDataset iterator: FAILED_PRECONDITION: Python interpreter state is not initialized. The process may be terminated.
[[{{node PyFunc}}]]
Process finished with exit code 1
Dataset.from_generator()train_genargstrain_gen, val_gen = create_generators(data_folder, BATCH_SIZE, NUM_OF_CLASSES)
train_dataset = tf.data.Dataset.from_generator(
data_generator,
args=(tf.constant(train_files), tf.constant(batch_size), tf.constant(num_classes))
output_signature=(
tf.TensorSpec(shape=(None, NUM_OF_FEATURES), dtype=tf.float32),
tf.TensorSpec(shape=(None, NUM_OF_CLASSES), dtype=tf.float32)
)
)
此外，我不会在生成器内批量处理数据。
Dataset.batch(size)padded_batches