UnpicklingError：遇到加载持久id指令，但未指定perpetitive_load函数

Question

我试图运行一个名为

api.py

的 python 文件。在此文件中，我正在加载使用 PyTorch 构建和训练的深度学习模型的 pickle 文件。

api.py 在

api.py

中，下面给出的函数是最重要的。

def load_model_weights(model_architecture, weights_path):
  if os.path.isfile(weights_path):
      cherrypy.log("CHERRYPYLOG Loading model from: {}".format(weights_path))
      model_architecture.load_state_dict(torch.load(weights_path))
  else:
      raise ValueError("Path not found {}".format(weights_path))

        
def load_recommender(vector_dim, hidden, activation, dropout, weights_path):

    rencoder_api = model.AutoEncoder(layer_sizes=[vector_dim] + [int(l) for l in hidden.split(',')],
                               nl_type=activation,
                               is_constrained=False,
                               dp_drop_prob=dropout,
                               last_layer_activations=False)
    load_model_weights(rencoder_api, weights_path) 
    rencoder_api.eval()
    rencoder_api = rencoder_api.cuda()
    return rencoder_api

目录结构

📦MP1
 ┣ 📂.ipynb_checkpoints
 ┃ ┗ 📜RS_netflix3months_100epochs_64,128,128-checkpoint.ipynb
 ┣ 📂data
 ┃ ┣ 📜AutoEncoder.png
 ┃ ┣ 📜collaborative_filtering.gif
 ┃ ┣ 📜movie_titles.txt
 ┃ ┗ 📜shut_up.gif
 ┣ 📂DeepRecommender
 ┃ ┣ 📂data_utils
 ┃ ┃ ┣ 📜movielens_data_convert.py
 ┃ ┃ ┗ 📜netflix_data_convert.py
 ┃ ┣ 📂reco_encoder
 ┃ ┃ ┣ 📂data
 ┃ ┃ ┃ ┣ 📂__pycache__
 ┃ ┃ ┃ ┃ ┣ 📜input_layer.cpython-37.pyc
 ┃ ┃ ┃ ┃ ┣ 📜input_layer_api.cpython-37.pyc
 ┃ ┃ ┃ ┃ ┗ 📜__init__.cpython-37.pyc
 ┃ ┃ ┃ ┣ 📜input_layer.py
 ┃ ┃ ┃ ┣ 📜input_layer_api.py
 ┃ ┃ ┃ ┗ 📜__init__.py
 ┃ ┃ ┣ 📂model
 ┃ ┃ ┃ ┣ 📂__pycache__
 ┃ ┃ ┃ ┃ ┣ 📜model.cpython-37.pyc
 ┃ ┃ ┃ ┃ ┗ 📜__init__.cpython-37.pyc
 ┃ ┃ ┃ ┣ 📜model.py
 ┃ ┃ ┃ ┗ 📜__init__.py
 ┃ ┃ ┣ 📂__pycache__
 ┃ ┃ ┃ ┗ 📜__init__.cpython-37.pyc
 ┃ ┃ ┗ 📜__init__.py
 ┃ ┣ 📂__pycache__
 ┃ ┃ ┗ 📜__init__.cpython-37.pyc
 ┃ ┣ 📜compute_RMSE.py
 ┃ ┣ 📜infer.py
 ┃ ┣ 📜run.py
 ┃ ┗ 📜__init__.py
 ┣ 📂model_save
 ┃ ┣ 📂model.epoch_99
 ┃ ┃ ┗ 📂archive
 ┃ ┃ ┃ ┣ 📂data
 ┃ ┃ ┃ ┃ ┣ 📜92901648
 ┃ ┃ ┃ ┃ ┣ 📜92901728
 ┃ ┃ ┃ ┃ ┣ 📜92901808
 ┃ ┃ ┃ ┃ ┣ 📜92901888
 ┃ ┃ ┃ ┃ ┣ 📜92901968
 ┃ ┃ ┃ ┃ ┣ 📜92902048
 ┃ ┃ ┃ ┃ ┣ 📜92902128
 ┃ ┃ ┃ ┃ ┣ 📜92902208
 ┃ ┃ ┃ ┃ ┣ 📜92902288
 ┃ ┃ ┃ ┃ ┣ 📜92902368
 ┃ ┃ ┃ ┃ ┣ 📜92902448
 ┃ ┃ ┃ ┃ ┗ 📜92902608
 ┃ ┃ ┃ ┣ 📜data.pkl
 ┃ ┃ ┃ ┗ 📜version
 ┃ ┣ 📜model.epoch_99.zip
 ┃ ┗ 📜model.onnx
 ┣ 📂Netflix
 ┃ ┣ 📂N1Y_TEST
 ┃ ┃ ┗ 📜n1y.test.txt
 ┃ ┣ 📂N1Y_TRAIN
 ┃ ┃ ┗ 📜n1y.train.txt
 ┃ ┣ 📂N1Y_VALID
 ┃ ┃ ┗ 📜n1y.valid.txt
 ┃ ┣ 📂N3M_TEST
 ┃ ┃ ┗ 📜n3m.test.txt
 ┃ ┣ 📂N3M_TRAIN
 ┃ ┃ ┗ 📜n3m.train.txt
 ┃ ┣ 📂N3M_VALID
 ┃ ┃ ┗ 📜n3m.valid.txt
 ┃ ┣ 📂N6M_TEST
 ┃ ┃ ┗ 📜n6m.test.txt
 ┃ ┣ 📂N6M_TRAIN
 ┃ ┃ ┗ 📜n6m.train.txt
 ┃ ┣ 📂N6M_VALID
 ┃ ┃ ┗ 📜n6m.valid.txt
 ┃ ┣ 📂NF_TEST
 ┃ ┃ ┗ 📜nf.test.txt
 ┃ ┣ 📂NF_TRAIN
 ┃ ┃ ┗ 📜nf.train.txt
 ┃ ┗ 📂NF_VALID
 ┃ ┃ ┗ 📜nf.valid.txt
 ┣ 📂test
 ┃ ┣ 📂testData_iRec
 ┃ ┃ ┣ 📜.part-00199-f683aa3b-8840-4835-b8bc-a8d1eaa11c78.txt.crc
 ┃ ┃ ┣ 📜part-00000-f683aa3b-8840-4835-b8bc-a8d1eaa11c78.txt
 ┃ ┃ ┣ 📜part-00003-f683aa3b-8840-4835-b8bc-a8d1eaa11c78.txt
 ┃ ┃ ┗ 📜_SUCCESS
 ┃ ┣ 📂testData_uRec
 ┃ ┃ ┣ 📜.part-00000-4a844096-8dd9-425e-9d9d-bd9062cc6940.txt.crc
 ┃ ┃ ┣ 📜._SUCCESS.crc
 ┃ ┃ ┣ 📜part-00161-4a844096-8dd9-425e-9d9d-bd9062cc6940.txt
 ┃ ┃ ┣ 📜part-00196-4a844096-8dd9-425e-9d9d-bd9062cc6940.txt
 ┃ ┃ ┗ 📜part-00199-4a844096-8dd9-425e-9d9d-bd9062cc6940.txt
 ┃ ┣ 📜data_layer_tests.py
 ┃ ┣ 📜test_model.py
 ┃ ┗ 📜__init__.py
 ┣ 📂__pycache__
 ┃ ┣ 📜api.cpython-37.pyc
 ┃ ┣ 📜load_test.cpython-37.pyc
 ┃ ┣ 📜parameters.cpython-37.pyc
 ┃ ┗ 📜utils.cpython-37.pyc
 ┣ 📜api.py
 ┣ 📜compute_RMSE.py
 ┣ 📜load_test.py
 ┣ 📜logger.py
 ┣ 📜netflix_1y_test.csv
 ┣ 📜netflix_1y_train.csv
 ┣ 📜netflix_1y_valid.csv
 ┣ 📜netflix_3m_test.csv
 ┣ 📜netflix_3m_train.csv
 ┣ 📜netflix_3m_valid.csv
 ┣ 📜netflix_6m_test.csv
 ┣ 📜netflix_6m_train.csv
 ┣ 📜netflix_6m_valid.csv
 ┣ 📜netflix_full_test.csv
 ┣ 📜netflix_full_train.csv
 ┣ 📜netflix_full_valid.csv
 ┣ 📜parameters.py
 ┣ 📜preds.txt
 ┣ 📜RS_netflix3months_100epochs_64,128,128.ipynb
 ┗ 📜utils.py

我收到这样的错误（serialization.py）。有人可以帮我解决这个错误吗？

D:\Anaconda\envs\practise\lib\site-packages\torch\serialization.py in _legacy_load(f, map_location, pickle_module, **pickle_load_args)
    762             "functionality.")
    763 
--> 764     magic_number = pickle_module.load(f, **pickle_load_args)
    765     if magic_number != MAGIC_NUMBER:
    766         raise RuntimeError("Invalid magic number; corrupt file?")

UnpicklingError: A load persistent id instruction was encountered,
but no persistent_load function was specified.

Answer 1

搜索 PyTorch 文档后，我最终将模型保存为 ONNX 格式，然后将该 ONNX 模型加载到 PyTorch 模型中并将其用于推理。

import onnx
from onnx2pytorch import ConvertModel


def load_model_weights(model_architecture, weights_path):
    if os.path.isfile("model.onnx"):
        cherrypy.log("CHERRYPYLOG Loading model from: {}".format(weights_path))
        onnx_model = onnx.load("model.onnx")
        pytorch_model = ConvertModel(onnx_model)
        ## model_architecture.load_state_dict(torch.load(weights_path))
    else:
        raise ValueError("Path not found {}".format(weights_path))

        
def load_recommender(vector_dim, hidden, activation, dropout, weights_path):

    rencoder_api = model.AutoEncoder(layer_sizes=[vector_dim] + [int(l) for l in hidden.split(',')],
                               nl_type=activation,
                               is_constrained=False,
                               dp_drop_prob=dropout,
                               last_layer_activations=False)
    load_model_weights(rencoder_api, weights_path) 
    rencoder_api.eval()
    rencoder_api = rencoder_api.cuda()
    return rencoder_api

一些有用的资源：

火炬.保存

火炬.负载

ONNX 教程