Pandarallel 在 Openai 升级时失败并出现 SSLContext 错误

Question

我已经将openai从0.28.0升级到openai==1.23.5。

我使用 Pandarallel 对 openai 的并行调用在 openai==0.28.0 版本上运行良好。

但是升级到 openai==1.23.5 后失败并出现以下错误

File "/app/imssumm/Summ_parallel.py", line 239, in call_iterative_summ_logic

prompt_df_1["result"] = prompt_df_1.parallel_apply(lambda x: 
self.summarize(x["to_be_summarized"],x["token_len"]), axis=1)

File "/usr/local/lib/python3.8/site-packages/pandarallel/core.py", line 265, in closure
dilled_user_defined_function = dill.dumps(user_defined_function)

File "/usr/local/lib/python3.8/site-packages/dill/_dill.py", line 263, in dumps
dump(obj, file, protocol, byref, fmode, recurse, **kwds)#, strictio)

File "/usr/local/lib/python3.8/site-packages/dill/_dill.py", line 235, in dump
Pickler(file, protocol, **_kwds).dump(obj)

File "/usr/local/lib/python3.8/site-packages/dill/_dill.py", line 394, in dump
StockPickler.dump(self, obj)

File "/usr/lib64/python3.8/pickle.py", line 487, in dump
self.save(obj)
File "/usr/local/lib/python3.8/site-packages/dill/_dill.py", line 388, in save
StockPickler.save(self, obj, save_persistent_id)

File "/usr/lib64/python3.8/pickle.py", line 560, in save
f(self, obj) # Call unbound method with explicit self
File "/usr/local/lib/python3.8/site-packages/dill/_dill.py", line 1824, in save_function
_save_with_postproc(pickler, (_create_function, (
File "/usr/local/lib/python3.8/site-packages/dill/_dill.py", line 1089, in _save_with_postproc
pickler.save_reduce(*reduction)
File "/usr/lib64/python3.8/pickle.py", line 692, in save_reduce
save(args)
File "/usr/local/lib/python3.8/site-packages/dill/_dill.py", line 388, in save
StockPickler.save(self, obj, save_persistent_id)
File "/usr/lib64/python3.8/pickle.py", line 560, in save
f(self, obj) # Call unbound method with explicit self
File "/usr/lib64/python3.8/pickle.py", line 886, in save_tuple
save(element)
File "/usr/local/lib/python3.8/site-packages/dill/_dill.py", line 388, in save
StockPickler.save(self, obj, save_persistent_id)
File "/usr/lib64/python3.8/pickle.py", line 603, in save
self.save_reduce(obj=obj, *rv)
File "/usr/lib64/python3.8/pickle.py", line 717, in save_reduce
save(state)
File "/usr/local/lib/python3.8/site-packages/dill/_dill.py", line 388, in save
StockPickler.save(self, obj, save_persistent_id)
File "/usr/lib64/python3.8/pickle.py", line 560, in save
f(self, obj) # Call unbound method with explicit self
File "/usr/local/lib/python3.8/site-packages/dill/_dill.py", line 1186, in save_module_dict
StockPickler.save_dict(pickler, obj)
File "/usr/lib64/python3.8/pickle.py", line 971, in save_dict
self._batch_setitems(obj.items())
File "/usr/lib64/python3.8/pickle.py", line 997, in _batch_setitems
save(v)
File "/usr/local/lib/python3.8/site-packages/dill/_dill.py", line 388, in save
StockPickler.save(self, obj, save_persistent_id)
File "/usr/lib64/python3.8/pickle.py", line 603, in save
self.save_reduce(obj=obj, *rv)
File "/usr/lib64/python3.8/pickle.py", line 717, in save_reduce
save(state)
File "/usr/local/lib/python3.8/site-packages/dill/_dill.py", line 388, in save
StockPickler.save(self, obj, save_persistent_id)
File "/usr/lib64/python3.8/pickle.py", line 560, in save
f(self, obj) # Call unbound method with explicit self
File "/usr/local/lib/python3.8/site-packages/dill/_dill.py", line 1186, in save_module_dict
StockPickler.save_dict(pickler, obj)
File "/usr/lib64/python3.8/pickle.py", line 971, in save_dict
self._batch_setitems(obj.items())
File "/usr/lib64/python3.8/pickle.py", line 997, in _batch_setitems
save(v)
File "/usr/local/lib/python3.8/site-packages/dill/_dill.py", line 388, in save
StockPickler.save(self, obj, save_persistent_id)
File "/usr/lib64/python3.8/pickle.py", line 603, in save
self.save_reduce(obj=obj, *rv)
File "/usr/lib64/python3.8/pickle.py", line 717, in save_reduce
save(state)
File "/usr/local/lib/python3.8/site-packages/dill/_dill.py", line 388, in save
StockPickler.save(self, obj, save_persistent_id)
File "/usr/lib64/python3.8/pickle.py", line 560, in save
f(self, obj) # Call unbound method with explicit self
File "/usr/local/lib/python3.8/site-packages/dill/_dill.py", line 1186, in save_module_dict
StockPickler.save_dict(pickler, obj)
File "/usr/lib64/python3.8/pickle.py", line 971, in save_dict
self._batch_setitems(obj.items())
File "/usr/lib64/python3.8/pickle.py", line 997, in _batch_setitems
save(v)
File "/usr/local/lib/python3.8/site-packages/dill/_dill.py", line 388, in save
StockPickler.save(self, obj, save_persistent_id)
File "/usr/lib64/python3.8/pickle.py", line 603, in save
self.save_reduce(obj=obj, *rv)
File "/usr/lib64/python3.8/pickle.py", line 717, in save_reduce
save(state)
File "/usr/local/lib/python3.8/site-packages/dill/_dill.py", line 388, in save
StockPickler.save(self, obj, save_persistent_id)
File "/usr/lib64/python3.8/pickle.py", line 560, in save
f(self, obj) # Call unbound method with explicit self
File "/usr/local/lib/python3.8/site-packages/dill/_dill.py", line 1186, in save_module_dict
StockPickler.save_dict(pickler, obj)
File "/usr/lib64/python3.8/pickle.py", line 971, in save_dict
self._batch_setitems(obj.items())
File "/usr/lib64/python3.8/pickle.py", line 1002, in _batch_setitems
save(v)
File "/usr/local/lib/python3.8/site-packages/dill/_dill.py", line 388, in save
StockPickler.save(self, obj, save_persistent_id)
File "/usr/lib64/python3.8/pickle.py", line 603, in save
self.save_reduce(obj=obj, *rv)
File "/usr/lib64/python3.8/pickle.py", line 717, in save_reduce
save(state)
File "/usr/local/lib/python3.8/site-packages/dill/_dill.py", line 388, in save
StockPickler.save(self, obj, save_persistent_id)
File "/usr/lib64/python3.8/pickle.py", line 560, in save
f(self, obj) # Call unbound method with explicit self
File "/usr/local/lib/python3.8/site-packages/dill/_dill.py", line 1186, in save_module_dict
StockPickler.save_dict(pickler, obj)
File "/usr/lib64/python3.8/pickle.py", line 971, in save_dict
self._batch_setitems(obj.items())
File "/usr/lib64/python3.8/pickle.py", line 997, in _batch_setitems
save(v)
File "/usr/local/lib/python3.8/site-packages/dill/_dill.py", line 388, in save
StockPickler.save(self, obj, save_persistent_id)
File "/usr/lib64/python3.8/pickle.py", line 578, in save
rv = reduce(self.proto)

TypeError: cannot pickle 'SSLContext' object

我的带有 pandarallel 调用的代码结构如下所示：

import pandas as pd
from pandarallel import pandarallel
from openai import AzureOpenAI

openai_client = AzureOpenAI(
             api_key=config_nlp.openai_api_key,
             api_version=config_nlp.openai_api_version,
            azure_endpoint =config_nlp.openai_api_base,)
            
            
def call_summ_logic_new(self,prompt,max_len):


    api_start_time = datetime.now()
    response = self.openai_client.chat.completions.create(model=summary_model,
    messages = messages,
    temperature=0.8,
    max_tokens=int(max_len)
    )



pandarallel.initialize(progress_bar=True, nb_workers = 7)
recs_pre_summ_tkn = [("data1",23),("data2",24),("data4",123),("data5",243)]
prompt_df_1 = pd.DataFrame(recs_pre_summ_tkn,columns =["to_be_summarized","token_len"])
prompt_df_1["result"] = prompt_df_1.parallel_apply(lambda x: self.summarize(x["to_be_summarized"],x["token_len"]), axis=1)

除了恢复到 openai 0.28.0 之外还有什么建议吗？

谢谢

Answer 1

这就是我要尝试的：

import pandas as pd
from pandarallel import pandarallel
from openai import AzureOpenAI
from httpx import Client

openai_client = AzureOpenAI(
    api_key=config_nlp.openai_api_key,
    api_version=config_nlp.openai_api_version,
    azure_endpoint=config_nlp.openai_api_base,
    http_client=Client(verify=False),
)