需要调优以下代码的建议。
import sys
import pandas as pd
import spacy
#Spliting tokens using the Spacy
def parsetext(df):
nlp = spacy.load("en_core_web_sm")
parsed_tokens = []
for index, row in df.iterrows():
filtered_tokens=[]
doc = nlp(str(row['Column1Text']))
for word in doc:
if word.is_stop==False:
filtered_tokens.append(word)
parsed_tokens.append(filtered_tokens)
df['Tokens'] = parsed_tokens
df['Processed'] = 0
# Main method point of entry | Read the excel and generate tokens
def main():
df = pd.read_excel (sys.argv[1], sheet_name='Sheet1', header=None)
df.columns = ["Column1Text","Column2Text","C3","C4","C5","C6","C7"]
#Replace the Nan with empty values
df["Column1Text"].fillna('',inplace=True)
df["Column2Text"].fillna('',inplace=True)
#Parse for tokens
parsetext(df)
export_csv = df[['Column1Text', 'Column2Text','Tokens','Processed']].to_csv(sys.path[0]+ r'\parsed_file.csv', index = None, header=True)
parsetext(df)方法需要2到3秒才能完成
任何提速建议?
从数据框中提取文本,使用nlp.pipe()
应该更快,以便可以更快速地批量处理文本,然后将结果保存回数据框中。