加速加速和csv导出

Question

需要调优以下代码的建议。

import sys
import pandas as pd
import spacy

#Spliting tokens using the Spacy
def parsetext(df):
    nlp = spacy.load("en_core_web_sm")
    parsed_tokens = []
    for index, row in df.iterrows():
        filtered_tokens=[]
        doc = nlp(str(row['Column1Text']))
        for word in doc:
            if word.is_stop==False:
                filtered_tokens.append(word)
        parsed_tokens.append(filtered_tokens)

    df['Tokens'] = parsed_tokens
    df['Processed'] = 0

# Main method point of entry | Read the excel and generate tokens
def main():
        df = pd.read_excel (sys.argv[1], sheet_name='Sheet1', header=None)
        df.columns = ["Column1Text","Column2Text","C3","C4","C5","C6","C7"]
        #Replace the Nan with empty values
        df["Column1Text"].fillna('',inplace=True)
        df["Column2Text"].fillna('',inplace=True)
        #Parse for tokens
        parsetext(df)
        export_csv = df[['Column1Text', 'Column2Text','Tokens','Processed']].to_csv(sys.path[0]+ r'\parsed_file.csv', index = None, header=True)

parsetext（df）方法需要2到3秒才能完成

任何提速建议？

Answer 1

从数据框中提取文本，使用nlp.pipe()应该更快，以便可以更快速地批量处理文本，然后将结果保存回数据框中。