Python：有什么方法可以改进或自动化下面类中的任何方法吗？

Question

以下是我希望改进的预处理脚本。任何建议都会有所帮助 - 我是否应该考虑添加更多方法来拆分任何功能并自动化任何东西？我该如何改进这段代码？提前致谢。

class PreprocessData(Base Estimator, TransformerMixin): def __init__(self, df, content_type="Post"):        
    self.X = df
    self.content_type = content_type
# select features = visual labels, categories, description
# lowercase, remove alphanumeric characters, remove stopwords
# flatten visual labels, categories
# name each label group and change dtype to category
# concatenate dataframes (visual labels, categories)
        
# --Helper Functions Below--  
        
def lowercase(self, text):
    return str(text).lower()

def clean_alphanumeric_stopwords(self, text, category=False):
    stop = set(stopwords.words('english'))
    text = self.lowercase(text)
    tok_text = word_tokenize(text)
    clean_text = [word for word in tok_text if word.isalpha()]
    clean_text = [word for word in clean_text if not word in stop]
    if category:
        return ','.join(clean_text)
    else:
        return ' '.join(clean_text)
   
#def load_data(self):
    #return pd.read_csv('/Users/melodyzap/Downloads/ds_train_jk (public).csv')

def set_index(self):
    return self.X.set_index('Id')

def remove_empty_char(self, text):
    if type(text)==list: 
        while '' in text:
            text.remove('')
    else:
        print(text)
    return text

def fit(self, X):
    return self

def transform(self):
    
    # self.X = self.set_index()
    
    self.X['Categories'] = self.X['Categories'].apply(self.clean_alphanumeric_stopwords, category=True)
    self.X['Visual Labels'] = self.X['Visual Labels'].apply(self.clean_alphanumeric_stopwords, category=True)
    
    self.X['all_category_data'] = self.X['Visual Labels'] + str(',') + self.X['Categories']
    self.X['all_category_data'] = self.X['all_category_data'].str.split(r',',expand=False)
    self.X['all_category_data']  = self.X['all_category_data'].apply(lambda x: self.remove_empty_char(x))
    self.X.drop(labels=['Categories','Visual Labels'],axis=1, inplace=True)

    if self.content_type != "Image":
        self.X['Description'] = self.X['Description'].apply(self.clean_alphanumeric_stopwords)    
        filter_lang = FilterLanguages()
        self.X = filter_lang.transform(self.X)

    return self.X

Answer 1

您可以使用正则表达式模块以给定模式清理一些不需要的数据。：

import re

word="hello_world$$$!'^!'99"
cleaned_word=re.sub('[^A-Za-z0-9]+', ' ', word).strip()
print([cleaned_word])
    >>> ["hello world 99"]