def missing_delete_var(df, threshold=None):"""df:数据集threshold:缺失率删除的阈值return :删除缺失后的数据集"""df2 = df.copy()missing_df = missing_cal(df)missing_col_num = missing_df[missing_df.missing_pct >= threshold].shape[0]missing_col = list(missing_df[missing_df.missing_pct >= threshold].col)df2 = df2.drop(missing_col, axis=1)return df2def missing_delete_user(df, threshold=None):"""df:数据集threshold:缺失个数删除的阈值return :删除缺失后的数据集"""df2 = df.copy()missing_series = df.isnull().sum(axis=1)missing_list = list(missing_series)missing_index_list = []for i, j in enumerate(missing_list):if j >= threshold:missing_index_list.append(i)df2 = df2[~(df2.index.isin(missing_index_list))]return df2def const_delete(df, col_list, threshold=None):"""df:数据集col_list:变量list集合threshold:同值化处理的阈值return :处理后的数据集"""df2 = df.copy()const_col = []for col in col_list:const_pct = df2[col].value_counts().iloc[0] / df2[df2[col].notnull()].shape[0]if const_pct >= threshold:const_col.append(col)df2 = df2.drop(const_col, axis=1)return df2def missing_cal(df):"""df :数据集return：每个变量的缺失率"""missing_series = df.isnull().sum() / df.shape[0]missing_df = pd.DataFrame(missing_series).reset_index()missing_df = missing_df.rename(columns={'index': 'col',0: 'missing_pct'})missing_df = missing_df.sort_values('missing_pct', ascending=False).reset_index(drop=True)return missing_dfdef data_processing(df, target):"""df:包含了label和特征的宽表return:df :清洗后的数据集"""# 特征缺失处理df = missing_delete_var(df, threshold=0.8)# 样本缺失处理df = missing_delete_user(df, threshold=int(df.shape[1] * 0.8))col_list = [x for x in df.columns if x != target]# 常变量处理df = const_delete(df, col_list, threshold=0.9)desc = df.describe().T# 剔除方差为0的特征std_0_col = list(desc[desc['std'] == 0].index)if len(std_0_col) > 0:df = df.drop(std_0_col, axis=1)df.reset_index(drop=True, inplace=True)# 缺失值计算和填充miss_df = missing_cal(df)cate_col = list(df.select_dtypes(include=['O']).columns)num_col = [x for x in list(df.select_dtypes(include=['int64', 'float64']).columns) if x != 'label']# 分类型特征填充cate_miss_col1 = [x for x in list(miss_df[miss_df.missing_pct > 0.05]['col']) if x in cate_col]cate_miss_col2 = [x for x in list(miss_df[miss_df.missing_pct <= 0.05]['col']) if x in cate_col]num_miss_col1 = [x for x in list(miss_df[miss_df.missing_pct > 0.05]['col']) if x in num_col]num_miss_col2 = [x for x in list(miss_df[miss_df.missing_pct <= 0.05]['col']) if x in num_col]for col in cate_miss_col1:df[col] = df[col].fillna('未知')for col in cate_miss_col2:df[col] = df[col].fillna(df[col].mode()[0])for col in num_miss_col1:df[col] = df[col].fillna(-999)for col in num_miss_col2:df[col] = df[col].fillna(df[col].median())return df, miss_df
对数据分析、机器学习、数据科学、金融风控等感兴趣的小伙伴，需要数据集、代码、行业报告等各类学习资料，可添加微信：wu805686220（记得要备注喔！），也可关注微信公众号：风控圏子（别打错字，是圏子，不是圈子，算了直接复制吧！）
关注公众号后，可联系圈子助手加入如下社群：
机器学习风控讨论群（微信群）反欺诈讨论群（微信群）python学习交流群（微信群）研习社资料（qq群：102755159）（干货、资料、项目、代码、报告、课件）
相互学习，共同成长。
