1
比赛介绍
本次竞赛聚焦解决大规模生产系统中的硬盘故障预测问题，需解决数据噪声、正负样本不均衡等技术问题，同时也要保障预测算法长期稳定有效。
给定一段连续采集(天粒度)的硬盘状态监控数据（Self-Monitoring, Analysis, and Reporting Technology; often written as SMART)以及故障标签数据，参赛者需要自己提出方案，按天粒度判断每块硬盘是否会在未来30日内发生故障。例如，可以将预测故障问题转化为传统的二分类问题，通过分类模型来判断哪些硬盘会坏；或者可以转化为排序问题，通过Learning to rank的方式判断硬盘的损坏严重程度等。
2
比赛baseline
# 因为题目数据量比较大，所以先对原始数据进行处理，方便以后调试import numpy as npimport pandas as pdimport gcimport seaborn as snsimport xgboost as xgbimport lightgbm as lgbimport catboost as cbtimport mathfrom lightgbm.sklearn import LGBMClassifierfrom collections import Counter import timefrom scipy.stats import kurtosis,iqrfrom scipy import ptpfrom tqdm import tqdmimport datetimefrom sklearn.metrics import accuracy_score, roc_auc_score,log_loss,f1_scorefrom sklearn.model_selection import KFold,StratifiedKFoldfrom sklearn.preprocessing import LabelEncoderimport matplotlib.pyplot as pltfrom datetime import datetime,timedeltaimport warningsimport osimport joblibpd.options.display.max_columns = Nonepd.options.display.max_rows=None
test = pd.read_csv('./disk_sample_smart_log_test_a.csv/disk_sample_smart_log_test_a.csv')tag = pd.read_csv('./disk_sample_fault_tag.csv/tag.csv')test['dt'] = test['dt'].apply(lambda x:''.join(str(x)[0:4] +'-'+ str(x)[4:6] +'-'+ str(x)[6:]))test['dt'] = pd.to_datetime(test['dt'])tag['fault_time'] = pd.to_datetime(tag['fault_time'])###tag表里面有的硬盘同一天发生几种故障tag['tag'] = tag['tag'].astype(str)tag = tag.groupby(['serial_number','fault_time','model'])['tag'].apply(lambda x :'|'.join(x)).reset_index()test = test.sort_values(['serial_number','dt'])test = test.drop_duplicates().reset_index(drop=True)
###去掉全为空值和nunique为1的特征drop_list =[]for i in tqdm([col for col in test.columns if col not in ['manufacturer','model']]):if (test[i].nunique() == 1)&(test[i].isnull().sum() == 0):drop_list.append(i)df= pd.DataFrame()df['fea'] = test.isnull().sum().indexdf['isnull_sum'] = test.isnull().sum().valuesfea_list = list(set(df.loc[df.isnull_sum != test.shape[0]]['fea']) - set(drop_list))test=test[fea_list]###去掉无用特征后给每个样本打上label，下次直接读取##这里只处理了456月份的数据，其他同理，但是7月份的数据因为tag的时间也只到了七月份不好直接打labeldef get_label(df):df = df[fea_list]df['dt'] = df['dt'].apply(lambda x:''.join(str(x)[0:4] +'-'+ str(x)[4:6] +'-'+ str(x)[6:]))df['dt'] = pd.to_datetime(df['dt']) df = df.merge(tag[['serial_number','model','fault_time']],how = 'left',on =['serial_number','model'])df['diff_day'] = (df['fault_time'] - df['dt']).dt.daysdf['label'] = 0df.loc[(df['diff_day']>=0)&(df['diff_day']<=30),'label'] = 1return dftrain__6 = pd.read_csv('./disk_sample_smart_log__Q2/disk_sample_smart_log_06.csv')train__6 = get_label(train__6)joblib.dump(train__6, './disk_sample_smart_log__Q2/train__6.jl.z')train__5 = pd.read_csv('./disk_sample_smart_log__Q2/disk_sample_smart_log_05.csv')train__5 = get_label(train__5)joblib.dump(train__5, './disk_sample_smart_log__Q2/train__5.jl.z')train__4 = pd.read_csv('./disk_sample_smart_log__Q2/disk_sample_smart_log_06.csv')train__4 = get_label(train__6)joblib.dump(train__4, './disk_sample_smart_log__Q2/train__6.jl.z')###这里只搞了三个月的数据其他一样处理
###提取出每个硬盘最早出现的时间日期，内存不够的话，只能一个一个读取train__7 = joblib.load('./disk_sample_smart_log__Q2/train__7.jl.z')train__6 = joblib.load('./disk_sample_smart_log__Q2/train__6.jl.z')train__5 = joblib.load('./disk_sample_smart_log__Q2/train__5.jl.z')train__4 = joblib.load('./disk_sample_smart_log__Q2/train__4.jl.z')train__3 = joblib.load('./disk_sample_smart_log__Q1/train__3.jl.z')train__2 = joblib.load('./disk_sample_smart_log__Q1/train__2.jl.z')train__1 = joblib.load('./disk_sample_smart_log__Q1/train__1.jl.z')train__7 = joblib.load('./disk_sample_smart_log_/train__7.jl.z')train__8 = joblib.load('./disk_sample_smart_log_/train__8.jl.z')train__9 = joblib.load('./disk_sample_smart_log_/train__9.jl.z')train__10 = joblib.load('./disk_sample_smart_log_/train__10.jl.z')train__11 = joblib.load('./disk_sample_smart_log_/train__11.jl.z')train__12 = joblib.load('./disk_sample_smart_log_/train__12.jl.z')serial__7 = train__7[['serial_number','dt']].sort_values('dt').drop_duplicates('serial_number')serial__8 = train__8[['serial_number','dt']].sort_values('dt').drop_duplicates('serial_number')serial__9 = train__9[['serial_number','dt']].sort_values('dt').drop_duplicates('serial_number')serial__10 = train__10[['serial_number','dt']].sort_values('dt').drop_duplicates('serial_number')serial__11 = train__11[['serial_number','dt']].sort_values('dt').drop_duplicates('serial_number')serial__12 = train__12[['serial_number','dt']].sort_values('dt').drop_duplicates('serial_number')serial__1 = train__1[['serial_number','dt']].sort_values('dt').drop_duplicates('serial_number')serial__2 = train__2[['serial_number','dt']].sort_values('dt').drop_duplicates('serial_number')serial__3 = train__3[['serial_number','dt']].sort_values('dt').drop_duplicates('serial_number')serial__4 = train__4[['serial_number','dt']].sort_values('dt').drop_duplicates('serial_number')serial__5 = train__5[['serial_number','dt']].sort_values('dt').drop_duplicates('serial_number')serial__6 = train__6[['serial_number','dt']].sort_values('dt').drop_duplicates('serial_number')serial__7 = train__7[['serial_number','dt']].sort_values('dt').drop_duplicates('serial_number')serial__8 = test[['serial_number','dt']].sort_values('dt').drop_duplicates('serial_number')serial = pd.concat((serial__7,serial__8,serial__9,serial__10,serial__11,serial__12),axis = 0)serial = pd.concat((serial,serial__1,serial__2,serial__3,serial__4,serial__5,serial__6,serial__7,serial__8),axis = 0)serial = serial.sort_values('dt').drop_duplicates('serial_number').reset_index(drop=True)serial.columns = ['serial_number','dt_first']serial.dt_first = pd.to_datetime(serial.dt_first)serial.to_csv("serial.csv",index=False
本题是给定一段连续采集(天粒度)的硬盘状态监控数据（Self-Monitoring, Analysis, and Reporting Technology; often written as SMART)以及故障标签数据，参赛者需要自己提出方案，按天粒度判断每块硬盘是否会在未来30日内发生故障。例如，可以将预测故障问题转化为传统的二分类问题，通过分类模型来判断哪些硬盘会坏；因为七月份的数据不知道label，所以线下验证的时候也必须模拟线上隔一个月用四月份训练，六月份验证。
import numpy as npimport pandas as pdimport gcimport seaborn as snsimport xgboost as xgbimport lightgbm as lgbimport catboost as cbtimport mathfrom lightgbm.sklearn import LGBMClassifierfrom collections import Counter import timefrom scipy.stats import kurtosis,iqrfrom scipy import ptpfrom tqdm import tqdmimport datetimefrom sklearn.metrics import accuracy_score, roc_auc_score,log_loss,f1_scorefrom sklearn.model_selection import KFold,StratifiedKFoldfrom sklearn.preprocessing import LabelEncoderimport matplotlib.pyplot as pltfrom datetime import datetime,timedeltaimport warningsimport osimport joblibpd.options.display.max_columns = Nonepd.options.display.max_rows = None
test = pd.read_csv('./disk_sample_smart_log_test_a.csv/disk_sample_smart_log_test_a.csv')tag = pd.read_csv('./disk_sample_fault_tag.csv/tag.csv')test['dt'] = test['dt'].apply(lambda x:''.join(str(x)[0:4] +'-'+ str(x)[4:6] +'-'+ str(x)[6:]))test['dt'] = pd.to_datetime(test['dt'])tag['fault_time'] = pd.to_datetime(tag['fault_time'])test = test.sort_values(['serial_number','dt'])test = test.drop_duplicates().reset_index(drop=True)sub = test[['manufacturer','model','serial_number','dt']]serial = pd.read_csv('serial.csv')serial.columns = ['serial_number','dt_first']serial.dt_first = pd.to_datetime(serial.dt_first)tag['tag'] = tag['tag'].astype(str)tag = tag.groupby(['serial_number','fault_time','model'])['tag'].apply(lambda x :'|'.join(x)).reset_index()tag.columns = ['serial_number','fault_time_1','model','tag']tag.model = tag.model.map({1:2,2:1})map_dict = dict(zip(tag['tag'].unique(), range(tag['tag'].nunique())))tag['tag'] = tag['tag'].map(map_dict).fillna(-1).astype('int32')###用到的特征feature_name = [i for i in test.columns if i not in ['dt','manufacturer']] + ['days','days_1','days_2','tag']
###验证train_x = train__4del train__4train_x = train_x.append(train__3).reset_index(drop=True)del train__3train_x = train_x.merge(serial,how = 'left',on = 'serial_number')###硬盘的使用时常train_x['days'] = (train_x['dt'] - train_x['dt_first']).dt.daystrain_x = train_x.merge(tag,how = 'left',on = ['serial_number','model'])###当前时间与另一个model故障的时间差，train_x['days_1'] = (train_x['dt'] - train_x['fault_time_1']).dt.daystrain_x.loc[train_x.days_1 <= 0,'tag'] = Nonetrain_x.loc[train_x.days_1 <= 0,'days_1'] = None###同一硬盘第一次使用到开始故障的时间train_x['days_2'] = (train_x['fault_time_1'] - train_x['dt_first']).dt.daystrain_x.loc[train_x.fault_time_1 >= train_x.dt,'days_2'] = Nonetrain_x['serial_number'] = train_x['serial_number'].apply(lambda x:int(x.split('_')[1]))train_y = train_x.label.valuestrain_x = train_x[feature_name]gc.collect()val_x = train__6del train__6val_x = val_x.merge(serial,how = 'left',on = 'serial_number')val_x['days'] = (val_x['dt'] - val_x['dt_first']).dt.daysval_x = val_x.merge(tag,how = 'left',on = ['serial_number','model'])val_x['days_1'] = (val_x['dt'] - val_x['fault_time_1']).dt.daysval_x.loc[val_x.days_1 <= 0,'tag'] = Noneval_x.loc[val_x.days_1 <= 0,'days_1'] = Noneval_x['days_2'] = (val_x['fault_time_1'] - val_x['dt_first']).dt.daysval_x.loc[val_x.fault_time_1 >= val_x.dt,'days_2'] = Noneval_x['serial_number'] = val_x['serial_number'].apply(lambda x:int(x.split('_')[1]))val_y = val_x.label.valuesval_x = val_x[feature_name]gc.collect()gc.collect()clf = LGBMClassifier( learning_rate=0.001,n_estimators=10000,num_leaves=127,subsample=0.8,colsample_bytree=0.8,random_state=,is_unbalenced = 'True',metric=None)print('************** training **************')print(train_x.shape,val_x.shape)clf.fit(train_x, train_y,eval_set=[(val_x, val_y)],eval_metric='auc',early_stopping_rounds=50,verbose=100)
##根据验证的轮数训练train_df = train__6.append(train__5).reset_index(drop=True)del train__6del train__5train_df = train_df.append(train__4).reset_index(drop=True)del train__4train_df = train_df.merge(serial,how = 'left',on = 'serial_number')train_df['days'] = (train_df['dt'] - train_df['dt_first']).dt.daystrain_df = train_df.merge(tag,how = 'left',on = ['serial_number','model'])train_df['days_1'] = (train_df['dt'] - train_df['fault_time_1']).dt.daystrain_df.loc[train_df.days_1 <= 0,'tag'] = Nonetrain_df.loc[train_df.days_1 <= 0,'days_1'] = Nonetrain_df['days_2'] = (train_df['fault_time_1'] - train_df['dt_first']).dt.daystrain_df.loc[train_df.fault_time_1 >= train_df.dt,'days_2'] = Nonetrain_df['serial_number'] = train_df['serial_number'].apply(lambda x:int(x.split('_')[1]))labels = train_df.label.valuestrain_df = train_df[feature_name]gc.collect()test = test.merge(serial,how = 'left',on = 'serial_number')test['days'] = (test['dt'] - test['dt_first']).dt.daystest = test.merge(tag,how = 'left',on = ['serial_number','model'])test['days_1'] = (test['dt'] - test['fault_time_1']).dt.daystest.loc[test.days_1 <= 0,'tag'] = Nonetest.loc[test.days_1 <= 0,'days_1'] = Nonetest['days_2'] = (test['fault_time_1'] - test['dt_first']).dt.daystest.loc[test.fault_time_1 >= test.dt,'days_2'] = Nonetest['serial_number'] = test['serial_number'].apply(lambda x:int(x.split('_')[1]))test_x = test[feature_name]del testgc.collect()clf = LGBMClassifier(learning_rate=0.001,n_estimators=100,num_leaves=127,subsample=0.8,colsample_bytree=0.8,random_state=,#is_unbalenced = 'True',metric=None)print('************** training **************')print(train_df.shape,test_x.shape)clf.fit(train_df, labels,eval_set=[(train_df, labels)],eval_metric='auc',early_stopping_rounds=10,verbose=10)sub['p'] = clf.predict_proba(test_x)[:,1]sub['label'] = sub['p'].rank()sub['label']= (sub['label']>=sub.shape[0] * 0.996).astype(int)submit = sub.loc[sub.label == 1]###这里我取故障率最大的一天进行提交，线上测了几个阈值，100个左右好像比较好。。。。submit = submit.sort_values('p',ascending=False)submit = submit.drop_duplicates(['serial_number','model'])submit[['manufacturer','model','serial_number','dt']].to_csv("sub.csv",index=False,header = None)submit.shape
一些其他思路 可以试试多分类，label分为1，2....30天，超过30，也可也试试回归做。
点击阅读原文，报名链接。
