⼆⼿车交易价格预测：建模调参建模与调参
内容介绍
1. 线性回归模型：
线性回归对于特征的要求；
处理长尾分布；
理解线性回归模型；弹力布
2. 模型性能验证：
评价函数与⽬标函数；
交叉验证⽅法；
留⼀验证⽅法；
针对时间序列问题的验证；
绘制学习率曲线；
绘制验证曲线；
3. 嵌⼊式特征选择：
Lasso回归；
Ridge回归；
决策树；
4. 模型对⽐：
常⽤线性模型；
常⽤⾮线性模型；
5. 模型调参：
贪⼼调参⽅法；
⽹格调参⽅法；
贝叶斯调参⽅法；
代码⽰例
import pandas as pd
import numpy as np
import warnings
warnings.filterwarnings('ignore')
reduce_mem_usage 函数通过调整数据类型，帮助我们减少数据在内存中占⽤的空间
def reduce_mem_usage(df):
""" iterate through all the columns of a dataframe and modify the data type
to reduce memory usage.
"""
start_mem = df.memory_usage().sum()
print('Memory usage of dataframe is {:.2f} MB'.format(start_mem))
for col lumns:
col_type = df[col].dtype
if col_type != object:
c_min = df[col].min()
c_max = df[col].max()
if str(col_type)[:3] == 'int':
if c_min > np.iinfo(np.int8).min and c_max < np.iinfo(np.int8).max:
df[col] = df[col].astype(np.int8)
elif c_min > np.iinfo(np.int16).min and c_max < np.iinfo(np.int16).max:
df[col] = df[col].astype(np.int16)
folkselif c_min > np.iinfo(np.int32).min and c_max < np.iinfo(np.int32).max:
df[col] = df[col].astype(np.int32)
elif c_min > np.iinfo(np.int64).min and c_max < np.iinfo(np.int64).max:
df[col] = df[col].astype(np.int64)
el:
if c_min > np.finfo(np.float16).min and c_max < np.finfo(np.float16).max:
df[col] = df[col].astype(np.float16)
elif c_min > np.finfo(np.float32).min and c_max < np.finfo(np.float32).max:
df[col] = df[col].astype(np.float32)
omg视频
el:
df[col] = df[col].astype(np.float64)
el:
df[col] = df[col].astype('category')
end_mem = df.memory_usage().sum()
print('Memory usage after optimization is: {:.2f} MB'.format(end_mem))
print('Decread by {:.1f}%'.format(100 * (start_mem - end_mem) / start_mem))
return df
sample_feature = reduce_mem_ad_csv('data_for_tree.csv'))
Memory usage of dataframe is 60507328.00 MB
Memory usage after optimization is: 15724107.00 MB
Decread by 74.0%
continuous_feature_names = [x for x in lumns if x not in ['price','brand','model','brand']]线性回归 & 五折交叉验证 & 模拟真实业务情况
sample_feature = sample_feature.dropna().replace('-', 0).ret_index(drop=True)2020年9月四级成绩查询时间
sample_feature['notRepairedDamage'] = sample_feature['notRepairedDamage'].astype(np.float32) train = sample_feature[continuous_feature_names + ['price']]
train_X = train[continuous_feature_names]
train_y = train['price']
简单建模英语书信
#导⼊线性回归模块
from sklearn.linear_model import LinearRegression
model = LinearRegression(normalize=True)
model = model.fit(train_X, train_y)
'intercept:'+ str(model.intercept_)
sorted(dict(zip(continuous_feature_names, f_)).items(), key=lambda x:x[1], rever=True)
[('v_6', 3509133.3556335964),
('v_8', 739866.7711616001),
('v_9', 173781.00967028155),
('v_7', 41044.805433941016),
('v_12', 30888.95721072973),
('v_5', 27453.39738680756),
('v_3', 23664.188371126955),
('v_11', 15953.34701993885),
('v_13', 13071.48191385408),
('v_10', 7815.353298260309),
('gearbox', 900.8564809596271),
('fuelType', 426.18396873493714),
('bodyType', 190.321639675651),
('city', 44.487589413971016),
('power', 27.430553534775616),
('brand_price_median', 0.5498384842815008),
('brand_price_std', 0.48508518798148187),
('brand_amount', 0.14940527531407094),
('ud_time', 0.02179830302065399),
('brand_price_max', 0.003136932045778858),
('SaleID', 2.091664025644962e-05),
('offerType', 3.907829523086548e-06),
('train', -1.862645149230957e-09),
('ller', -1.0849907994270325e-06),
('brand_price_sum', -2.165152734579202e-05),
('name', -0.0003995079681757267),
('brand_price_average', -0.4597090013419026),
('brand_price_min', -2.2063783078698163),
('power_bin', -15.98197690477394),
('v_14', -363.8704254957781),
老友记中英文字幕版
('kilometer', -388.47363328983346),
('notRepairedDamage', -429.0950578648204),
('v_0', -2092.782811714247),
('v_4', -16184.453743213142),
('v_2', -36879.522402007824),
('v_1', -43460.2165225294)]
from matplotlib import pyplot as plt
绘制特征v_9的值与标签的散点图，图⽚发现模型的预测结果（蓝⾊点）与真实标签（⿊⾊点）的分布差异较⼤，且部分预测值出现了⼩于0的情况，说明我们的模型存在⼀些问题
plt.scatter(train_X['v_9'][subsample_index], train_y[subsample_index], color='black')
plt.scatter(train_X['v_9'][subsample_index], model.predict(train_X.loc[subsample_index]), color='blue') plt.xlabel('v_9')
plt.ylabel('price')
plt.legend(['True Price','Predicted Price'],loc='upper right')
print('The predicted price is obvious different from true price')
plt.show()
import aborn as sns
print('It is clear to e the price shows a typical exponential distribution')
dsteplt.figure(figsize=(15,5))
plt.subplot(1,2,1)
sns.distplot(train_y)
plt.subplot(1,2,2)
sns.distplot(train_y[train_y < np.quantile(train_y, 0.9)])
train_y_ln = np.log(train_y + 1)
import aborn as sns
print('The transformed price ems like normal distribution')
plt.figure(figsize=(15,5))
plt.subplot(1,2,1)
sns.distplot(train_y_ln)
plt.subplot(1,2,2)
sns.distplot(train_y_ln[train_y_ln < np.quantile(train_y_ln, 0.9)])
model = model.fit(train_X, train_y_ln)
print('intercept:'+ str(model.intercept_))
sorted(dict(zip(continuous_feature_names, f_)).items(), key=lambda x:x[1], rever=True)
('ller', 9.308109838457312e-12),
('brand_price_sum', -1.3473184925468486e-10),
('name', -7.11403461065247e-08),
('brand_price_median', -1.7608143661053008e-06),
('brand_price_std', -2.7899058266986454e-06),
('ud_time', -5.6142735899344175e-06),
('city', -0.0024992974087053223),
('v_14', -0.012754139659375262),
('kilometer', -0.013999175312751872),
('v_0', -0.04553774829634237),
('notRepairedDamage', -0.273686961116076),
('v_7', -0.7455902679730504),
('v_4', -0.9281349233755761),
('v_2', -1.2781892166433606),
('v_5', -1.5458846136756323),
('v_10', -1.8059217242413748),
('v_8', -42.611729973490604),
('v_6', -241.30992120503035)]
aerodrome再次进⾏可视化，发现预测结果与真实值较为接近，且未出现异常状况
plt.scatter(train_X['v_9'][subsample_index], train_y[subsample_index], color='black')
plt.scatter(train_X['v_9'][subsample_index], np.exp(model.predict(train_X.loc[subsample_index])), color='blue')
plt.xlabel('v_9')
plt.ylabel('price')
plt.legend(['True Price','Predicted Price'],loc='upper right')
print('The predicted price ems normal after np.log transforming')
plt.show()
五折交叉验证
美就在身边在使⽤训练集对参数进⾏训练的时候，经常会发现⼈们通常会将⼀整个训练集分为三个部分（⽐如mnist⼿写训练集）。⼀般分为：训练集（train_t），评估集（valid_t），测试集（test_t）这三个部分。这其实是为了保证训练效果⽽特意设置的。其中测试集很好理解，其实就是完全不参与训练的数据，仅仅⽤来观测测试效果的数据。⽽训练集和评估集则牵涉到下⾯的知识了。
因为在实际的训练中，训练的结果对于训练集的拟合程度通常还是挺好的（初始条件敏感），但是对于训练集之外的数据的拟合程度通常就不那么令⼈满意了。因此我们通常并不会把所有的数据集都拿来训练，⽽是分出⼀部分来（这⼀部分不参加训练）对训练集⽣成的参数进⾏测试，相对客观的判断这些参数对训练集之外的数据的符合程度。这种思想就称为交叉验证（Cross Validation）
眼线笔颜色
del_lection import cross_val_score
ics import mean_absolute_error, make_scorer
