数据偏见简介
了解数据偏差的含义及其来源。
import pandas as pd import numpy as np from sklearn.ensemble import RandomForestClassifier from sklearn.metrics import accuracy_score
训练数据
从随机分布生成变量
train_target = np.random.choice([0, 1], size=10000, replace=True, p=[0.90, 0.10]) train_var1 = np.random.normal(loc=0, scale=1, size=10000) train_var2 = np.random.normal(loc=0, scale=1, size=10000) train = pd.DataFrame(data={'var1': train_var1, 'var2': train_var2})
测试数据
从随机分布生成变量
test_target = np.random.choice([0, 1], size=10000, replace=True, p=[0.90, 0.10]) test_var1 = np.random.normal(loc=0, scale=1, size=10000) test_var2 = np.random.normal(loc=0, scale=1, size=10000) test = pd.DataFrame(data={'var1': test_var1, 'var2': test_var2})
构建随机森林模型
rfc = RandomForestClassifier(random_state=101) rfc.fit(train, train_target)
在测试数据上进行预测
predictions = rfc.predict(test) print(pd.crosstab(test_target, predictions, rownames=['实际值'], colnames=['预测值'])) print('准确率: {}'.format(accuracy_score(test_target, predictions)))
import pandas as pd import numpy as np from sklearn.ensemble import RandomForestClassifier from sklearn.metrics import accuracy_score
训练数据
从正态分布生成数据
train_target = np.random.choice([0, 1], size=10000, replace=True, p=[0.90, 0.10]) train_var1 = np.random.normal(loc=0, scale=1, size=10000) train_var2 = np.random.normal(loc=0, scale=1, size=10000) train = pd.DataFrame(data={'var1': train_var1, 'var2': train_var2})
测试数据
从正态分布生成数据
test_target = np.random.choice([0, 1], size=10000, replace=True, p=[0.90, 0.10]) test_var1 = np.random.normal(loc=0, scale=1, size=10000) test_var2 = np.random.normal(loc=0, scale=1, size=10000) test = pd.DataFrame(data={'var1': test_var1, 'var2': test_var2})
训练随机森林模型
rfc = RandomForestClassifier(random_state=101) rfc.fit(train, train_target)
在测试数据上进行预测
predictions = rfc.predict(test)
真实世界数据
从正态分布生成数据
real_target = np.random.choice([0, 1], size=10000, replace=True, p=[0.50, 0.50]) real_var1 = np.random.normal(loc=0, scale=1, size=10000) real_var2 = np.random.normal(loc=0, scale=1, size=10000) real = pd.DataFrame(data={'var1': real_var1, 'var2': real_var2})
在真实世界数据上进行预测
real_predictions = rfc.predict(real) print(pd.crosstab(real_target, real_predictions, rownames=['实际值'], colnames=['预测值'])) print('准确率: {}'.format(accuracy_score(real_target, real_predictions)))
0: \frac{n_{零类正确}}{n_{零类总数}}=\frac{4945}{4945+51}=98.98%
1: \frac{n_{一类正确}}{n_{一类总数}}=\frac{61}{61+4943}=1.22%
import pandas as pd import numpy as np from sklearn.ensemble import RandomForestClassifier from sklearn.model_selection import train_test_split from sklearn.metrics import accuracy_score
定义随机森林模型
rfc = RandomForestClassifier(random_state=101)
读取数据
titanic = pd.read_csv('titanic.csv')[['Sex', 'Pclass', 'Age', 'SibSp', 'Parch', 'Fare', 'Survived']].dropna(axis=0) y = np.where(titanic['Sex'] == 'male', 1, 0) # 男性标记为1,女性为0 X = titanic.drop(columns=['Sex'])
划分训练集和测试集
x_train, x_test, y_train, y_test = train_test_split(X, y, random_state=101)
训练模型
rfc.fit(x_train, y_train) predictions = rfc.predict(x_test) print(accuracy_score(predictions, y_test)) print(pd.crosstab(predictions, y_test, rownames=['预测值'], colnames=['实际值']))
import pandas as pd import numpy as np from sklearn.ensemble import RandomForestClassifier from sklearn.model_selection import train_test_split from sklearn.metrics import accuracy_score
定义随机森林模型
rfc = RandomForestClassifier(random_state=101)
读取数据
titanic = pd.read_csv('titanic.csv')[['Sex', 'Pclass', 'Age', 'SibSp', 'Parch', 'Fare']].dropna(axis=0) y = np.where(titanic['Sex'] == 'male', 1, 0) # 男性标记为1,女性为0 X = titanic.drop(columns=['Sex'])
划分训练集和测试集
x_train, x_test, y_train, y_test = train_test_split(X, y, random_state=101)
训练模型
rfc.fit(x_train, y_train) predictions = rfc.predict(x_test) print(accuracy_score(predictions, y_test)) print(pd.crosstab(predictions, y_test, rownames=['预测值'], colnames=['实际值']))