Logo
Section 1 / 12

数据偏见简介

了解数据偏差的含义及其来源。

导入

数据偏见简介

了解数据偏差的含义及其来源。

import pandas as pd import numpy as np from sklearn.ensemble import RandomForestClassifier from sklearn.metrics import accuracy_score

训练数据

从随机分布生成变量

train_target = np.random.choice([0, 1], size=10000, replace=True, p=[0.90, 0.10]) train_var1 = np.random.normal(loc=0, scale=1, size=10000) train_var2 = np.random.normal(loc=0, scale=1, size=10000) train = pd.DataFrame(data={'var1': train_var1, 'var2': train_var2})

测试数据

从随机分布生成变量

test_target = np.random.choice([0, 1], size=10000, replace=True, p=[0.90, 0.10]) test_var1 = np.random.normal(loc=0, scale=1, size=10000) test_var2 = np.random.normal(loc=0, scale=1, size=10000) test = pd.DataFrame(data={'var1': test_var1, 'var2': test_var2})

构建随机森林模型

rfc = RandomForestClassifier(random_state=101) rfc.fit(train, train_target)

在测试数据上进行预测

predictions = rfc.predict(test) print(pd.crosstab(test_target, predictions, rownames=['实际值'], colnames=['预测值'])) print('准确率: {}'.format(accuracy_score(test_target, predictions)))

import pandas as pd import numpy as np from sklearn.ensemble import RandomForestClassifier from sklearn.metrics import accuracy_score

训练数据

从正态分布生成数据

train_target = np.random.choice([0, 1], size=10000, replace=True, p=[0.90, 0.10]) train_var1 = np.random.normal(loc=0, scale=1, size=10000) train_var2 = np.random.normal(loc=0, scale=1, size=10000) train = pd.DataFrame(data={'var1': train_var1, 'var2': train_var2})

测试数据

从正态分布生成数据

test_target = np.random.choice([0, 1], size=10000, replace=True, p=[0.90, 0.10]) test_var1 = np.random.normal(loc=0, scale=1, size=10000) test_var2 = np.random.normal(loc=0, scale=1, size=10000) test = pd.DataFrame(data={'var1': test_var1, 'var2': test_var2})

训练随机森林模型

rfc = RandomForestClassifier(random_state=101) rfc.fit(train, train_target)

在测试数据上进行预测

predictions = rfc.predict(test)

真实世界数据

从正态分布生成数据

real_target = np.random.choice([0, 1], size=10000, replace=True, p=[0.50, 0.50]) real_var1 = np.random.normal(loc=0, scale=1, size=10000) real_var2 = np.random.normal(loc=0, scale=1, size=10000) real = pd.DataFrame(data={'var1': real_var1, 'var2': real_var2})

在真实世界数据上进行预测

real_predictions = rfc.predict(real) print(pd.crosstab(real_target, real_predictions, rownames=['实际值'], colnames=['预测值'])) print('准确率: {}'.format(accuracy_score(real_target, real_predictions)))

0: \frac{n_{零类正确}}{n_{零类总数}}=\frac{4945}{4945+51}=98.98%

1: \frac{n_{一类正确}}{n_{一类总数}}=\frac{61}{61+4943}=1.22%

import pandas as pd import numpy as np from sklearn.ensemble import RandomForestClassifier from sklearn.model_selection import train_test_split from sklearn.metrics import accuracy_score

定义随机森林模型

rfc = RandomForestClassifier(random_state=101)

读取数据

titanic = pd.read_csv('titanic.csv')[['Sex', 'Pclass', 'Age', 'SibSp', 'Parch', 'Fare', 'Survived']].dropna(axis=0) y = np.where(titanic['Sex'] == 'male', 1, 0) # 男性标记为1,女性为0 X = titanic.drop(columns=['Sex'])

划分训练集和测试集

x_train, x_test, y_train, y_test = train_test_split(X, y, random_state=101)

训练模型

rfc.fit(x_train, y_train) predictions = rfc.predict(x_test) print(accuracy_score(predictions, y_test)) print(pd.crosstab(predictions, y_test, rownames=['预测值'], colnames=['实际值']))

import pandas as pd import numpy as np from sklearn.ensemble import RandomForestClassifier from sklearn.model_selection import train_test_split from sklearn.metrics import accuracy_score

定义随机森林模型

rfc = RandomForestClassifier(random_state=101)

读取数据

titanic = pd.read_csv('titanic.csv')[['Sex', 'Pclass', 'Age', 'SibSp', 'Parch', 'Fare']].dropna(axis=0) y = np.where(titanic['Sex'] == 'male', 1, 0) # 男性标记为1,女性为0 X = titanic.drop(columns=['Sex'])

划分训练集和测试集

x_train, x_test, y_train, y_test = train_test_split(X, y, random_state=101)

训练模型

rfc.fit(x_train, y_train) predictions = rfc.predict(x_test) print(accuracy_score(predictions, y_test)) print(pd.crosstab(predictions, y_test, rownames=['预测值'], colnames=['实际值']))

简介

诊断数据偏差