Rev. | 372bf58aa1eba13e378314c8b9d343fd7fd4210b |
---|---|
サイズ | 5,530 バイト |
日時 | 2013-06-25 02:04:38 |
作者 | Lorenzo Isella |
ログメッセージ | A code to carry out logistic regression in Python.
|
__author__ = 'Miroslaw Horbal'
__email__ = 'miroslaw@gmail.com'
__date__ = '14-06-2013'
from numpy import array, hstack
from sklearn import metrics, cross_validation, linear_model
from scipy import sparse
from itertools import combinations
import numpy as np
import pandas as pd
SEED = 25
def group_data(data, degree=3, hash=hash):
"""
numpy.array -> numpy.array
Groups all columns of data into all combinations of triples
"""
new_data = []
m,n = data.shape
for indicies in combinations(range(n), degree):
new_data.append([hash(tuple(v)) for v in data[:,indicies]])
return array(new_data).T
def OneHotEncoder(data, keymap=None):
"""
OneHotEncoder takes data matrix with categorical columns and
converts it to a sparse binary matrix.
Returns sparse binary matrix and keymap mapping categories to indicies.
If a keymap is supplied on input it will be used instead of creating one
and any categories appearing in the data that are not in the keymap are
ignored
"""
if keymap is None:
keymap = []
for col in data.T:
uniques = set(list(col))
keymap.append(dict((key, i) for i, key in enumerate(uniques)))
total_pts = data.shape[0]
outdat = []
for i, col in enumerate(data.T):
km = keymap[i]
num_labels = len(km)
spmat = sparse.lil_matrix((total_pts, num_labels))
for j, val in enumerate(col):
if val in km:
spmat[j, km[val]] = 1
outdat.append(spmat)
outdat = sparse.hstack(outdat).tocsr()
return outdat, keymap
def create_test_submission(filename, prediction):
content = ['id,ACTION']
for i, p in enumerate(prediction):
content.append('%i,%f' %(i+1,p))
f = open(filename, 'w')
f.write('\n'.join(content))
f.close()
print 'Saved'
# This loop essentially from Paul's starter code
def cv_loop(X, y, model, N):
mean_auc = 0.
for i in range(N):
X_train, X_cv, y_train, y_cv = cross_validation.train_test_split(
X, y, test_size=.20,
random_state = i*SEED)
model.fit(X_train, y_train)
preds = model.predict_proba(X_cv)[:,1]
auc = metrics.auc_score(y_cv, preds)
print "AUC (fold %d/%d): %f" % (i + 1, N, auc)
mean_auc += auc
return mean_auc/N
def main(train='train.csv', test='test.csv', submit='logistic_pred.csv'):
print "Reading dataset..."
train_data = pd.read_csv(train)
test_data = pd.read_csv(test)
all_data = np.vstack((train_data.ix[:,1:-1], test_data.ix[:,1:-1]))
num_train = np.shape(train_data)[0]
# Transform data
print "Transforming data..."
dp = group_data(all_data, degree=2)
dt = group_data(all_data, degree=3)
y = array(train_data.ACTION)
X = all_data[:num_train]
X_2 = dp[:num_train]
X_3 = dt[:num_train]
X_test = all_data[num_train:]
X_test_2 = dp[num_train:]
X_test_3 = dt[num_train:]
X_train_all = np.hstack((X, X_2, X_3))
X_test_all = np.hstack((X_test, X_test_2, X_test_3))
num_features = X_train_all.shape[1]
model = linear_model.LogisticRegression()
# Xts holds one hot encodings for each individual feature in memory
# speeding up feature selection
Xts = [OneHotEncoder(X_train_all[:,[i]])[0] for i in range(num_features)]
print "Performing greedy feature selection..."
score_hist = []
N = 10
good_features = set([])
# Greedy feature selection loop
while len(score_hist) < 2 or score_hist[-1][0] > score_hist[-2][0]:
scores = []
for f in range(len(Xts)):
if f not in good_features:
feats = list(good_features) + [f]
Xt = sparse.hstack([Xts[j] for j in feats]).tocsr()
score = cv_loop(Xt, y, model, N)
scores.append((score, f))
print "Feature: %i Mean AUC: %f" % (f, score)
good_features.add(sorted(scores)[-1][1])
score_hist.append(sorted(scores)[-1])
print "Current features: %s" % sorted(list(good_features))
# Remove last added feature from good_features
good_features.remove(score_hist[-1][1])
good_features = sorted(list(good_features))
print "Selected features %s" % good_features
print "Performing hyperparameter selection..."
# Hyperparameter selection loop
score_hist = []
Xt = sparse.hstack([Xts[j] for j in good_features]).tocsr()
Cvals = np.logspace(-4, 4, 15, base=2)
for C in Cvals:
model.C = C
score = cv_loop(Xt, y, model, N)
score_hist.append((score,C))
print "C: %f Mean AUC: %f" %(C, score)
bestC = sorted(score_hist)[-1][1]
print "Best C value: %f" % (bestC)
print "Performing One Hot Encoding on entire dataset..."
Xt = np.vstack((X_train_all[:,good_features], X_test_all[:,good_features]))
Xt, keymap = OneHotEncoder(Xt)
X_train = Xt[:num_train]
X_test = Xt[num_train:]
print "Training full model..."
model.fit(X_train, y)
print "Making prediction and saving results..."
preds = model.predict_proba(X_test)[:,1]
create_test_submission(submit, preds)
if __name__ == "__main__":
args = { 'train': 'train.csv',
'test': 'test.csv',
'submit': 'logistic_regression_pred.csv' }
main(**args)