Building Machine Learning Systems with Python - Richert, Coelho

Chapter 6
To keep our experimentation agile, let us wrap everything together in a train_
model() function, which takes a function as a parameter that creates the classifier:
from sklearn.metrics import precision_recall_curve, auc
from sklearn.cross_validation import ShuffleSplit
def train_model(clf_factory, X, Y):
# setting random_state to get deterministic behavior
cv = ShuffleSplit(n=len(X), n_iter=10, test_size=0.3,
indices=True, random_state=0)
scores = []
pr_scores = []
for train, test in cv:
X_train, y_train = X[train], Y[train]
X_test, y_test = X[test], Y[test]
clf = clf_factory()
clf.fit(X_train, y_train)
train_score = clf.score(X_train, y_train)
test_score = clf.score(X_test, y_test)
scores.append(test_score)
proba = clf.predict_proba(X_test)
precision, recall, pr_thresholds = precision_recall_curve
(y_test, proba[:,1])
pr_scores.append(auc(recall, precision))
summary = (np.mean(scores), np.std(scores),
np.mean(pr_scores), np.std(pr_scores))
print "%.3f\t%.3f\t%.3f\t%.3f"%summary
>>> X, Y = load_sanders_data()
>>> pos_neg_idx=np.logical_or(Y=="positive", Y=="negative")
>>> X = X[pos_neg_idx]
>>> Y = Y[pos_neg_idx]
>>> Y = Y=="positive"
>>> train_model(create_ngram_model)
0.805 0.024 0.878 0.016
[ 129 ]