xgb_test.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import pandas as pd\n",
    "import sklearn.cross_validation\n",
    "import seaborn as sns\n",
    "import matplotlib as mpl\n",
    "import matplotlib.pyplot as plt\n",
    "from IPython.display import display, HTML\n",
    "\n",
    "%matplotlib inline"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>ACTION</th>\n",
       "      <th>RESOURCE</th>\n",
       "      <th>MGR_ID</th>\n",
       "      <th>ROLE_ROLLUP_1</th>\n",
       "      <th>ROLE_ROLLUP_2</th>\n",
       "      <th>ROLE_DEPTNAME</th>\n",
       "      <th>ROLE_TITLE</th>\n",
       "      <th>ROLE_FAMILY_DESC</th>\n",
       "      <th>ROLE_FAMILY</th>\n",
       "      <th>ROLE_CODE</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1</td>\n",
       "      <td>39353</td>\n",
       "      <td>85475</td>\n",
       "      <td>117961</td>\n",
       "      <td>118300</td>\n",
       "      <td>123472</td>\n",
       "      <td>117905</td>\n",
       "      <td>117906</td>\n",
       "      <td>290919</td>\n",
       "      <td>117908</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1</td>\n",
       "      <td>17183</td>\n",
       "      <td>1540</td>\n",
       "      <td>117961</td>\n",
       "      <td>118343</td>\n",
       "      <td>123125</td>\n",
       "      <td>118536</td>\n",
       "      <td>118536</td>\n",
       "      <td>308574</td>\n",
       "      <td>118539</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>1</td>\n",
       "      <td>36724</td>\n",
       "      <td>14457</td>\n",
       "      <td>118219</td>\n",
       "      <td>118220</td>\n",
       "      <td>117884</td>\n",
       "      <td>117879</td>\n",
       "      <td>267952</td>\n",
       "      <td>19721</td>\n",
       "      <td>117880</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>1</td>\n",
       "      <td>36135</td>\n",
       "      <td>5396</td>\n",
       "      <td>117961</td>\n",
       "      <td>118343</td>\n",
       "      <td>119993</td>\n",
       "      <td>118321</td>\n",
       "      <td>240983</td>\n",
       "      <td>290919</td>\n",
       "      <td>118322</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>1</td>\n",
       "      <td>42680</td>\n",
       "      <td>5905</td>\n",
       "      <td>117929</td>\n",
       "      <td>117930</td>\n",
       "      <td>119569</td>\n",
       "      <td>119323</td>\n",
       "      <td>123932</td>\n",
       "      <td>19793</td>\n",
       "      <td>119325</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   ACTION  RESOURCE  MGR_ID  ROLE_ROLLUP_1  ROLE_ROLLUP_2  ROLE_DEPTNAME  \\\n",
       "0       1     39353   85475         117961         118300         123472   \n",
       "1       1     17183    1540         117961         118343         123125   \n",
       "2       1     36724   14457         118219         118220         117884   \n",
       "3       1     36135    5396         117961         118343         119993   \n",
       "4       1     42680    5905         117929         117930         119569   \n",
       "\n",
       "   ROLE_TITLE  ROLE_FAMILY_DESC  ROLE_FAMILY  ROLE_CODE  \n",
       "0      117905            117906       290919     117908  \n",
       "1      118536            118536       308574     118539  \n",
       "2      117879            267952        19721     117880  \n",
       "3      118321            240983       290919     118322  \n",
       "4      119323            123932        19793     119325  "
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Number of instances:32769\n"
     ]
    }
   ],
   "source": [
    "train = pd.read_csv('data/train.csv')\n",
    "test = pd.read_csv('data/test.csv', index_col='id')\n",
    "\n",
    "display(train.head(5))\n",
    "\n",
    "# Break the training data into a target (\"dependent\") and inputs (\"inpedendents\")\n",
    "y_train = train.ACTION\n",
    "X_train = train.drop([\"ACTION\"], axis=1)\n",
    "print(\"Number of instances:{}\".format(len(train.index)))\n",
    "\n",
    "from collections import Counter\n",
    "\n",
    "def tally_predictions(predictions):\n",
    "    count = Counter()\n",
    "    for pred in predictions:\n",
    "        if pred == 1:\n",
    "            count[1] += 1\n",
    "        else:\n",
    "            count[0] += 1\n",
    "    print(count[0])\n",
    "    print(count[1])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## XGBoost Parameter Tuning:\n",
    "(From most important/succint to least):\n",
    "- http://datascience.stackexchange.com/questions/9364/hypertuning-xgboost-parameters\n",
    "- http://www.slideshare.net/OwenZhang2/tips-for-data-science-competitions\n",
    "- http://www.slideshare.net/odsc/owen-zhangopen-sourcetoolsanddscompetitions1\n",
    "- http://www.analyticsvidhya.com/blog/2016/03/complete-guide-parameter-tuning-xgboost-with-codes-python/"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Best parameter set found on development set with cv=10:\n",
      "\n",
      "{'min_child_weight': 1.0259783520851542, 'max_depth': 8, 'n_estimators': 200, 'learning_rate': 0.1, 'colsample_bytree': 0.4}\n",
      "\n",
      "0.778 (+/-0.039) for {'min_child_weight': 4.47213595499958, 'max_depth': 8, 'n_estimators': 100, 'learning_rate': 0.01, 'colsample_bytree': 0.3}\n",
      "0.791 (+/-0.035) for {'min_child_weight': 1.0259783520851542, 'max_depth': 7, 'n_estimators': 200, 'learning_rate': 0.01, 'colsample_bytree': 0.3}\n",
      "0.840 (+/-0.024) for {'min_child_weight': 1.0259783520851542, 'max_depth': 8, 'n_estimators': 100, 'learning_rate': 0.1, 'colsample_bytree': 0.3}\n",
      "0.830 (+/-0.027) for {'min_child_weight': 1.0259783520851542, 'max_depth': 7, 'n_estimators': 100, 'learning_rate': 0.1, 'colsample_bytree': 0.3}\n",
      "0.765 (+/-0.041) for {'min_child_weight': 1.0259783520851542, 'max_depth': 7, 'n_estimators': 100, 'learning_rate': 0.01, 'colsample_bytree': 0.3}\n",
      "0.793 (+/-0.037) for {'min_child_weight': 4.47213595499958, 'max_depth': 8, 'n_estimators': 100, 'learning_rate': 0.01, 'colsample_bytree': 0.4}\n",
      "0.823 (+/-0.032) for {'min_child_weight': 4.47213595499958, 'max_depth': 6, 'n_estimators': 100, 'learning_rate': 0.1, 'colsample_bytree': 0.5}\n",
      "0.797 (+/-0.033) for {'min_child_weight': 4.47213595499958, 'max_depth': 7, 'n_estimators': 200, 'learning_rate': 0.01, 'colsample_bytree': 0.4}\n",
      "0.836 (+/-0.030) for {'min_child_weight': 1.0259783520851542, 'max_depth': 7, 'n_estimators': 100, 'learning_rate': 0.1, 'colsample_bytree': 0.4}\n",
      "0.861 (+/-0.028) for {'min_child_weight': 1.0259783520851542, 'max_depth': 8, 'n_estimators': 200, 'learning_rate': 0.1, 'colsample_bytree': 0.5}\n",
      "0.794 (+/-0.035) for {'min_child_weight': 4.47213595499958, 'max_depth': 8, 'n_estimators': 100, 'learning_rate': 0.01, 'colsample_bytree': 0.5}\n",
      "0.836 (+/-0.027) for {'min_child_weight': 1.0259783520851542, 'max_depth': 6, 'n_estimators': 200, 'learning_rate': 0.1, 'colsample_bytree': 0.3}\n",
      "0.861 (+/-0.027) for {'min_child_weight': 1.0259783520851542, 'max_depth': 8, 'n_estimators': 200, 'learning_rate': 0.1, 'colsample_bytree': 0.4}\n",
      "0.788 (+/-0.035) for {'min_child_weight': 4.47213595499958, 'max_depth': 7, 'n_estimators': 200, 'learning_rate': 0.01, 'colsample_bytree': 0.3}\n",
      "0.842 (+/-0.027) for {'min_child_weight': 4.47213595499958, 'max_depth': 6, 'n_estimators': 200, 'learning_rate': 0.1, 'colsample_bytree': 0.5}\n",
      "0.754 (+/-0.037) for {'min_child_weight': 1.0259783520851542, 'max_depth': 6, 'n_estimators': 100, 'learning_rate': 0.01, 'colsample_bytree': 0.5}\n",
      "0.831 (+/-0.030) for {'min_child_weight': 4.47213595499958, 'max_depth': 7, 'n_estimators': 100, 'learning_rate': 0.1, 'colsample_bytree': 0.4}\n",
      "0.855 (+/-0.025) for {'min_child_weight': 1.0259783520851542, 'max_depth': 8, 'n_estimators': 200, 'learning_rate': 0.1, 'colsample_bytree': 0.3}\n",
      "0.851 (+/-0.028) for {'min_child_weight': 1.0259783520851542, 'max_depth': 8, 'n_estimators': 100, 'learning_rate': 0.1, 'colsample_bytree': 0.5}\n",
      "0.834 (+/-0.029) for {'min_child_weight': 4.47213595499958, 'max_depth': 7, 'n_estimators': 100, 'learning_rate': 0.1, 'colsample_bytree': 0.5}\n",
      "0.852 (+/-0.028) for {'min_child_weight': 1.0259783520851542, 'max_depth': 7, 'n_estimators': 200, 'learning_rate': 0.1, 'colsample_bytree': 0.4}\n",
      "0.841 (+/-0.028) for {'min_child_weight': 1.0259783520851542, 'max_depth': 7, 'n_estimators': 100, 'learning_rate': 0.1, 'colsample_bytree': 0.5}\n",
      "0.771 (+/-0.036) for {'min_child_weight': 1.0259783520851542, 'max_depth': 6, 'n_estimators': 200, 'learning_rate': 0.01, 'colsample_bytree': 0.3}\n",
      "0.855 (+/-0.028) for {'min_child_weight': 1.0259783520851542, 'max_depth': 7, 'n_estimators': 200, 'learning_rate': 0.1, 'colsample_bytree': 0.5}\n",
      "0.766 (+/-0.040) for {'min_child_weight': 4.47213595499958, 'max_depth': 7, 'n_estimators': 100, 'learning_rate': 0.01, 'colsample_bytree': 0.3}\n",
      "0.819 (+/-0.033) for {'min_child_weight': 1.0259783520851542, 'max_depth': 8, 'n_estimators': 200, 'learning_rate': 0.01, 'colsample_bytree': 0.5}\n",
      "0.820 (+/-0.028) for {'min_child_weight': 4.47213595499958, 'max_depth': 6, 'n_estimators': 100, 'learning_rate': 0.1, 'colsample_bytree': 0.4}\n",
      "0.816 (+/-0.033) for {'min_child_weight': 1.0259783520851542, 'max_depth': 8, 'n_estimators': 200, 'learning_rate': 0.01, 'colsample_bytree': 0.4}\n",
      "0.777 (+/-0.040) for {'min_child_weight': 1.0259783520851542, 'max_depth': 7, 'n_estimators': 100, 'learning_rate': 0.01, 'colsample_bytree': 0.4}\n",
      "0.845 (+/-0.029) for {'min_child_weight': 1.0259783520851542, 'max_depth': 6, 'n_estimators': 200, 'learning_rate': 0.1, 'colsample_bytree': 0.5}\n",
      "0.845 (+/-0.027) for {'min_child_weight': 4.47213595499958, 'max_depth': 8, 'n_estimators': 100, 'learning_rate': 0.1, 'colsample_bytree': 0.5}\n",
      "0.758 (+/-0.039) for {'min_child_weight': 4.47213595499958, 'max_depth': 6, 'n_estimators': 100, 'learning_rate': 0.01, 'colsample_bytree': 0.5}\n",
      "0.777 (+/-0.037) for {'min_child_weight': 1.0259783520851542, 'max_depth': 7, 'n_estimators': 100, 'learning_rate': 0.01, 'colsample_bytree': 0.5}\n",
      "0.778 (+/-0.040) for {'min_child_weight': 1.0259783520851542, 'max_depth': 8, 'n_estimators': 100, 'learning_rate': 0.01, 'colsample_bytree': 0.3}\n",
      "0.811 (+/-0.031) for {'min_child_weight': 4.47213595499958, 'max_depth': 8, 'n_estimators': 200, 'learning_rate': 0.01, 'colsample_bytree': 0.4}\n",
      "0.826 (+/-0.028) for {'min_child_weight': 1.0259783520851542, 'max_depth': 6, 'n_estimators': 100, 'learning_rate': 0.1, 'colsample_bytree': 0.4}\n",
      "0.800 (+/-0.035) for {'min_child_weight': 1.0259783520851542, 'max_depth': 7, 'n_estimators': 200, 'learning_rate': 0.01, 'colsample_bytree': 0.4}\n",
      "0.849 (+/-0.026) for {'min_child_weight': 4.47213595499958, 'max_depth': 7, 'n_estimators': 200, 'learning_rate': 0.1, 'colsample_bytree': 0.5}\n",
      "0.832 (+/-0.028) for {'min_child_weight': 4.47213595499958, 'max_depth': 6, 'n_estimators': 200, 'learning_rate': 0.1, 'colsample_bytree': 0.3}\n",
      "0.779 (+/-0.038) for {'min_child_weight': 4.47213595499958, 'max_depth': 7, 'n_estimators': 100, 'learning_rate': 0.01, 'colsample_bytree': 0.5}\n",
      "0.753 (+/-0.042) for {'min_child_weight': 4.47213595499958, 'max_depth': 6, 'n_estimators': 100, 'learning_rate': 0.01, 'colsample_bytree': 0.3}\n",
      "0.847 (+/-0.027) for {'min_child_weight': 4.47213595499958, 'max_depth': 7, 'n_estimators': 200, 'learning_rate': 0.1, 'colsample_bytree': 0.4}\n",
      "0.813 (+/-0.030) for {'min_child_weight': 4.47213595499958, 'max_depth': 6, 'n_estimators': 100, 'learning_rate': 0.1, 'colsample_bytree': 0.3}\n",
      "0.833 (+/-0.025) for {'min_child_weight': 4.47213595499958, 'max_depth': 8, 'n_estimators': 100, 'learning_rate': 0.1, 'colsample_bytree': 0.3}\n",
      "0.842 (+/-0.023) for {'min_child_weight': 4.47213595499958, 'max_depth': 7, 'n_estimators': 200, 'learning_rate': 0.1, 'colsample_bytree': 0.3}\n",
      "0.818 (+/-0.026) for {'min_child_weight': 1.0259783520851542, 'max_depth': 6, 'n_estimators': 100, 'learning_rate': 0.1, 'colsample_bytree': 0.3}\n",
      "0.762 (+/-0.041) for {'min_child_weight': 4.47213595499958, 'max_depth': 6, 'n_estimators': 100, 'learning_rate': 0.01, 'colsample_bytree': 0.4}\n",
      "0.802 (+/-0.033) for {'min_child_weight': 1.0259783520851542, 'max_depth': 7, 'n_estimators': 200, 'learning_rate': 0.01, 'colsample_bytree': 0.5}\n",
      "0.802 (+/-0.033) for {'min_child_weight': 4.47213595499958, 'max_depth': 8, 'n_estimators': 200, 'learning_rate': 0.01, 'colsample_bytree': 0.3}\n",
      "0.807 (+/-0.032) for {'min_child_weight': 1.0259783520851542, 'max_depth': 8, 'n_estimators': 200, 'learning_rate': 0.01, 'colsample_bytree': 0.3}\n",
      "\n"
     ]
    }
   ],
   "source": [
    "import xgboost as xgb\n",
    "from sklearn.grid_search import RandomizedSearchCV\n",
    "\n",
    "# Grid search XGB\n",
    "parameters = {\n",
    "    'max_depth': [6, 7, 8],\n",
    "    'learning_rate': [0.1, 0.01],\n",
    "    'n_estimators': [100, 200],\n",
    "    'min_child_weight': [1/(0.05**(1/2)), 1/(0.95**(1/2))],\n",
    "    'colsample_bytree': [0.3, 0.4, 0.5]\n",
    "}\n",
    "\n",
    "xg_clf = RandomizedSearchCV(xgb.XGBClassifier(), parameters, n_iter=50, cv=10, n_jobs=-1, scoring='roc_auc')\n",
    "xg_clf.fit(X_train, y_train)\n",
    "\n",
    "print(\"Best parameter set found on development set with cv=10:\\n\")\n",
    "print(xg_clf.best_params_)\n",
    "print()\n",
    "for params, mean_score, scores in xg_clf.grid_scores_:\n",
    "    print(\"{0:.3f} (+/-{1:.03f}) for {2}\".format(mean_score, scores.std() * 2, params))\n",
    "print()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "collapsed": false,
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Best parameter set found on development set with cv=10:\n",
      "\n",
      "{'min_child_weight': 1, 'colsample_bytree': 0.5, 'n_estimators': 200, 'learning_rate': 0.3, 'max_depth': 8}\n",
      "\n",
      "0.864 (+/-0.032) for {'min_child_weight': 1, 'colsample_bytree': 0.5, 'n_estimators': 200, 'learning_rate': 0.3, 'max_depth': 8}\n",
      "0.864 (+/-0.032) for {'min_child_weight': 1, 'colsample_bytree': 0.5, 'n_estimators': 250, 'learning_rate': 0.3, 'max_depth': 8}\n",
      "0.863 (+/-0.032) for {'min_child_weight': 1, 'colsample_bytree': 0.5, 'n_estimators': 300, 'learning_rate': 0.3, 'max_depth': 8}\n",
      "0.864 (+/-0.032) for {'min_child_weight': 1.0259783520851542, 'colsample_bytree': 0.5, 'n_estimators': 200, 'learning_rate': 0.3, 'max_depth': 8}\n",
      "0.864 (+/-0.032) for {'min_child_weight': 1.0259783520851542, 'colsample_bytree': 0.5, 'n_estimators': 250, 'learning_rate': 0.3, 'max_depth': 8}\n",
      "0.863 (+/-0.031) for {'min_child_weight': 1.0259783520851542, 'colsample_bytree': 0.5, 'n_estimators': 300, 'learning_rate': 0.3, 'max_depth': 8}\n",
      "0.863 (+/-0.031) for {'min_child_weight': 1, 'colsample_bytree': 0.5, 'n_estimators': 200, 'learning_rate': 0.3, 'max_depth': 9}\n",
      "0.863 (+/-0.028) for {'min_child_weight': 1, 'colsample_bytree': 0.5, 'n_estimators': 250, 'learning_rate': 0.3, 'max_depth': 9}\n",
      "0.862 (+/-0.030) for {'min_child_weight': 1, 'colsample_bytree': 0.5, 'n_estimators': 300, 'learning_rate': 0.3, 'max_depth': 9}\n",
      "0.863 (+/-0.031) for {'min_child_weight': 1.0259783520851542, 'colsample_bytree': 0.5, 'n_estimators': 200, 'learning_rate': 0.3, 'max_depth': 9}\n",
      "0.862 (+/-0.032) for {'min_child_weight': 1.0259783520851542, 'colsample_bytree': 0.5, 'n_estimators': 250, 'learning_rate': 0.3, 'max_depth': 9}\n",
      "0.863 (+/-0.032) for {'min_child_weight': 1.0259783520851542, 'colsample_bytree': 0.5, 'n_estimators': 300, 'learning_rate': 0.3, 'max_depth': 9}\n",
      "0.863 (+/-0.031) for {'min_child_weight': 1, 'colsample_bytree': 0.5, 'n_estimators': 200, 'learning_rate': 0.4, 'max_depth': 8}\n",
      "0.861 (+/-0.030) for {'min_child_weight': 1, 'colsample_bytree': 0.5, 'n_estimators': 250, 'learning_rate': 0.4, 'max_depth': 8}\n",
      "0.861 (+/-0.031) for {'min_child_weight': 1, 'colsample_bytree': 0.5, 'n_estimators': 300, 'learning_rate': 0.4, 'max_depth': 8}\n",
      "0.861 (+/-0.028) for {'min_child_weight': 1.0259783520851542, 'colsample_bytree': 0.5, 'n_estimators': 200, 'learning_rate': 0.4, 'max_depth': 8}\n",
      "0.860 (+/-0.029) for {'min_child_weight': 1.0259783520851542, 'colsample_bytree': 0.5, 'n_estimators': 250, 'learning_rate': 0.4, 'max_depth': 8}\n",
      "0.859 (+/-0.030) for {'min_child_weight': 1.0259783520851542, 'colsample_bytree': 0.5, 'n_estimators': 300, 'learning_rate': 0.4, 'max_depth': 8}\n",
      "0.860 (+/-0.031) for {'min_child_weight': 1, 'colsample_bytree': 0.5, 'n_estimators': 200, 'learning_rate': 0.4, 'max_depth': 9}\n",
      "0.861 (+/-0.032) for {'min_child_weight': 1, 'colsample_bytree': 0.5, 'n_estimators': 250, 'learning_rate': 0.4, 'max_depth': 9}\n",
      "0.860 (+/-0.034) for {'min_child_weight': 1, 'colsample_bytree': 0.5, 'n_estimators': 300, 'learning_rate': 0.4, 'max_depth': 9}\n",
      "0.859 (+/-0.032) for {'min_child_weight': 1.0259783520851542, 'colsample_bytree': 0.5, 'n_estimators': 200, 'learning_rate': 0.4, 'max_depth': 9}\n",
      "0.858 (+/-0.032) for {'min_child_weight': 1.0259783520851542, 'colsample_bytree': 0.5, 'n_estimators': 250, 'learning_rate': 0.4, 'max_depth': 9}\n",
      "0.858 (+/-0.030) for {'min_child_weight': 1.0259783520851542, 'colsample_bytree': 0.5, 'n_estimators': 300, 'learning_rate': 0.4, 'max_depth': 9}\n",
      "0.861 (+/-0.028) for {'min_child_weight': 1, 'colsample_bytree': 0.6, 'n_estimators': 200, 'learning_rate': 0.3, 'max_depth': 8}\n",
      "0.861 (+/-0.028) for {'min_child_weight': 1, 'colsample_bytree': 0.6, 'n_estimators': 250, 'learning_rate': 0.3, 'max_depth': 8}\n",
      "0.860 (+/-0.029) for {'min_child_weight': 1, 'colsample_bytree': 0.6, 'n_estimators': 300, 'learning_rate': 0.3, 'max_depth': 8}\n",
      "0.863 (+/-0.032) for {'min_child_weight': 1.0259783520851542, 'colsample_bytree': 0.6, 'n_estimators': 200, 'learning_rate': 0.3, 'max_depth': 8}\n",
      "0.863 (+/-0.031) for {'min_child_weight': 1.0259783520851542, 'colsample_bytree': 0.6, 'n_estimators': 250, 'learning_rate': 0.3, 'max_depth': 8}\n",
      "0.863 (+/-0.029) for {'min_child_weight': 1.0259783520851542, 'colsample_bytree': 0.6, 'n_estimators': 300, 'learning_rate': 0.3, 'max_depth': 8}\n",
      "0.861 (+/-0.033) for {'min_child_weight': 1, 'colsample_bytree': 0.6, 'n_estimators': 200, 'learning_rate': 0.3, 'max_depth': 9}\n",
      "0.861 (+/-0.031) for {'min_child_weight': 1, 'colsample_bytree': 0.6, 'n_estimators': 250, 'learning_rate': 0.3, 'max_depth': 9}\n",
      "0.861 (+/-0.032) for {'min_child_weight': 1, 'colsample_bytree': 0.6, 'n_estimators': 300, 'learning_rate': 0.3, 'max_depth': 9}\n",
      "0.863 (+/-0.036) for {'min_child_weight': 1.0259783520851542, 'colsample_bytree': 0.6, 'n_estimators': 200, 'learning_rate': 0.3, 'max_depth': 9}\n",
      "0.862 (+/-0.036) for {'min_child_weight': 1.0259783520851542, 'colsample_bytree': 0.6, 'n_estimators': 250, 'learning_rate': 0.3, 'max_depth': 9}\n",
      "0.861 (+/-0.035) for {'min_child_weight': 1.0259783520851542, 'colsample_bytree': 0.6, 'n_estimators': 300, 'learning_rate': 0.3, 'max_depth': 9}\n",
      "0.859 (+/-0.030) for {'min_child_weight': 1, 'colsample_bytree': 0.6, 'n_estimators': 200, 'learning_rate': 0.4, 'max_depth': 8}\n",
      "0.859 (+/-0.028) for {'min_child_weight': 1, 'colsample_bytree': 0.6, 'n_estimators': 250, 'learning_rate': 0.4, 'max_depth': 8}\n",
      "0.859 (+/-0.029) for {'min_child_weight': 1, 'colsample_bytree': 0.6, 'n_estimators': 300, 'learning_rate': 0.4, 'max_depth': 8}\n",
      "0.858 (+/-0.036) for {'min_child_weight': 1.0259783520851542, 'colsample_bytree': 0.6, 'n_estimators': 200, 'learning_rate': 0.4, 'max_depth': 8}\n",
      "0.857 (+/-0.034) for {'min_child_weight': 1.0259783520851542, 'colsample_bytree': 0.6, 'n_estimators': 250, 'learning_rate': 0.4, 'max_depth': 8}\n",
      "0.857 (+/-0.033) for {'min_child_weight': 1.0259783520851542, 'colsample_bytree': 0.6, 'n_estimators': 300, 'learning_rate': 0.4, 'max_depth': 8}\n",
      "0.861 (+/-0.031) for {'min_child_weight': 1, 'colsample_bytree': 0.6, 'n_estimators': 200, 'learning_rate': 0.4, 'max_depth': 9}\n",
      "0.859 (+/-0.031) for {'min_child_weight': 1, 'colsample_bytree': 0.6, 'n_estimators': 250, 'learning_rate': 0.4, 'max_depth': 9}\n",
      "0.858 (+/-0.033) for {'min_child_weight': 1, 'colsample_bytree': 0.6, 'n_estimators': 300, 'learning_rate': 0.4, 'max_depth': 9}\n",
      "0.860 (+/-0.030) for {'min_child_weight': 1.0259783520851542, 'colsample_bytree': 0.6, 'n_estimators': 200, 'learning_rate': 0.4, 'max_depth': 9}\n",
      "0.860 (+/-0.028) for {'min_child_weight': 1.0259783520851542, 'colsample_bytree': 0.6, 'n_estimators': 250, 'learning_rate': 0.4, 'max_depth': 9}\n",
      "0.859 (+/-0.030) for {'min_child_weight': 1.0259783520851542, 'colsample_bytree': 0.6, 'n_estimators': 300, 'learning_rate': 0.4, 'max_depth': 9}\n",
      "\n"
     ]
    }
   ],
   "source": [
    "import xgboost as xgb\n",
    "from sklearn.grid_search import GridSearchCV\n",
    "\n",
    "# Grid search XGB\n",
    "parameters = {\n",
    "    'max_depth': [8, 9],\n",
    "    'learning_rate': [0.3, 0.4],\n",
    "    'n_estimators': [200, 250, 300],\n",
    "    'min_child_weight': [1, 1/(0.95**(1/2))],\n",
    "    'colsample_bytree': [0.5, 0.6]\n",
    "}\n",
    "\n",
    "xg_clf = GridSearchCV(xgb.XGBClassifier(), parameters, cv=10, n_jobs=-1, scoring='roc_auc')\n",
    "xg_clf.fit(X_train, y_train)\n",
    "\n",
    "print(\"Best parameter set found on development set with cv=10:\\n\")\n",
    "print(xg_clf.best_params_)\n",
    "print()\n",
    "for params, mean_score, scores in xg_clf.grid_scores_:\n",
    "    print(\"{0:.3f} (+/-{1:.03f}) for {2}\".format(mean_score, scores.std() * 2, params))\n",
    "print()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Best parameter set found on development set with cv=10:\n",
      "\n",
      "{'min_child_weight': 1, 'colsample_bytree': 0.5, 'n_estimators': 160, 'learning_rate': 0.3, 'max_depth': 8}\n",
      "\n",
      "0.863 (+/-0.030) for {'min_child_weight': 1, 'colsample_bytree': 0.5, 'n_estimators': 120, 'learning_rate': 0.3, 'max_depth': 8}\n",
      "0.865 (+/-0.030) for {'min_child_weight': 1, 'colsample_bytree': 0.5, 'n_estimators': 140, 'learning_rate': 0.3, 'max_depth': 8}\n",
      "0.866 (+/-0.030) for {'min_child_weight': 1, 'colsample_bytree': 0.5, 'n_estimators': 160, 'learning_rate': 0.3, 'max_depth': 8}\n",
      "0.865 (+/-0.030) for {'min_child_weight': 1, 'colsample_bytree': 0.5, 'n_estimators': 180, 'learning_rate': 0.3, 'max_depth': 8}\n",
      "0.864 (+/-0.032) for {'min_child_weight': 1, 'colsample_bytree': 0.5, 'n_estimators': 200, 'learning_rate': 0.3, 'max_depth': 8}\n",
      "\n"
     ]
    }
   ],
   "source": [
    "import xgboost as xgb\n",
    "from sklearn.grid_search import GridSearchCV\n",
    "\n",
    "# Grid search XGB\n",
    "parameters = {\n",
    "    'max_depth': [8],\n",
    "    'learning_rate': [0.3],\n",
    "    'n_estimators': [120, 140, 160, 180, 200],\n",
    "    'min_child_weight': [1],\n",
    "    'colsample_bytree': [0.5]\n",
    "}\n",
    "\n",
    "xg_clf = GridSearchCV(xgb.XGBClassifier(), parameters, cv=10, n_jobs=-1, scoring='roc_auc')\n",
    "xg_clf.fit(X_train, y_train)\n",
    "\n",
    "print(\"Best parameter set found on development set with cv=10:\\n\")\n",
    "print(xg_clf.best_params_)\n",
    "print()\n",
    "for params, mean_score, scores in xg_clf.grid_scores_:\n",
    "    print(\"{0:.3f} (+/-{1:.03f}) for {2}\".format(mean_score, scores.std() * 2, params))\n",
    "print()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Best parameter set found on development set with cv=10:\n",
      "\n",
      "{'min_child_weight': 1, 'colsample_bytree': 0.5, 'n_estimators': 155, 'learning_rate': 0.3, 'max_depth': 8}\n",
      "\n",
      "0.865 (+/-0.030) for {'min_child_weight': 1, 'colsample_bytree': 0.5, 'n_estimators': 145, 'learning_rate': 0.3, 'max_depth': 8}\n",
      "0.866 (+/-0.030) for {'min_child_weight': 1, 'colsample_bytree': 0.5, 'n_estimators': 150, 'learning_rate': 0.3, 'max_depth': 8}\n",
      "0.866 (+/-0.029) for {'min_child_weight': 1, 'colsample_bytree': 0.5, 'n_estimators': 155, 'learning_rate': 0.3, 'max_depth': 8}\n",
      "0.866 (+/-0.030) for {'min_child_weight': 1, 'colsample_bytree': 0.5, 'n_estimators': 160, 'learning_rate': 0.3, 'max_depth': 8}\n",
      "0.866 (+/-0.030) for {'min_child_weight': 1, 'colsample_bytree': 0.5, 'n_estimators': 165, 'learning_rate': 0.3, 'max_depth': 8}\n",
      "\n"
     ]
    }
   ],
   "source": [
    "import xgboost as xgb\n",
    "from sklearn.grid_search import GridSearchCV\n",
    "\n",
    "# Grid search XGB\n",
    "parameters = {\n",
    "    'max_depth': [8],\n",
    "    'learning_rate': [0.2, 0.3],\n",
    "    'n_estimators': [145, 150, 155, 160, 165],\n",
    "    'min_child_weight': [1],\n",
    "    'colsample_bytree': [0.5]\n",
    "}\n",
    "\n",
    "xg_clf = GridSearchCV(xgb.XGBClassifier(), parameters, cv=10, n_jobs=-1, scoring='roc_auc')\n",
    "xg_clf.fit(X_train, y_train)\n",
    "\n",
    "print(\"Best parameter set found on development set with cv=10:\\n\")\n",
    "print(xg_clf.best_params_)\n",
    "print()\n",
    "for params, mean_score, scores in xg_clf.grid_scores_:\n",
    "    print(\"{0:.3f} (+/-{1:.03f}) for {2}\".format(mean_score, scores.std() * 2, params))\n",
    "print()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Best parameter set found on development set with cv=10:\n",
      "\n",
      "{'min_child_weight': 1, 'colsample_bytree': 0.5, 'n_estimators': 155, 'learning_rate': 0.3, 'max_depth': 8}\n",
      "\n",
      "0.863 (+/-0.029) for {'min_child_weight': 1, 'colsample_bytree': 0.5, 'n_estimators': 155, 'learning_rate': 0.29, 'max_depth': 8}\n",
      "0.866 (+/-0.029) for {'min_child_weight': 1, 'colsample_bytree': 0.5, 'n_estimators': 155, 'learning_rate': 0.3, 'max_depth': 8}\n",
      "0.865 (+/-0.029) for {'min_child_weight': 1, 'colsample_bytree': 0.5, 'n_estimators': 155, 'learning_rate': 0.31, 'max_depth': 8}\n",
      "\n"
     ]
    }
   ],
   "source": [
    "import xgboost as xgb\n",
    "from sklearn.grid_search import GridSearchCV\n",
    "\n",
    "# Grid search XGB\n",
    "parameters = {\n",
    "    'max_depth': [8],\n",
    "    'learning_rate': [0.29, 0.3, 0.31],\n",
    "    'n_estimators': [155],\n",
    "    'min_child_weight': [1],\n",
    "    'colsample_bytree': [0.5]\n",
    "}\n",
    "\n",
    "xg_clf = GridSearchCV(xgb.XGBClassifier(), parameters, cv=10, n_jobs=-1, scoring='roc_auc')\n",
    "xg_clf.fit(X_train, y_train)\n",
    "\n",
    "print(\"Best parameter set found on development set with cv=10:\\n\")\n",
    "print(xg_clf.best_params_)\n",
    "print()\n",
    "for params, mean_score, scores in xg_clf.grid_scores_:\n",
    "    print(\"{0:.3f} (+/-{1:.03f}) for {2}\".format(mean_score, scores.std() * 2, params))\n",
    "print()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Best parameter set found on development set with cv=10:\n",
      "\n",
      "{'min_child_weight': 1, 'colsample_bytree': 0.5, 'n_estimators': 155, 'learning_rate': 0.3, 'max_depth': 8}\n",
      "\n",
      "0.862 (+/-0.033) for {'min_child_weight': 0.99, 'colsample_bytree': 0.5, 'n_estimators': 155, 'learning_rate': 0.3, 'max_depth': 8}\n",
      "0.863 (+/-0.031) for {'min_child_weight': 0.995, 'colsample_bytree': 0.5, 'n_estimators': 155, 'learning_rate': 0.3, 'max_depth': 8}\n",
      "0.866 (+/-0.029) for {'min_child_weight': 1, 'colsample_bytree': 0.5, 'n_estimators': 155, 'learning_rate': 0.3, 'max_depth': 8}\n",
      "\n"
     ]
    }
   ],
   "source": [
    "import xgboost as xgb\n",
    "from sklearn.grid_search import GridSearchCV\n",
    "\n",
    "# Grid search XGB\n",
    "parameters = {\n",
    "    'max_depth': [8],\n",
    "    'learning_rate': [0.3],\n",
    "    'n_estimators': [155],\n",
    "    'min_child_weight': [.99, .995, 1],\n",
    "    'colsample_bytree': [0.5]\n",
    "}\n",
    "\n",
    "xg_clf = GridSearchCV(xgb.XGBClassifier(), parameters, cv=10, n_jobs=-1, scoring='roc_auc')\n",
    "xg_clf.fit(X_train, y_train)\n",
    "\n",
    "print(\"Best parameter set found on development set with cv=10:\\n\")\n",
    "print(xg_clf.best_params_)\n",
    "print()\n",
    "for params, mean_score, scores in xg_clf.grid_scores_:\n",
    "    print(\"{0:.3f} (+/-{1:.03f}) for {2}\".format(mean_score, scores.std() * 2, params))\n",
    "print()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Best parameter set found on development set with cv=10:\n",
      "\n",
      "{'min_child_weight': 1, 'colsample_bytree': 0.5, 'n_estimators': 155, 'learning_rate': 0.3, 'max_depth': 8}\n",
      "\n",
      "0.866 (+/-0.029) for {'min_child_weight': 1, 'colsample_bytree': 0.5, 'n_estimators': 155, 'learning_rate': 0.3, 'max_depth': 8}\n",
      "0.860 (+/-0.029) for {'min_child_weight': 1, 'colsample_bytree': 0.6, 'n_estimators': 155, 'learning_rate': 0.3, 'max_depth': 8}\n",
      "0.858 (+/-0.032) for {'min_child_weight': 1, 'colsample_bytree': 0.7, 'n_estimators': 155, 'learning_rate': 0.3, 'max_depth': 8}\n",
      "0.858 (+/-0.035) for {'min_child_weight': 1, 'colsample_bytree': 0.8, 'n_estimators': 155, 'learning_rate': 0.3, 'max_depth': 8}\n",
      "0.854 (+/-0.042) for {'min_child_weight': 1, 'colsample_bytree': 0.9, 'n_estimators': 155, 'learning_rate': 0.3, 'max_depth': 8}\n",
      "0.854 (+/-0.039) for {'min_child_weight': 1, 'colsample_bytree': 1, 'n_estimators': 155, 'learning_rate': 0.3, 'max_depth': 8}\n",
      "\n"
     ]
    }
   ],
   "source": [
    "import xgboost as xgb\n",
    "from sklearn.grid_search import GridSearchCV\n",
    "\n",
    "# Grid search XGB\n",
    "parameters = {\n",
    "    'max_depth': [8],\n",
    "    'learning_rate': [0.3],\n",
    "    'n_estimators': [155],\n",
    "    'min_child_weight': [1],\n",
    "    'colsample_bytree': [0.5, 0.6, 0.7, 0.8, 0.9, 1]\n",
    "}\n",
    "\n",
    "xg_clf = GridSearchCV(xgb.XGBClassifier(), parameters, cv=10, n_jobs=-1, scoring='roc_auc')\n",
    "xg_clf.fit(X_train, y_train)\n",
    "\n",
    "print(\"Best parameter set found on development set with cv=10:\\n\")\n",
    "print(xg_clf.best_params_)\n",
    "print()\n",
    "for params, mean_score, scores in xg_clf.grid_scores_:\n",
    "    print(\"{0:.3f} (+/-{1:.03f}) for {2}\".format(mean_score, scores.std() * 2, params))\n",
    "print()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Best parameter set found on development set with cv=10:\n",
      "\n",
      "{'min_child_weight': 1, 'colsample_bytree': 0.5, 'n_estimators': 155, 'learning_rate': 0.3, 'max_depth': 8}\n",
      "\n",
      "0.859 (+/-0.030) for {'min_child_weight': 1, 'colsample_bytree': 0.3, 'n_estimators': 155, 'learning_rate': 0.3, 'max_depth': 8}\n",
      "0.862 (+/-0.034) for {'min_child_weight': 1, 'colsample_bytree': 0.4, 'n_estimators': 155, 'learning_rate': 0.3, 'max_depth': 8}\n",
      "0.866 (+/-0.029) for {'min_child_weight': 1, 'colsample_bytree': 0.5, 'n_estimators': 155, 'learning_rate': 0.3, 'max_depth': 8}\n",
      "\n"
     ]
    }
   ],
   "source": [
    "import xgboost as xgb\n",
    "from sklearn.grid_search import GridSearchCV\n",
    "\n",
    "# Grid search XGB\n",
    "parameters = {\n",
    "    'max_depth': [8],\n",
    "    'learning_rate': [0.3],\n",
    "    'n_estimators': [155],\n",
    "    'min_child_weight': [1],\n",
    "    'colsample_bytree': [0.3, 0.4, 0.5]\n",
    "}\n",
    "\n",
    "xg_clf = GridSearchCV(xgb.XGBClassifier(), parameters, cv=10, n_jobs=-1, scoring='roc_auc')\n",
    "xg_clf.fit(X_train, y_train)\n",
    "\n",
    "print(\"Best parameter set found on development set with cv=10:\\n\")\n",
    "print(xg_clf.best_params_)\n",
    "print()\n",
    "for params, mean_score, scores in xg_clf.grid_scores_:\n",
    "    print(\"{0:.3f} (+/-{1:.03f}) for {2}\".format(mean_score, scores.std() * 2, params))\n",
    "print()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "predictions = xg_clf.predict_proba(test)\n",
    "\n",
    "# Spit out predictions to a file\n",
    "pred_write = enumerate(predictions, start=1)\n",
    "with open('output/xgb_155trees_colsampletree.5_learningrate0.3_maxdepth8.csv', 'w') as f:\n",
    "    f.write('Id,Action\\n')\n",
    "    for instance, prediction in pred_write:\n",
    "        f.write('{},{}\\n'.format(instance, prediction[1]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Best parameter set found on development set with cv=10:\n",
      "\n",
      "{'min_child_weight': 1, 'colsample_bytree': 0.5, 'n_estimators': 155, 'learning_rate': 0.3, 'max_depth': 8}\n",
      "Best score: 0.8658009059401618\n",
      "\n",
      "\n",
      "0.844 (+/-0.026) for {'min_child_weight': 1, 'colsample_bytree': 0.3, 'n_estimators': 130, 'learning_rate': 0.3, 'max_depth': 6}\n",
      "0.845 (+/-0.025) for {'min_child_weight': 1, 'colsample_bytree': 0.3, 'n_estimators': 135, 'learning_rate': 0.3, 'max_depth': 6}\n",
      "0.846 (+/-0.026) for {'min_child_weight': 1, 'colsample_bytree': 0.3, 'n_estimators': 140, 'learning_rate': 0.3, 'max_depth': 6}\n",
      "0.847 (+/-0.027) for {'min_child_weight': 1, 'colsample_bytree': 0.3, 'n_estimators': 145, 'learning_rate': 0.3, 'max_depth': 6}\n",
      "0.848 (+/-0.027) for {'min_child_weight': 1, 'colsample_bytree': 0.3, 'n_estimators': 150, 'learning_rate': 0.3, 'max_depth': 6}\n",
      "0.848 (+/-0.026) for {'min_child_weight': 1, 'colsample_bytree': 0.3, 'n_estimators': 155, 'learning_rate': 0.3, 'max_depth': 6}\n",
      "0.849 (+/-0.027) for {'min_child_weight': 1, 'colsample_bytree': 0.3, 'n_estimators': 160, 'learning_rate': 0.3, 'max_depth': 6}\n",
      "0.849 (+/-0.027) for {'min_child_weight': 1, 'colsample_bytree': 0.3, 'n_estimators': 165, 'learning_rate': 0.3, 'max_depth': 6}\n",
      "0.852 (+/-0.031) for {'min_child_weight': 1, 'colsample_bytree': 0.3, 'n_estimators': 130, 'learning_rate': 0.3, 'max_depth': 7}\n",
      "0.852 (+/-0.031) for {'min_child_weight': 1, 'colsample_bytree': 0.3, 'n_estimators': 135, 'learning_rate': 0.3, 'max_depth': 7}\n",
      "0.852 (+/-0.031) for {'min_child_weight': 1, 'colsample_bytree': 0.3, 'n_estimators': 140, 'learning_rate': 0.3, 'max_depth': 7}\n",
      "0.853 (+/-0.032) for {'min_child_weight': 1, 'colsample_bytree': 0.3, 'n_estimators': 145, 'learning_rate': 0.3, 'max_depth': 7}\n",
      "0.854 (+/-0.032) for {'min_child_weight': 1, 'colsample_bytree': 0.3, 'n_estimators': 150, 'learning_rate': 0.3, 'max_depth': 7}\n",
      "0.854 (+/-0.031) for {'min_child_weight': 1, 'colsample_bytree': 0.3, 'n_estimators': 155, 'learning_rate': 0.3, 'max_depth': 7}\n",
      "0.855 (+/-0.032) for {'min_child_weight': 1, 'colsample_bytree': 0.3, 'n_estimators': 160, 'learning_rate': 0.3, 'max_depth': 7}\n",
      "0.855 (+/-0.032) for {'min_child_weight': 1, 'colsample_bytree': 0.3, 'n_estimators': 165, 'learning_rate': 0.3, 'max_depth': 7}\n",
      "0.856 (+/-0.029) for {'min_child_weight': 1, 'colsample_bytree': 0.3, 'n_estimators': 130, 'learning_rate': 0.3, 'max_depth': 8}\n",
      "0.856 (+/-0.029) for {'min_child_weight': 1, 'colsample_bytree': 0.3, 'n_estimators': 135, 'learning_rate': 0.3, 'max_depth': 8}\n",
      "0.857 (+/-0.029) for {'min_child_weight': 1, 'colsample_bytree': 0.3, 'n_estimators': 140, 'learning_rate': 0.3, 'max_depth': 8}\n",
      "0.858 (+/-0.029) for {'min_child_weight': 1, 'colsample_bytree': 0.3, 'n_estimators': 145, 'learning_rate': 0.3, 'max_depth': 8}\n",
      "0.859 (+/-0.030) for {'min_child_weight': 1, 'colsample_bytree': 0.3, 'n_estimators': 150, 'learning_rate': 0.3, 'max_depth': 8}\n",
      "0.859 (+/-0.030) for {'min_child_weight': 1, 'colsample_bytree': 0.3, 'n_estimators': 155, 'learning_rate': 0.3, 'max_depth': 8}\n",
      "0.859 (+/-0.030) for {'min_child_weight': 1, 'colsample_bytree': 0.3, 'n_estimators': 160, 'learning_rate': 0.3, 'max_depth': 8}\n",
      "0.859 (+/-0.030) for {'min_child_weight': 1, 'colsample_bytree': 0.3, 'n_estimators': 165, 'learning_rate': 0.3, 'max_depth': 8}\n",
      "0.852 (+/-0.029) for {'min_child_weight': 1, 'colsample_bytree': 0.4, 'n_estimators': 130, 'learning_rate': 0.3, 'max_depth': 6}\n",
      "0.851 (+/-0.029) for {'min_child_weight': 1, 'colsample_bytree': 0.4, 'n_estimators': 135, 'learning_rate': 0.3, 'max_depth': 6}\n",
      "0.852 (+/-0.029) for {'min_child_weight': 1, 'colsample_bytree': 0.4, 'n_estimators': 140, 'learning_rate': 0.3, 'max_depth': 6}\n",
      "0.852 (+/-0.030) for {'min_child_weight': 1, 'colsample_bytree': 0.4, 'n_estimators': 145, 'learning_rate': 0.3, 'max_depth': 6}\n",
      "0.853 (+/-0.029) for {'min_child_weight': 1, 'colsample_bytree': 0.4, 'n_estimators': 150, 'learning_rate': 0.3, 'max_depth': 6}\n",
      "0.853 (+/-0.030) for {'min_child_weight': 1, 'colsample_bytree': 0.4, 'n_estimators': 155, 'learning_rate': 0.3, 'max_depth': 6}\n",
      "0.853 (+/-0.031) for {'min_child_weight': 1, 'colsample_bytree': 0.4, 'n_estimators': 160, 'learning_rate': 0.3, 'max_depth': 6}\n",
      "0.854 (+/-0.030) for {'min_child_weight': 1, 'colsample_bytree': 0.4, 'n_estimators': 165, 'learning_rate': 0.3, 'max_depth': 6}\n",
      "0.858 (+/-0.026) for {'min_child_weight': 1, 'colsample_bytree': 0.4, 'n_estimators': 130, 'learning_rate': 0.3, 'max_depth': 7}\n",
      "0.858 (+/-0.025) for {'min_child_weight': 1, 'colsample_bytree': 0.4, 'n_estimators': 135, 'learning_rate': 0.3, 'max_depth': 7}\n",
      "0.859 (+/-0.024) for {'min_child_weight': 1, 'colsample_bytree': 0.4, 'n_estimators': 140, 'learning_rate': 0.3, 'max_depth': 7}\n",
      "0.859 (+/-0.024) for {'min_child_weight': 1, 'colsample_bytree': 0.4, 'n_estimators': 145, 'learning_rate': 0.3, 'max_depth': 7}\n",
      "0.860 (+/-0.024) for {'min_child_weight': 1, 'colsample_bytree': 0.4, 'n_estimators': 150, 'learning_rate': 0.3, 'max_depth': 7}\n",
      "0.859 (+/-0.025) for {'min_child_weight': 1, 'colsample_bytree': 0.4, 'n_estimators': 155, 'learning_rate': 0.3, 'max_depth': 7}\n",
      "0.859 (+/-0.024) for {'min_child_weight': 1, 'colsample_bytree': 0.4, 'n_estimators': 160, 'learning_rate': 0.3, 'max_depth': 7}\n",
      "0.859 (+/-0.025) for {'min_child_weight': 1, 'colsample_bytree': 0.4, 'n_estimators': 165, 'learning_rate': 0.3, 'max_depth': 7}\n",
      "0.862 (+/-0.033) for {'min_child_weight': 1, 'colsample_bytree': 0.4, 'n_estimators': 130, 'learning_rate': 0.3, 'max_depth': 8}\n",
      "0.862 (+/-0.033) for {'min_child_weight': 1, 'colsample_bytree': 0.4, 'n_estimators': 135, 'learning_rate': 0.3, 'max_depth': 8}\n",
      "0.862 (+/-0.032) for {'min_child_weight': 1, 'colsample_bytree': 0.4, 'n_estimators': 140, 'learning_rate': 0.3, 'max_depth': 8}\n",
      "0.862 (+/-0.032) for {'min_child_weight': 1, 'colsample_bytree': 0.4, 'n_estimators': 145, 'learning_rate': 0.3, 'max_depth': 8}\n",
      "0.862 (+/-0.033) for {'min_child_weight': 1, 'colsample_bytree': 0.4, 'n_estimators': 150, 'learning_rate': 0.3, 'max_depth': 8}\n",
      "0.862 (+/-0.034) for {'min_child_weight': 1, 'colsample_bytree': 0.4, 'n_estimators': 155, 'learning_rate': 0.3, 'max_depth': 8}\n",
      "0.862 (+/-0.033) for {'min_child_weight': 1, 'colsample_bytree': 0.4, 'n_estimators': 160, 'learning_rate': 0.3, 'max_depth': 8}\n",
      "0.862 (+/-0.033) for {'min_child_weight': 1, 'colsample_bytree': 0.4, 'n_estimators': 165, 'learning_rate': 0.3, 'max_depth': 8}\n",
      "0.856 (+/-0.029) for {'min_child_weight': 1, 'colsample_bytree': 0.5, 'n_estimators': 130, 'learning_rate': 0.3, 'max_depth': 6}\n",
      "0.856 (+/-0.030) for {'min_child_weight': 1, 'colsample_bytree': 0.5, 'n_estimators': 135, 'learning_rate': 0.3, 'max_depth': 6}\n",
      "0.857 (+/-0.031) for {'min_child_weight': 1, 'colsample_bytree': 0.5, 'n_estimators': 140, 'learning_rate': 0.3, 'max_depth': 6}\n",
      "0.858 (+/-0.031) for {'min_child_weight': 1, 'colsample_bytree': 0.5, 'n_estimators': 145, 'learning_rate': 0.3, 'max_depth': 6}\n",
      "0.858 (+/-0.032) for {'min_child_weight': 1, 'colsample_bytree': 0.5, 'n_estimators': 150, 'learning_rate': 0.3, 'max_depth': 6}\n",
      "0.858 (+/-0.033) for {'min_child_weight': 1, 'colsample_bytree': 0.5, 'n_estimators': 155, 'learning_rate': 0.3, 'max_depth': 6}\n",
      "0.858 (+/-0.034) for {'min_child_weight': 1, 'colsample_bytree': 0.5, 'n_estimators': 160, 'learning_rate': 0.3, 'max_depth': 6}\n",
      "0.859 (+/-0.035) for {'min_child_weight': 1, 'colsample_bytree': 0.5, 'n_estimators': 165, 'learning_rate': 0.3, 'max_depth': 6}\n",
      "0.862 (+/-0.028) for {'min_child_weight': 1, 'colsample_bytree': 0.5, 'n_estimators': 130, 'learning_rate': 0.3, 'max_depth': 7}\n",
      "0.862 (+/-0.027) for {'min_child_weight': 1, 'colsample_bytree': 0.5, 'n_estimators': 135, 'learning_rate': 0.3, 'max_depth': 7}\n",
      "0.862 (+/-0.027) for {'min_child_weight': 1, 'colsample_bytree': 0.5, 'n_estimators': 140, 'learning_rate': 0.3, 'max_depth': 7}\n",
      "0.862 (+/-0.028) for {'min_child_weight': 1, 'colsample_bytree': 0.5, 'n_estimators': 145, 'learning_rate': 0.3, 'max_depth': 7}\n",
      "0.863 (+/-0.027) for {'min_child_weight': 1, 'colsample_bytree': 0.5, 'n_estimators': 150, 'learning_rate': 0.3, 'max_depth': 7}\n",
      "0.863 (+/-0.028) for {'min_child_weight': 1, 'colsample_bytree': 0.5, 'n_estimators': 155, 'learning_rate': 0.3, 'max_depth': 7}\n",
      "0.863 (+/-0.028) for {'min_child_weight': 1, 'colsample_bytree': 0.5, 'n_estimators': 160, 'learning_rate': 0.3, 'max_depth': 7}\n",
      "0.863 (+/-0.029) for {'min_child_weight': 1, 'colsample_bytree': 0.5, 'n_estimators': 165, 'learning_rate': 0.3, 'max_depth': 7}\n",
      "0.864 (+/-0.030) for {'min_child_weight': 1, 'colsample_bytree': 0.5, 'n_estimators': 130, 'learning_rate': 0.3, 'max_depth': 8}\n",
      "0.865 (+/-0.030) for {'min_child_weight': 1, 'colsample_bytree': 0.5, 'n_estimators': 135, 'learning_rate': 0.3, 'max_depth': 8}\n",
      "0.865 (+/-0.030) for {'min_child_weight': 1, 'colsample_bytree': 0.5, 'n_estimators': 140, 'learning_rate': 0.3, 'max_depth': 8}\n",
      "0.865 (+/-0.030) for {'min_child_weight': 1, 'colsample_bytree': 0.5, 'n_estimators': 145, 'learning_rate': 0.3, 'max_depth': 8}\n",
      "0.866 (+/-0.030) for {'min_child_weight': 1, 'colsample_bytree': 0.5, 'n_estimators': 150, 'learning_rate': 0.3, 'max_depth': 8}\n",
      "0.866 (+/-0.029) for {'min_child_weight': 1, 'colsample_bytree': 0.5, 'n_estimators': 155, 'learning_rate': 0.3, 'max_depth': 8}\n",
      "0.866 (+/-0.030) for {'min_child_weight': 1, 'colsample_bytree': 0.5, 'n_estimators': 160, 'learning_rate': 0.3, 'max_depth': 8}\n",
      "0.866 (+/-0.030) for {'min_child_weight': 1, 'colsample_bytree': 0.5, 'n_estimators': 165, 'learning_rate': 0.3, 'max_depth': 8}\n",
      "\n"
     ]
    }
   ],
   "source": [
    "# Grid search XGB\n",
    "parameters = {\n",
    "    'max_depth': list(range(6, 9)),\n",
    "    'learning_rate': [0.3],\n",
    "    'n_estimators': list(range(130, 170, 5)),\n",
    "    'min_child_weight': [1],\n",
    "    'colsample_bytree': [0.3, 0.4, 0.5]\n",
    "}\n",
    "\n",
    "xg_clf = GridSearchCV(xgb.XGBClassifier(), parameters, cv=10, n_jobs=-1, scoring='roc_auc')\n",
    "xg_clf.fit(X_train, y_train)\n",
    "\n",
    "print(\"Best parameter set found on development set with cv=10:\\n\")\n",
    "print(xg_clf.best_params_)\n",
    "print(\"Best score: {}\\n\".format(xg_clf.best_score_))\n",
    "print()\n",
    "for params, mean_score, scores in xg_clf.grid_scores_:\n",
    "    print(\"{0:.3f} (+/-{1:.03f}) for {2}\".format(mean_score, scores.std() * 2, params))\n",
    "print()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Best parameter set found on development set with cv=10:\n",
      "\n",
      "{'min_child_weight': 1, 'colsample_bytree': 0.5, 'n_estimators': 155, 'learning_rate': 0.3, 'max_depth': 8}\n",
      "Best score: 0.8658009059401618\n",
      "\n",
      "\n",
      "0.866 (+/-0.029) for {'min_child_weight': 1, 'colsample_bytree': 0.5, 'n_estimators': 155, 'learning_rate': 0.3, 'max_depth': 8}\n",
      "0.864 (+/-0.033) for {'min_child_weight': 1, 'colsample_bytree': 0.5, 'n_estimators': 400, 'learning_rate': 0.3, 'max_depth': 8}\n",
      "0.861 (+/-0.033) for {'min_child_weight': 1, 'colsample_bytree': 0.5, 'n_estimators': 1000, 'learning_rate': 0.3, 'max_depth': 8}\n",
      "\n"
     ]
    }
   ],
   "source": [
    "# Grid search XGB\n",
    "parameters = {\n",
    "    'max_depth': [8],\n",
    "    'learning_rate': [0.3],\n",
    "    'n_estimators': [155, 400, 1000],\n",
    "    'min_child_weight': [1],\n",
    "    'colsample_bytree': [0.5],\n",
    "}\n",
    "\n",
    "xg_clf = GridSearchCV(xgb.XGBClassifier(), parameters, cv=10, n_jobs=-1, scoring='roc_auc')\n",
    "xg_clf.fit(X_train, y_train)\n",
    "\n",
    "print(\"Best parameter set found on development set with cv=10:\\n\")\n",
    "print(xg_clf.best_params_)\n",
    "print(\"Best score: {}\\n\".format(xg_clf.best_score_))\n",
    "print()\n",
    "for params, mean_score, scores in xg_clf.grid_scores_:\n",
    "    print(\"{0:.3f} (+/-{1:.03f}) for {2}\".format(mean_score, scores.std() * 2, params))\n",
    "print()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Best parameter set found on development set with cv=10:\n",
      "\n",
      "{'min_child_weight': 1, 'colsample_bytree': 0.5, 'n_estimators': 155, 'learning_rate': 0.06451612903225806, 'max_depth': 8}\n",
      "Best score: 0.8516473386907805\n",
      "\n",
      "\n",
      "0.818 (+/-0.030) for {'min_child_weight': 1, 'colsample_bytree': 0.5, 'n_estimators': 155, 'learning_rate': 0.012903225806451613, 'max_depth': 8}\n",
      "0.831 (+/-0.031) for {'min_child_weight': 1, 'colsample_bytree': 0.5, 'n_estimators': 155, 'learning_rate': 0.01935483870967742, 'max_depth': 8}\n",
      "0.838 (+/-0.027) for {'min_child_weight': 1, 'colsample_bytree': 0.5, 'n_estimators': 155, 'learning_rate': 0.025806451612903226, 'max_depth': 8}\n",
      "0.843 (+/-0.025) for {'min_child_weight': 1, 'colsample_bytree': 0.5, 'n_estimators': 155, 'learning_rate': 0.03225806451612903, 'max_depth': 8}\n",
      "0.844 (+/-0.025) for {'min_child_weight': 1, 'colsample_bytree': 0.5, 'n_estimators': 155, 'learning_rate': 0.03870967741935484, 'max_depth': 8}\n",
      "0.846 (+/-0.027) for {'min_child_weight': 1, 'colsample_bytree': 0.5, 'n_estimators': 155, 'learning_rate': 0.04516129032258064, 'max_depth': 8}\n",
      "0.847 (+/-0.029) for {'min_child_weight': 1, 'colsample_bytree': 0.5, 'n_estimators': 155, 'learning_rate': 0.05161290322580645, 'max_depth': 8}\n",
      "0.850 (+/-0.027) for {'min_child_weight': 1, 'colsample_bytree': 0.5, 'n_estimators': 155, 'learning_rate': 0.05806451612903226, 'max_depth': 8}\n",
      "0.852 (+/-0.023) for {'min_child_weight': 1, 'colsample_bytree': 0.5, 'n_estimators': 155, 'learning_rate': 0.06451612903225806, 'max_depth': 8}\n",
      "\n"
     ]
    }
   ],
   "source": [
    "# Grid search XGB\n",
    "learning_rate = [x/155 for x in range(2, 11)] # 2~10 divided by # of trees.\n",
    "parameters = {\n",
    "    'max_depth': [8],\n",
    "    'learning_rate': learning_rate,\n",
    "    'n_estimators': [155],\n",
    "    'min_child_weight': [1],\n",
    "    'colsample_bytree': [0.5],\n",
    "}\n",
    "\n",
    "xg_clf = GridSearchCV(xgb.XGBClassifier(), parameters, cv=10, n_jobs=-1, scoring='roc_auc')\n",
    "xg_clf.fit(X_train, y_train)\n",
    "\n",
    "print(\"Best parameter set found on development set with cv=10:\\n\")\n",
    "print(xg_clf.best_params_)\n",
    "print(\"Best score: {}\\n\".format(xg_clf.best_score_))\n",
    "print()\n",
    "for params, mean_score, scores in xg_clf.grid_scores_:\n",
    "    print(\"{0:.3f} (+/-{1:.03f}) for {2}\".format(mean_score, scores.std() * 2, params))\n",
    "print()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Best parameter set found on development set with cv=10:\n",
      "\n",
      "{'min_child_weight': 0.6, 'colsample_bytree': 0.5, 'n_estimators': 155, 'learning_rate': 0.3, 'max_depth': 8}\n",
      "Best score: 0.8667531493479463\n",
      "\n",
      "\n",
      "0.866 (+/-0.029) for {'min_child_weight': 1, 'colsample_bytree': 0.5, 'n_estimators': 155, 'learning_rate': 0.3, 'max_depth': 8}\n",
      "0.867 (+/-0.027) for {'min_child_weight': 0.6, 'colsample_bytree': 0.5, 'n_estimators': 155, 'learning_rate': 0.3, 'max_depth': 8}\n",
      "\n"
     ]
    }
   ],
   "source": [
    "# Grid search XGB\n",
    "learning_rate = [0.3]\n",
    "subsample = []\n",
    "parameters = {\n",
    "    'max_depth': [8],\n",
    "    'learning_rate': learning_rate,\n",
    "    'n_estimators': [155],\n",
    "    'min_child_weight': [1, 3/5],\n",
    "    'colsample_bytree': [0.5],\n",
    "}\n",
    "\n",
    "xg_clf = GridSearchCV(xgb.XGBClassifier(), parameters, cv=10, n_jobs=-1, scoring='roc_auc')\n",
    "xg_clf.fit(X_train, y_train)\n",
    "\n",
    "print(\"Best parameter set found on development set with cv=10:\\n\")\n",
    "print(xg_clf.best_params_)\n",
    "print(\"Best score: {}\\n\".format(xg_clf.best_score_))\n",
    "print()\n",
    "for params, mean_score, scores in xg_clf.grid_scores_:\n",
    "    print(\"{0:.3f} (+/-{1:.03f}) for {2}\".format(mean_score, scores.std() * 2, params))\n",
    "print()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "GridSearchCV(cv=10, error_score='raise',\n",
       "       estimator=XGBClassifier(base_score=0.5, colsample_bylevel=1, colsample_bytree=1,\n",
       "       gamma=0, learning_rate=0.1, max_delta_step=0, max_depth=3,\n",
       "       min_child_weight=1, missing=None, n_estimators=100, nthread=-1,\n",
       "       objective='binary:logistic', reg_alpha=0, reg_lambda=1,\n",
       "       scale_pos_weight=1, seed=0, silent=True, subsample=1),\n",
       "       fit_params={}, iid=True, n_jobs=-1,\n",
       "       param_grid={'min_child_weight': [1, 0.6], 'max_depth': [8], 'n_estimators': [155], 'learning_rate': [0.3], 'colsample_bytree': [0.5]},\n",
       "       pre_dispatch='2*n_jobs', refit=True, scoring='roc_auc', verbose=0)"
      ]
     },
     "execution_count": 28,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "xg_clf"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Best parameter set found on development set with cv=10:\n",
      "\n",
      "{'min_child_weight': 0.6, 'max_depth': 8, 'n_estimators': 155, 'learning_rate': 0.3, 'subsample': 1.0, 'colsample_bytree': 0.4}\n",
      "Best score: 0.8635881122375122\n",
      "\n",
      "\n",
      "0.852 (+/-0.030) for {'min_child_weight': 0.6, 'max_depth': 8, 'n_estimators': 155, 'learning_rate': 0.3, 'subsample': 0.5, 'colsample_bytree': 0.4}\n",
      "0.861 (+/-0.030) for {'min_child_weight': 0.6, 'max_depth': 8, 'n_estimators': 155, 'learning_rate': 0.3, 'subsample': 0.75, 'colsample_bytree': 0.4}\n",
      "0.864 (+/-0.034) for {'min_child_weight': 0.6, 'max_depth': 8, 'n_estimators': 155, 'learning_rate': 0.3, 'subsample': 1.0, 'colsample_bytree': 0.4}\n",
      "0.851 (+/-0.038) for {'min_child_weight': 0.6, 'max_depth': 8, 'n_estimators': 155, 'learning_rate': 0.3, 'subsample': 0.5, 'colsample_bytree': 0.6}\n",
      "0.860 (+/-0.032) for {'min_child_weight': 0.6, 'max_depth': 8, 'n_estimators': 155, 'learning_rate': 0.3, 'subsample': 0.75, 'colsample_bytree': 0.6}\n",
      "0.861 (+/-0.032) for {'min_child_weight': 0.6, 'max_depth': 8, 'n_estimators': 155, 'learning_rate': 0.3, 'subsample': 1.0, 'colsample_bytree': 0.6}\n",
      "0.847 (+/-0.033) for {'min_child_weight': 0.6, 'max_depth': 8, 'n_estimators': 155, 'learning_rate': 0.3, 'subsample': 0.5, 'colsample_bytree': 0.8}\n",
      "0.856 (+/-0.035) for {'min_child_weight': 0.6, 'max_depth': 8, 'n_estimators': 155, 'learning_rate': 0.3, 'subsample': 0.75, 'colsample_bytree': 0.8}\n",
      "0.858 (+/-0.035) for {'min_child_weight': 0.6, 'max_depth': 8, 'n_estimators': 155, 'learning_rate': 0.3, 'subsample': 1.0, 'colsample_bytree': 0.8}\n",
      "0.846 (+/-0.030) for {'min_child_weight': 0.6, 'max_depth': 8, 'n_estimators': 155, 'learning_rate': 0.3, 'subsample': 0.5, 'colsample_bytree': 1.0}\n",
      "0.851 (+/-0.033) for {'min_child_weight': 0.6, 'max_depth': 8, 'n_estimators': 155, 'learning_rate': 0.3, 'subsample': 0.75, 'colsample_bytree': 1.0}\n",
      "0.854 (+/-0.033) for {'min_child_weight': 0.6, 'max_depth': 8, 'n_estimators': 155, 'learning_rate': 0.3, 'subsample': 1.0, 'colsample_bytree': 1.0}\n",
      "\n"
     ]
    }
   ],
   "source": [
    "# Grid search XGB\n",
    "learning_rate = [0.3]\n",
    "subsample = [.5, .75, 1.0]\n",
    "colsample_bytree = [.4, .6, .8, 1.0]\n",
    "parameters = {\n",
    "    'max_depth': [8],\n",
    "    'learning_rate': learning_rate,\n",
    "    'n_estimators': [155],\n",
    "    'min_child_weight': [3/5],\n",
    "    'subsample': subsample,\n",
    "    'colsample_bytree': colsample_bytree,\n",
    "}\n",
    "\n",
    "xg_clf = GridSearchCV(xgb.XGBClassifier(), parameters, cv=10, n_jobs=-1, scoring='roc_auc')\n",
    "xg_clf.fit(X_train, y_train)\n",
    "\n",
    "print(\"Best parameter set found on development set with cv=10:\\n\")\n",
    "print(xg_clf.best_params_)\n",
    "print(\"Best score: {}\\n\".format(xg_clf.best_score_))\n",
    "print()\n",
    "for params, mean_score, scores in xg_clf.grid_scores_:\n",
    "    print(\"{0:.3f} (+/-{1:.03f}) for {2}\".format(mean_score, scores.std() * 2, params))\n",
    "print()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Best parameter set found on development set with cv=10:\n",
      "\n",
      "{'min_child_weight': 0.6, 'max_depth': 8, 'n_estimators': 155, 'learning_rate': 0.3, 'subsample': 1.0, 'colsample_bytree': 0.45}\n",
      "Best score: 0.8667531493479463\n",
      "\n",
      "\n",
      "0.865 (+/-0.032) for {'min_child_weight': 0.5182235108065366, 'max_depth': 8, 'n_estimators': 155, 'learning_rate': 0.3, 'subsample': 1.0, 'colsample_bytree': 0.45}\n",
      "0.862 (+/-0.033) for {'min_child_weight': 0.55, 'max_depth': 8, 'n_estimators': 155, 'learning_rate': 0.3, 'subsample': 1.0, 'colsample_bytree': 0.45}\n",
      "0.867 (+/-0.027) for {'min_child_weight': 0.6, 'max_depth': 8, 'n_estimators': 155, 'learning_rate': 0.3, 'subsample': 1.0, 'colsample_bytree': 0.45}\n",
      "0.865 (+/-0.032) for {'min_child_weight': 0.65, 'max_depth': 8, 'n_estimators': 155, 'learning_rate': 0.3, 'subsample': 1.0, 'colsample_bytree': 0.45}\n",
      "\n"
     ]
    }
   ],
   "source": [
    "# Grid search XGB\n",
    "learning_rate = [0.3]\n",
    "subsample = [1.0]\n",
    "colsample_bytree = [.45]\n",
    "parameters = {\n",
    "    'max_depth': [8],\n",
    "    'learning_rate': learning_rate,\n",
    "    'n_estimators': [155],\n",
    "    'min_child_weight': [3/(100*1897/(1897+30872)), .55, 3/5, .65],\n",
    "    'subsample': subsample,\n",
    "    'colsample_bytree': colsample_bytree,\n",
    "}\n",
    "\n",
    "xg_clf = GridSearchCV(xgb.XGBClassifier(), parameters, cv=10, n_jobs=-1, scoring='roc_auc')\n",
    "xg_clf.fit(X_train, y_train)\n",
    "\n",
    "print(\"Best parameter set found on development set with cv=10:\\n\")\n",
    "print(xg_clf.best_params_)\n",
    "print(\"Best score: {}\\n\".format(xg_clf.best_score_))\n",
    "print()\n",
    "for params, mean_score, scores in xg_clf.grid_scores_:\n",
    "    print(\"{0:.3f} (+/-{1:.03f}) for {2}\".format(mean_score, scores.std() * 2, params))\n",
    "print()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "predictions = xg_clf.predict_proba(test)\n",
    "\n",
    "# Spit out predictions to a file\n",
    "pred_write = enumerate(predictions, start=1)\n",
    "with open('output/xgb_155trees_minchildweight.6_colsampletree.45_learningrate0.3_maxdepth8.csv', 'w') as f:\n",
    "    f.write('Id,Action\\n')\n",
    "    for instance, prediction in pred_write:\n",
    "        f.write('{},{}\\n'.format(instance, prediction[1]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# just tuning Max depth, nestimators, and colsamplebytree, i got 0.88."
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.1"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}