Untitled12.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-01-11T08:21:23.333371Z",
     "start_time": "2021-01-11T08:21:02.821226Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "access token : Dl7B5FgTbCjpqHwM7xR6ds5ZmMT67Z7r\n",
      "request token : HfSbQhTnZMHBAtdU2ufMc0s6LdPrdo4J\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "\n",
       "    <div class=\"bk-root\">\n",
       "        <a href=\"https://bokeh.org\" target=\"_blank\" class=\"bk-logo bk-logo-small bk-logo-notebook\"></a>\n",
       "        <span id=\"1001\">Loading BokehJS ...</span>\n",
       "    </div>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/javascript": [
       "\n",
       "(function(root) {\n",
       "  function now() {\n",
       "    return new Date();\n",
       "  }\n",
       "\n",
       "  var force = true;\n",
       "\n",
       "  if (typeof root._bokeh_onload_callbacks === \"undefined\" || force === true) {\n",
       "    root._bokeh_onload_callbacks = [];\n",
       "    root._bokeh_is_loading = undefined;\n",
       "  }\n",
       "\n",
       "  var JS_MIME_TYPE = 'application/javascript';\n",
       "  var HTML_MIME_TYPE = 'text/html';\n",
       "  var EXEC_MIME_TYPE = 'application/vnd.bokehjs_exec.v0+json';\n",
       "  var CLASS_NAME = 'output_bokeh rendered_html';\n",
       "\n",
       "  /**\n",
       "   * Render data to the DOM node\n",
       "   */\n",
       "  function render(props, node) {\n",
       "    var script = document.createElement(\"script\");\n",
       "    node.appendChild(script);\n",
       "  }\n",
       "\n",
       "  /**\n",
       "   * Handle when an output is cleared or removed\n",
       "   */\n",
       "  function handleClearOutput(event, handle) {\n",
       "    var cell = handle.cell;\n",
       "\n",
       "    var id = cell.output_area._bokeh_element_id;\n",
       "    var server_id = cell.output_area._bokeh_server_id;\n",
       "    // Clean up Bokeh references\n",
       "    if (id != null && id in Bokeh.index) {\n",
       "      Bokeh.index[id].model.document.clear();\n",
       "      delete Bokeh.index[id];\n",
       "    }\n",
       "\n",
       "    if (server_id !== undefined) {\n",
       "      // Clean up Bokeh references\n",
       "      var cmd = \"from bokeh.io.state import curstate; print(curstate().uuid_to_server['\" + server_id + \"'].get_sessions()[0].document.roots[0]._id)\";\n",
       "      cell.notebook.kernel.execute(cmd, {\n",
       "        iopub: {\n",
       "          output: function(msg) {\n",
       "            var id = msg.content.text.trim();\n",
       "            if (id in Bokeh.index) {\n",
       "              Bokeh.index[id].model.document.clear();\n",
       "              delete Bokeh.index[id];\n",
       "            }\n",
       "          }\n",
       "        }\n",
       "      });\n",
       "      // Destroy server and session\n",
       "      var cmd = \"import bokeh.io.notebook as ion; ion.destroy_server('\" + server_id + \"')\";\n",
       "      cell.notebook.kernel.execute(cmd);\n",
       "    }\n",
       "  }\n",
       "\n",
       "  /**\n",
       "   * Handle when a new output is added\n",
       "   */\n",
       "  function handleAddOutput(event, handle) {\n",
       "    var output_area = handle.output_area;\n",
       "    var output = handle.output;\n",
       "\n",
       "    // limit handleAddOutput to display_data with EXEC_MIME_TYPE content only\n",
       "    if ((output.output_type != \"display_data\") || (!output.data.hasOwnProperty(EXEC_MIME_TYPE))) {\n",
       "      return\n",
       "    }\n",
       "\n",
       "    var toinsert = output_area.element.find(\".\" + CLASS_NAME.split(' ')[0]);\n",
       "\n",
       "    if (output.metadata[EXEC_MIME_TYPE][\"id\"] !== undefined) {\n",
       "      toinsert[toinsert.length - 1].firstChild.textContent = output.data[JS_MIME_TYPE];\n",
       "      // store reference to embed id on output_area\n",
       "      output_area._bokeh_element_id = output.metadata[EXEC_MIME_TYPE][\"id\"];\n",
       "    }\n",
       "    if (output.metadata[EXEC_MIME_TYPE][\"server_id\"] !== undefined) {\n",
       "      var bk_div = document.createElement(\"div\");\n",
       "      bk_div.innerHTML = output.data[HTML_MIME_TYPE];\n",
       "      var script_attrs = bk_div.children[0].attributes;\n",
       "      for (var i = 0; i < script_attrs.length; i++) {\n",
       "        toinsert[toinsert.length - 1].firstChild.setAttribute(script_attrs[i].name, script_attrs[i].value);\n",
       "        toinsert[toinsert.length - 1].firstChild.textContent = bk_div.children[0].textContent\n",
       "      }\n",
       "      // store reference to server id on output_area\n",
       "      output_area._bokeh_server_id = output.metadata[EXEC_MIME_TYPE][\"server_id\"];\n",
       "    }\n",
       "  }\n",
       "\n",
       "  function register_renderer(events, OutputArea) {\n",
       "\n",
       "    function append_mime(data, metadata, element) {\n",
       "      // create a DOM node to render to\n",
       "      var toinsert = this.create_output_subarea(\n",
       "        metadata,\n",
       "        CLASS_NAME,\n",
       "        EXEC_MIME_TYPE\n",
       "      );\n",
       "      this.keyboard_manager.register_events(toinsert);\n",
       "      // Render to node\n",
       "      var props = {data: data, metadata: metadata[EXEC_MIME_TYPE]};\n",
       "      render(props, toinsert[toinsert.length - 1]);\n",
       "      element.append(toinsert);\n",
       "      return toinsert\n",
       "    }\n",
       "\n",
       "    /* Handle when an output is cleared or removed */\n",
       "    events.on('clear_output.CodeCell', handleClearOutput);\n",
       "    events.on('delete.Cell', handleClearOutput);\n",
       "\n",
       "    /* Handle when a new output is added */\n",
       "    events.on('output_added.OutputArea', handleAddOutput);\n",
       "\n",
       "    /**\n",
       "     * Register the mime type and append_mime function with output_area\n",
       "     */\n",
       "    OutputArea.prototype.register_mime_type(EXEC_MIME_TYPE, append_mime, {\n",
       "      /* Is output safe? */\n",
       "      safe: true,\n",
       "      /* Index of renderer in `output_area.display_order` */\n",
       "      index: 0\n",
       "    });\n",
       "  }\n",
       "\n",
       "  // register the mime type if in Jupyter Notebook environment and previously unregistered\n",
       "  if (root.Jupyter !== undefined) {\n",
       "    var events = require('base/js/events');\n",
       "    var OutputArea = require('notebook/js/outputarea').OutputArea;\n",
       "\n",
       "    if (OutputArea.prototype.mime_types().indexOf(EXEC_MIME_TYPE) == -1) {\n",
       "      register_renderer(events, OutputArea);\n",
       "    }\n",
       "  }\n",
       "\n",
       "  \n",
       "  if (typeof (root._bokeh_timeout) === \"undefined\" || force === true) {\n",
       "    root._bokeh_timeout = Date.now() + 5000;\n",
       "    root._bokeh_failed_load = false;\n",
       "  }\n",
       "\n",
       "  var NB_LOAD_WARNING = {'data': {'text/html':\n",
       "     \"<div style='background-color: #fdd'>\\n\"+\n",
       "     \"<p>\\n\"+\n",
       "     \"BokehJS does not appear to have successfully loaded. If loading BokehJS from CDN, this \\n\"+\n",
       "     \"may be due to a slow or bad network connection. Possible fixes:\\n\"+\n",
       "     \"</p>\\n\"+\n",
       "     \"<ul>\\n\"+\n",
       "     \"<li>re-rerun `output_notebook()` to attempt to load from CDN again, or</li>\\n\"+\n",
       "     \"<li>use INLINE resources instead, as so:</li>\\n\"+\n",
       "     \"</ul>\\n\"+\n",
       "     \"<code>\\n\"+\n",
       "     \"from bokeh.resources import INLINE\\n\"+\n",
       "     \"output_notebook(resources=INLINE)\\n\"+\n",
       "     \"</code>\\n\"+\n",
       "     \"</div>\"}};\n",
       "\n",
       "  function display_loaded() {\n",
       "    var el = document.getElementById(\"1001\");\n",
       "    if (el != null) {\n",
       "      el.textContent = \"BokehJS is loading...\";\n",
       "    }\n",
       "    if (root.Bokeh !== undefined) {\n",
       "      if (el != null) {\n",
       "        el.textContent = \"BokehJS \" + root.Bokeh.version + \" successfully loaded.\";\n",
       "      }\n",
       "    } else if (Date.now() < root._bokeh_timeout) {\n",
       "      setTimeout(display_loaded, 100)\n",
       "    }\n",
       "  }\n",
       "\n",
       "\n",
       "  function run_callbacks() {\n",
       "    try {\n",
       "      root._bokeh_onload_callbacks.forEach(function(callback) {\n",
       "        if (callback != null)\n",
       "          callback();\n",
       "      });\n",
       "    } finally {\n",
       "      delete root._bokeh_onload_callbacks\n",
       "    }\n",
       "    console.debug(\"Bokeh: all callbacks have finished\");\n",
       "  }\n",
       "\n",
       "  function load_libs(css_urls, js_urls, callback) {\n",
       "    if (css_urls == null) css_urls = [];\n",
       "    if (js_urls == null) js_urls = [];\n",
       "\n",
       "    root._bokeh_onload_callbacks.push(callback);\n",
       "    if (root._bokeh_is_loading > 0) {\n",
       "      console.debug(\"Bokeh: BokehJS is being loaded, scheduling callback at\", now());\n",
       "      return null;\n",
       "    }\n",
       "    if (js_urls == null || js_urls.length === 0) {\n",
       "      run_callbacks();\n",
       "      return null;\n",
       "    }\n",
       "    console.debug(\"Bokeh: BokehJS not loaded, scheduling load and callback at\", now());\n",
       "    root._bokeh_is_loading = css_urls.length + js_urls.length;\n",
       "\n",
       "    function on_load() {\n",
       "      root._bokeh_is_loading--;\n",
       "      if (root._bokeh_is_loading === 0) {\n",
       "        console.debug(\"Bokeh: all BokehJS libraries/stylesheets loaded\");\n",
       "        run_callbacks()\n",
       "      }\n",
       "    }\n",
       "\n",
       "    function on_error() {\n",
       "      console.error(\"failed to load \" + url);\n",
       "    }\n",
       "\n",
       "    for (var i = 0; i < css_urls.length; i++) {\n",
       "      var url = css_urls[i];\n",
       "      const element = document.createElement(\"link\");\n",
       "      element.onload = on_load;\n",
       "      element.onerror = on_error;\n",
       "      element.rel = \"stylesheet\";\n",
       "      element.type = \"text/css\";\n",
       "      element.href = url;\n",
       "      console.debug(\"Bokeh: injecting link tag for BokehJS stylesheet: \", url);\n",
       "      document.body.appendChild(element);\n",
       "    }\n",
       "\n",
       "    const hashes = {\"https://cdn.bokeh.org/bokeh/release/bokeh-2.2.3.min.js\": \"T2yuo9Oe71Cz/I4X9Ac5+gpEa5a8PpJCDlqKYO0CfAuEszu1JrXLl8YugMqYe3sM\", \"https://cdn.bokeh.org/bokeh/release/bokeh-widgets-2.2.3.min.js\": \"98GDGJ0kOMCUMUePhksaQ/GYgB3+NH9h996V88sh3aOiUNX3N+fLXAtry6xctSZ6\", \"https://cdn.bokeh.org/bokeh/release/bokeh-tables-2.2.3.min.js\": \"89bArO+nlbP3sgakeHjCo1JYxYR5wufVgA3IbUvDY+K7w4zyxJqssu7wVnfeKCq8\"};\n",
       "\n",
       "    for (var i = 0; i < js_urls.length; i++) {\n",
       "      var url = js_urls[i];\n",
       "      var element = document.createElement('script');\n",
       "      element.onload = on_load;\n",
       "      element.onerror = on_error;\n",
       "      element.async = false;\n",
       "      element.src = url;\n",
       "      if (url in hashes) {\n",
       "        element.crossOrigin = \"anonymous\";\n",
       "        element.integrity = \"sha384-\" + hashes[url];\n",
       "      }\n",
       "      console.debug(\"Bokeh: injecting script tag for BokehJS library: \", url);\n",
       "      document.head.appendChild(element);\n",
       "    }\n",
       "  };\n",
       "\n",
       "  function inject_raw_css(css) {\n",
       "    const element = document.createElement(\"style\");\n",
       "    element.appendChild(document.createTextNode(css));\n",
       "    document.body.appendChild(element);\n",
       "  }\n",
       "\n",
       "  \n",
       "  var js_urls = [\"https://cdn.bokeh.org/bokeh/release/bokeh-2.2.3.min.js\", \"https://cdn.bokeh.org/bokeh/release/bokeh-widgets-2.2.3.min.js\", \"https://cdn.bokeh.org/bokeh/release/bokeh-tables-2.2.3.min.js\"];\n",
       "  var css_urls = [];\n",
       "  \n",
       "\n",
       "  var inline_js = [\n",
       "    function(Bokeh) {\n",
       "      Bokeh.set_log_level(\"info\");\n",
       "    },\n",
       "    function(Bokeh) {\n",
       "    \n",
       "    \n",
       "    }\n",
       "  ];\n",
       "\n",
       "  function run_inline_js() {\n",
       "    \n",
       "    if (root.Bokeh !== undefined || force === true) {\n",
       "      \n",
       "    for (var i = 0; i < inline_js.length; i++) {\n",
       "      inline_js[i].call(root, root.Bokeh);\n",
       "    }\n",
       "    if (force === true) {\n",
       "        display_loaded();\n",
       "      }} else if (Date.now() < root._bokeh_timeout) {\n",
       "      setTimeout(run_inline_js, 100);\n",
       "    } else if (!root._bokeh_failed_load) {\n",
       "      console.log(\"Bokeh: BokehJS failed to load within specified timeout.\");\n",
       "      root._bokeh_failed_load = true;\n",
       "    } else if (force !== true) {\n",
       "      var cell = $(document.getElementById(\"1001\")).parents('.cell').data().cell;\n",
       "      cell.output_area.append_execute_result(NB_LOAD_WARNING)\n",
       "    }\n",
       "\n",
       "  }\n",
       "\n",
       "  if (root._bokeh_is_loading === 0) {\n",
       "    console.debug(\"Bokeh: BokehJS loaded, going straight to plotting\");\n",
       "    run_inline_js();\n",
       "  } else {\n",
       "    load_libs(css_urls, js_urls, function() {\n",
       "      console.debug(\"Bokeh: BokehJS plotting callback run at\", now());\n",
       "      run_inline_js();\n",
       "    });\n",
       "  }\n",
       "}(window));"
      ],
      "application/vnd.bokehjs_load.v0+json": "\n(function(root) {\n  function now() {\n    return new Date();\n  }\n\n  var force = true;\n\n  if (typeof root._bokeh_onload_callbacks === \"undefined\" || force === true) {\n    root._bokeh_onload_callbacks = [];\n    root._bokeh_is_loading = undefined;\n  }\n\n  \n\n  \n  if (typeof (root._bokeh_timeout) === \"undefined\" || force === true) {\n    root._bokeh_timeout = Date.now() + 5000;\n    root._bokeh_failed_load = false;\n  }\n\n  var NB_LOAD_WARNING = {'data': {'text/html':\n     \"<div style='background-color: #fdd'>\\n\"+\n     \"<p>\\n\"+\n     \"BokehJS does not appear to have successfully loaded. If loading BokehJS from CDN, this \\n\"+\n     \"may be due to a slow or bad network connection. Possible fixes:\\n\"+\n     \"</p>\\n\"+\n     \"<ul>\\n\"+\n     \"<li>re-rerun `output_notebook()` to attempt to load from CDN again, or</li>\\n\"+\n     \"<li>use INLINE resources instead, as so:</li>\\n\"+\n     \"</ul>\\n\"+\n     \"<code>\\n\"+\n     \"from bokeh.resources import INLINE\\n\"+\n     \"output_notebook(resources=INLINE)\\n\"+\n     \"</code>\\n\"+\n     \"</div>\"}};\n\n  function display_loaded() {\n    var el = document.getElementById(\"1001\");\n    if (el != null) {\n      el.textContent = \"BokehJS is loading...\";\n    }\n    if (root.Bokeh !== undefined) {\n      if (el != null) {\n        el.textContent = \"BokehJS \" + root.Bokeh.version + \" successfully loaded.\";\n      }\n    } else if (Date.now() < root._bokeh_timeout) {\n      setTimeout(display_loaded, 100)\n    }\n  }\n\n\n  function run_callbacks() {\n    try {\n      root._bokeh_onload_callbacks.forEach(function(callback) {\n        if (callback != null)\n          callback();\n      });\n    } finally {\n      delete root._bokeh_onload_callbacks\n    }\n    console.debug(\"Bokeh: all callbacks have finished\");\n  }\n\n  function load_libs(css_urls, js_urls, callback) {\n    if (css_urls == null) css_urls = [];\n    if (js_urls == null) js_urls = [];\n\n    root._bokeh_onload_callbacks.push(callback);\n    if (root._bokeh_is_loading > 0) {\n      console.debug(\"Bokeh: BokehJS is being loaded, scheduling callback at\", now());\n      return null;\n    }\n    if (js_urls == null || js_urls.length === 0) {\n      run_callbacks();\n      return null;\n    }\n    console.debug(\"Bokeh: BokehJS not loaded, scheduling load and callback at\", now());\n    root._bokeh_is_loading = css_urls.length + js_urls.length;\n\n    function on_load() {\n      root._bokeh_is_loading--;\n      if (root._bokeh_is_loading === 0) {\n        console.debug(\"Bokeh: all BokehJS libraries/stylesheets loaded\");\n        run_callbacks()\n      }\n    }\n\n    function on_error() {\n      console.error(\"failed to load \" + url);\n    }\n\n    for (var i = 0; i < css_urls.length; i++) {\n      var url = css_urls[i];\n      const element = document.createElement(\"link\");\n      element.onload = on_load;\n      element.onerror = on_error;\n      element.rel = \"stylesheet\";\n      element.type = \"text/css\";\n      element.href = url;\n      console.debug(\"Bokeh: injecting link tag for BokehJS stylesheet: \", url);\n      document.body.appendChild(element);\n    }\n\n    const hashes = {\"https://cdn.bokeh.org/bokeh/release/bokeh-2.2.3.min.js\": \"T2yuo9Oe71Cz/I4X9Ac5+gpEa5a8PpJCDlqKYO0CfAuEszu1JrXLl8YugMqYe3sM\", \"https://cdn.bokeh.org/bokeh/release/bokeh-widgets-2.2.3.min.js\": \"98GDGJ0kOMCUMUePhksaQ/GYgB3+NH9h996V88sh3aOiUNX3N+fLXAtry6xctSZ6\", \"https://cdn.bokeh.org/bokeh/release/bokeh-tables-2.2.3.min.js\": \"89bArO+nlbP3sgakeHjCo1JYxYR5wufVgA3IbUvDY+K7w4zyxJqssu7wVnfeKCq8\"};\n\n    for (var i = 0; i < js_urls.length; i++) {\n      var url = js_urls[i];\n      var element = document.createElement('script');\n      element.onload = on_load;\n      element.onerror = on_error;\n      element.async = false;\n      element.src = url;\n      if (url in hashes) {\n        element.crossOrigin = \"anonymous\";\n        element.integrity = \"sha384-\" + hashes[url];\n      }\n      console.debug(\"Bokeh: injecting script tag for BokehJS library: \", url);\n      document.head.appendChild(element);\n    }\n  };\n\n  function inject_raw_css(css) {\n    const element = document.createElement(\"style\");\n    element.appendChild(document.createTextNode(css));\n    document.body.appendChild(element);\n  }\n\n  \n  var js_urls = [\"https://cdn.bokeh.org/bokeh/release/bokeh-2.2.3.min.js\", \"https://cdn.bokeh.org/bokeh/release/bokeh-widgets-2.2.3.min.js\", \"https://cdn.bokeh.org/bokeh/release/bokeh-tables-2.2.3.min.js\"];\n  var css_urls = [];\n  \n\n  var inline_js = [\n    function(Bokeh) {\n      Bokeh.set_log_level(\"info\");\n    },\n    function(Bokeh) {\n    \n    \n    }\n  ];\n\n  function run_inline_js() {\n    \n    if (root.Bokeh !== undefined || force === true) {\n      \n    for (var i = 0; i < inline_js.length; i++) {\n      inline_js[i].call(root, root.Bokeh);\n    }\n    if (force === true) {\n        display_loaded();\n      }} else if (Date.now() < root._bokeh_timeout) {\n      setTimeout(run_inline_js, 100);\n    } else if (!root._bokeh_failed_load) {\n      console.log(\"Bokeh: BokehJS failed to load within specified timeout.\");\n      root._bokeh_failed_load = true;\n    } else if (force !== true) {\n      var cell = $(document.getElementById(\"1001\")).parents('.cell').data().cell;\n      cell.output_area.append_execute_result(NB_LOAD_WARNING)\n    }\n\n  }\n\n  if (root._bokeh_is_loading === 0) {\n    console.debug(\"Bokeh: BokehJS loaded, going straight to plotting\");\n    run_inline_js();\n  } else {\n    load_libs(css_urls, js_urls, function() {\n      console.debug(\"Bokeh: BokehJS plotting callback run at\", now());\n      run_inline_js();\n    });\n  }\n}(window));"
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "####################### AUTO LOGIN #####################\n",
    "import login\n",
    "from login import *\n",
    "import pandas as pd\n",
    "import joblib\n",
    "import ta\n",
    "# import autokeras as ak\n",
    "# import tensorflow as tf\n",
    "# tf.get_logger().setLevel('ERROR')\n",
    "import os\n",
    "os.environ['TF_CPP_MIN_LOG_LEVEL'] = '3'\n",
    "pd.set_option('plotting.backend', 'pandas_bokeh')\n",
    "import pandas_bokeh\n",
    "pandas_bokeh.output_notebook()\n",
    "from sklearn.model_selection import train_test_split"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-01-04T08:17:31.770013Z",
     "start_time": "2021-01-04T08:17:06.371405Z"
    },
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "# mydf = joblib.load('./TATASTEEL_DF.p')\n",
    "mydf['next_close'] = mydf['close'].shift(periods=-1)\n",
    "mydf = ta.utils.dropna(mydf)\n",
    "mydf = ta.add_all_ta_features(mydf, open=\"open\", high=\"high\", low=\"low\", close=\"close\", volume=\"volume\")\n",
    "mydf = mydf.fillna(0)[500:]\n",
    "mydf"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-01-05T20:07:33.760591Z",
     "start_time": "2021-01-05T20:07:22.461148Z"
    }
   },
   "outputs": [],
   "source": [
    "train,test = train_test_split(mydf, test_size=0.02, shuffle=True)\n",
    "\n",
    "train.to_csv('train.csv', index=False)\n",
    "test.to_csv('eval.csv', index=False)\n",
    "\n",
    "# train_file_path = 'train.csv'\n",
    "# test_file_path = 'eval.csv'\n",
    "\n",
    "# xtest = test.loc[:, test.columns != 'next_close']\n",
    "# ytest = test['next_close']\n",
    "# xtrain = train.loc[:, train.columns != 'next_close']\n",
    "# ytrain = train['next_close']\n",
    "# print('xtrain : '+str(xtrain.shape))\n",
    "# print('ytrain : '+str(ytrain.shape))\n",
    "# print('xtest : '+str(xtest.shape))\n",
    "# print('ytest : '+str(ytest.shape))\n",
    "\n",
    "# xtrain.to_csv('xtrain.csv', index=False)\n",
    "# xtest.to_csv('xtest.csv', index=False)\n",
    "# ytrain.to_csv('ytrain.csv', index=False)\n",
    "# ytest.to_csv('ytest.csv', index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-01-04T02:29:46.668559Z",
     "start_time": "2021-01-04T02:29:46.665437Z"
    }
   },
   "outputs": [],
   "source": [
    "# reg = ak.StructuredDataRegressor(overwrite=True,max_trials=100)\n",
    "# # reg.fit(train_file_path,'next_close',epochs=100)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-01-04T04:16:12.346852Z",
     "start_time": "2021-01-04T04:16:10.557734Z"
    }
   },
   "outputs": [],
   "source": [
    "# model = reg.export_model()\n",
    "\n",
    "# print(type(model))\n",
    "\n",
    "# try:\n",
    "#     model.save(\"model_autokeras4\", save_format=\"tf\")\n",
    "# except:\n",
    "#     model.save(\"model_autokeras4.h5\")\n",
    "\n",
    "from tensorflow.keras.models import load_model\n",
    "model = load_model(\"model_autokeras4\", custom_objects=ak.CUSTOM_OBJECTS)\n",
    "# predicted_y = loaded_model.predict(tf.expand_dims(x_test, -1))\n",
    "# print(predicted_y)\n",
    "model.summary()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-01-04T04:24:52.985591Z",
     "start_time": "2021-01-04T04:24:52.983169Z"
    }
   },
   "outputs": [],
   "source": [
    "# predicted_y = reg.predict(test_file_path)\n",
    "# print(reg.evaluate(test_file_path, 'next_close'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-01-04T04:43:12.382882Z",
     "start_time": "2021-01-04T04:42:18.741020Z"
    }
   },
   "outputs": [],
   "source": [
    "history = model.fit(xtrain.to_numpy().astype('str'),\n",
    "                    ytrain,\n",
    "                    batch_size=500,\n",
    "                    epochs=5,\n",
    "                    validation_data=(xtest.to_numpy().astype('str'), ytest))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-01-04T05:03:05.006998Z",
     "start_time": "2021-01-04T05:03:04.005568Z"
    }
   },
   "outputs": [],
   "source": [
    "# pred = model.predict(xtest.to_numpy().astype('str')[-1].reshape((1,89)))\n",
    "pred = model.predict(xtest.to_numpy().astype('str'))\n",
    "preddf = pd.DataFrame({'pred':pred.reshape((pred.shape[0],)),'orig':ytest})\n",
    "preddf['diff'] = preddf['orig'] - preddf['pred']\n",
    "preddf"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-01-04T05:03:08.313731Z",
     "start_time": "2021-01-04T05:03:08.301108Z"
    }
   },
   "outputs": [],
   "source": [
    "pltdf = pd.DataFrame({'pred':preddf['pred'],'orig':preddf['orig']})\n",
    "# pltdf.plot()\n",
    "# pltdf\n",
    "preddf['diff'].describe()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-01-04T05:03:09.955062Z",
     "start_time": "2021-01-04T05:03:09.807148Z"
    }
   },
   "outputs": [],
   "source": [
    "predicted_y = pred\n",
    "pred = pd.DataFrame(predicted_y)\n",
    "orig = pd.DataFrame(test['next_close'])\n",
    "pred.reset_index(drop=True, inplace=True)\n",
    "orig.reset_index(drop=True, inplace=True)\n",
    "pred['next'] = pred.shift(periods=-1)\n",
    "pred['diff'] = pred[0] - pred['next']\n",
    "orig['next'] = orig['next_close'].shift(periods=-1)\n",
    "orig['diff'] = orig['next_close'] - orig['next']\n",
    "\n",
    "newdf = pd.concat([pred['diff'],orig['diff']], axis=1, ignore_index=True)\n",
    "newdf[0:500].plot()\n",
    "\n",
    "plotdf = abs(newdf[0] -  newdf[1])\n",
    "plotdf.plot()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-01-04T05:03:11.603914Z",
     "start_time": "2021-01-04T05:03:11.588918Z"
    }
   },
   "outputs": [],
   "source": [
    "pred"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-01-04T05:03:13.098006Z",
     "start_time": "2021-01-04T05:03:13.087489Z"
    }
   },
   "outputs": [],
   "source": [
    "orig"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-01-04T05:06:17.931827Z",
     "start_time": "2021-01-04T05:06:17.702265Z"
    }
   },
   "outputs": [],
   "source": [
    "decide = []\n",
    "\n",
    "truelist = []\n",
    "falselist = []\n",
    "\n",
    "for i in range(len(orig['diff'])):\n",
    "    if((pred['diff'][i]<0 and orig['diff'][i]<0) or (pred['diff'][i]>0 and orig['diff'][i]>0)):\n",
    "#         decide.append(abs(pred['diff'][i]-orig['diff'][i]))\n",
    "        decide.append(abs(pred['diff'][i]))\n",
    "        truelist.append(abs(pred['diff'][i]))\n",
    "    else:\n",
    "#         decide.append(-abs(pred['diff'][i]-orig['diff'][i]))\n",
    "        decide.append(-abs(pred['diff'][i]))\n",
    "        falselist.append(abs(pred['diff'][i]))\n",
    "    \n",
    "dcdf = pd.DataFrame(decide)\n",
    "dcdf.plot()\n",
    "\n",
    "print(dcdf.sum())\n",
    "print(len(truelist) - len(falselist))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-01-04T05:08:00.359488Z",
     "start_time": "2021-01-04T05:08:00.354025Z"
    }
   },
   "outputs": [],
   "source": [
    "(len(truelist) - len(falselist))/len(truelist) "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-01-04T07:59:04.540166Z",
     "start_time": "2021-01-04T07:59:04.493708Z"
    }
   },
   "outputs": [],
   "source": [
    "model.save('./hj.h5')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-01-04T08:04:38.979639Z",
     "start_time": "2021-01-04T08:04:38.971986Z"
    }
   },
   "outputs": [],
   "source": [
    "import pickle\n",
    "def load_pickle_model_from_file(name_file):\n",
    "    with open(name_file, 'rb') as file:\n",
    "        loaded_model = pickle.load(file)\n",
    "    return loaded_model\n",
    "\n",
    "def save_pickle_model_to_file(model, name_model):\n",
    "    with open('./'+name_model+'.pickle', 'wb') as file:.\n",
    "        pickle.dump(model, file)\n",
    "    print(str('./'+name_model+'.pickle')+' - model saved!')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-01-04T08:05:49.271030Z",
     "start_time": "2021-01-04T08:05:49.267709Z"
    }
   },
   "outputs": [],
   "source": [
    "save_pickle_model_to_file(model, 'name_model')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-12-05T20:56:36.164373Z",
     "start_time": "2020-12-05T20:49:30.771576Z"
    }
   },
   "outputs": [],
   "source": [
    "insttkn = 895745\n",
    "interval='minute'\n",
    "\n",
    "old_lst=[]\n",
    "todaydt=datetime.date.today()\n",
    "hud_ago=todaydt-datetime.timedelta(days=59)\n",
    "to_date=datetime.date.isoformat(todaydt)\n",
    "from_date=datetime.date.isoformat(hud_ago)\n",
    "\n",
    "for i2 in range(50):\n",
    "    new_lst = kite.historical_data(insttkn, from_date, to_date, interval,continuous=False)\n",
    "    old_lst = new_lst + old_lst\n",
    "    todaydt=todaydt-datetime.timedelta(days=60)\n",
    "    hud_ago=hud_ago-datetime.timedelta(days=60)\n",
    "    to_date=datetime.date.isoformat(todaydt)\n",
    "    from_date=datetime.date.isoformat(hud_ago)\n",
    "\n",
    "print(len(old_lst))\n",
    "mydf = pd.DataFrame(old_lst)\n",
    "mydf['next_close'] = mydf['close'].shift(periods=-1)\n",
    "mydf = ta.utils.dropna(mydf)\n",
    "mydf = ta.add_all_ta_features(mydf, open=\"open\", high=\"high\", low=\"low\", close=\"close\", volume=\"volume\")\n",
    "mydf = mydf.fillna(0)[500:]\n",
    "mydf.to_csv('mydf.csv', index=False)\n",
    "\n",
    "train,test = train_test_split(mydf, test_size=0.02, shuffle=False)\n",
    "\n",
    "train.to_csv('train.csv', index=False)\n",
    "test.to_csv('test.csv', index=False)\n",
    "\n",
    "mydf"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# interval='minute'\n",
    "\n",
    "# nfifty_tokens_list = [3861249,60417,1510401,4267265,4268801,81153,2714625,134657,140033,\n",
    "#                       177665,5215745,2800641,225537,232961,1207553,315393,1850625,340481,\n",
    "#                       341249,119553,345089,348929,356865,1270529,1346049,408065,415745,424961,\n",
    "#                       3001089,492033,2939649,519937,2815745,4598529,2977281,633601,3834113,\n",
    "#                       738561,5582849,779521,794369,857857,884737,895745,2953217,3465729,\n",
    "#                       897537,2952193,2889473,969473]\n",
    "\n",
    "# mergeddf = pd.DataFrame()\n",
    "\n",
    "# for i in range(len(nfifty_tokens_list)):\n",
    "#     insttkn = nfifty_tokens_list[i]\n",
    "\n",
    "#     old_lst=[]\n",
    "#     todaydt=datetime.date.today()\n",
    "#     hud_ago=todaydt-datetime.timedelta(days=59)\n",
    "#     to_date=datetime.date.isoformat(todaydt)\n",
    "#     from_date=datetime.date.isoformat(hud_ago)\n",
    "\n",
    "#     for i2 in range(35):\n",
    "#         new_lst = kite.historical_data(insttkn, from_date, to_date, interval,continuous=False)\n",
    "#         old_lst = new_lst + old_lst\n",
    "#         todaydt=todaydt-datetime.timedelta(days=60)\n",
    "#         hud_ago=hud_ago-datetime.timedelta(days=60)\n",
    "#         to_date=datetime.date.isoformat(todaydt)\n",
    "#         from_date=datetime.date.isoformat(hud_ago)\n",
    "    \n",
    "#     print(len(old_lst))\n",
    "    \n",
    "#     globals()['mydf'+str(nfifty_tokens_list[i])] = pd.DataFrame(old_lst)\n",
    "    \n",
    "#     globals()['mydf'+str(nfifty_tokens_list[i])]['next_close'] = globals()['mydf'+str(nfifty_tokens_list[i])]['close'].shift(periods=-1)\n",
    "#     globals()['mydf'+str(nfifty_tokens_list[i])] = globals()['mydf'+str(nfifty_tokens_list[i])][:-1].fillna(0)\n",
    "#     globals()['mydf'+str(nfifty_tokens_list[i])] = ta.add_all_ta_features(globals()['mydf'+str(nfifty_tokens_list[i])], open=\"open\", high=\"high\", low=\"low\", close=\"close\", volume=\"volume\")\n",
    "#     globals()['mydf'+str(nfifty_tokens_list[i])] = globals()['mydf'+str(nfifty_tokens_list[i])].fillna(0)[500:]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "nfifty_tokens_list = [3861249,60417,1510401,4267265,4268801,81153,2714625,134657,140033,\n",
    "                      177665,5215745,2800641,225537,232961,1207553,315393,1850625,340481,\n",
    "                      341249,119553,345089,348929,356865,1270529,1346049,408065,415745,424961,\n",
    "                      3001089,492033,2939649,519937,2815745,4598529,2977281,633601,3834113,\n",
    "                      738561,5582849,779521,794369,857857,884737,895745,2953217,3465729,\n",
    "                      897537,2952193,2889473,969473]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "interval='5minute'\n",
    "\n",
    "mergeddf = pd.DataFrame()\n",
    "\n",
    "for i in range(len(nfifty_tokens_list)):\n",
    "    insttkn = nfifty_tokens_list[i]\n",
    "\n",
    "    old_lst=[]\n",
    "    todaydt=datetime.date.today()\n",
    "    hud_ago=todaydt-datetime.timedelta(days=59)\n",
    "    to_date=datetime.date.isoformat(todaydt)\n",
    "    from_date=datetime.date.isoformat(hud_ago)\n",
    "\n",
    "    for i2 in range(35):\n",
    "        new_lst = kite.historical_data(insttkn, from_date, to_date, interval,continuous=False)\n",
    "        old_lst = new_lst + old_lst\n",
    "        todaydt=todaydt-datetime.timedelta(days=60)\n",
    "        hud_ago=hud_ago-datetime.timedelta(days=60)\n",
    "        to_date=datetime.date.isoformat(todaydt)\n",
    "        from_date=datetime.date.isoformat(hud_ago)\n",
    "    \n",
    "    print(len(old_lst))\n",
    "    \n",
    "    globals()['mydf'+str(nfifty_tokens_list[i])] = pd.DataFrame(old_lst)\n",
    "    mydf = pd.DataFrame(old_lst)\n",
    "    mydf['next_close'] = mydf['close'].shift(periods=-1)\n",
    "    mydf = mydf[:-1].fillna(0)\n",
    "    mydf = ta.add_all_ta_features(mydf, open=\"open\", high=\"high\", low=\"low\", close=\"close\", volume=\"volume\")\n",
    "    mydf = mydf.fillna(0)[500:]\n",
    "    mydf.to_csv('./dfs2/mydf'+str(nfifty_tokens_list[i])+'.csv', index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#### COMBINE ALL CSV FILES ############\n",
    "import os\n",
    "import glob\n",
    "import pandas as pd\n",
    "os.chdir(\"/home/hemangjoshi37a_gmail_com/pfiles/dfs3\")\n",
    "\n",
    "extension = 'csv'\n",
    "all_filenames = [i for i in glob.glob('*.{}'.format(extension))]\n",
    "\n",
    "combined_csv = pd.concat([pd.read_csv(f) for f in all_filenames ])\n",
    "combined_csv.to_csv( \"combined_csv.csv\", index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "for ii in nfifty_tokens_list:\n",
    "    df1 = pd.read_csv('/home/hemangjoshi37a_gmail_com/pfiles/dfs2/mydf'+str(ii)+'.csv')\n",
    "    df1['bullbear']=0\n",
    "    for ind in df1.index:\n",
    "         if(df1['next_close'][ind]>df1['close'][ind]):\n",
    "                df1.iloc[ind, df1.columns.get_loc('bullbear')]=1\n",
    "         if(df1['next_close'][ind]<df1['close'][ind]):\n",
    "                df1.iloc[ind, df1.columns.get_loc('bullbear')]=-1\n",
    "    del df1['next_close']\n",
    "    df1.to_csv('./dfs3/mydf'+str(ii)+'.csv', index=False)\n",
    "    df1=pd.DataFrame()\n",
    "    print(ii)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-01-11T08:21:45.077929Z",
     "start_time": "2021-01-11T08:21:43.799963Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "131\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/hemang/.local/lib/python3.9/site-packages/ta/trend.py:768: RuntimeWarning: invalid value encountered in double_scalars\n",
      "  dip[i] = 100 * (self._dip[i] / self._trs[i])\n",
      "/home/hemang/.local/lib/python3.9/site-packages/ta/trend.py:772: RuntimeWarning: invalid value encountered in double_scalars\n",
      "  din[i] = 100 * (self._din[i] / self._trs[i])\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>date</th>\n",
       "      <th>open</th>\n",
       "      <th>high</th>\n",
       "      <th>low</th>\n",
       "      <th>close</th>\n",
       "      <th>volume</th>\n",
       "      <th>next_close</th>\n",
       "      <th>volume_adi</th>\n",
       "      <th>volume_obv</th>\n",
       "      <th>volume_cmf</th>\n",
       "      <th>...</th>\n",
       "      <th>momentum_wr</th>\n",
       "      <th>momentum_ao</th>\n",
       "      <th>momentum_kama</th>\n",
       "      <th>momentum_roc</th>\n",
       "      <th>momentum_ppo</th>\n",
       "      <th>momentum_ppo_signal</th>\n",
       "      <th>momentum_ppo_hist</th>\n",
       "      <th>others_dr</th>\n",
       "      <th>others_dlr</th>\n",
       "      <th>others_cr</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>50</th>\n",
       "      <td>2021-01-08 13:25:00+05:30</td>\n",
       "      <td>719.05</td>\n",
       "      <td>719.30</td>\n",
       "      <td>718.40</td>\n",
       "      <td>718.60</td>\n",
       "      <td>65754</td>\n",
       "      <td>718.20</td>\n",
       "      <td>-1.907681e+06</td>\n",
       "      <td>-1113458</td>\n",
       "      <td>-0.151159</td>\n",
       "      <td>...</td>\n",
       "      <td>-32.467532</td>\n",
       "      <td>-0.979265</td>\n",
       "      <td>718.980482</td>\n",
       "      <td>0.048730</td>\n",
       "      <td>-24.696958</td>\n",
       "      <td>-29.136560</td>\n",
       "      <td>4.439601</td>\n",
       "      <td>-0.055633</td>\n",
       "      <td>-0.055648</td>\n",
       "      <td>-0.553557</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>51</th>\n",
       "      <td>2021-01-08 13:30:00+05:30</td>\n",
       "      <td>718.60</td>\n",
       "      <td>719.10</td>\n",
       "      <td>717.70</td>\n",
       "      <td>718.20</td>\n",
       "      <td>71103</td>\n",
       "      <td>714.80</td>\n",
       "      <td>-1.927996e+06</td>\n",
       "      <td>-1184561</td>\n",
       "      <td>-0.161644</td>\n",
       "      <td>...</td>\n",
       "      <td>-42.857143</td>\n",
       "      <td>-0.808971</td>\n",
       "      <td>718.967298</td>\n",
       "      <td>0.139431</td>\n",
       "      <td>-25.535857</td>\n",
       "      <td>-28.416419</td>\n",
       "      <td>2.880563</td>\n",
       "      <td>-0.055664</td>\n",
       "      <td>-0.055679</td>\n",
       "      <td>-0.608912</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>52</th>\n",
       "      <td>2021-01-08 13:35:00+05:30</td>\n",
       "      <td>718.20</td>\n",
       "      <td>718.40</td>\n",
       "      <td>714.35</td>\n",
       "      <td>714.80</td>\n",
       "      <td>348034</td>\n",
       "      <td>714.20</td>\n",
       "      <td>-2.198689e+06</td>\n",
       "      <td>-1532595</td>\n",
       "      <td>-0.272495</td>\n",
       "      <td>...</td>\n",
       "      <td>-91.818182</td>\n",
       "      <td>-1.012206</td>\n",
       "      <td>718.703349</td>\n",
       "      <td>-0.223339</td>\n",
       "      <td>-7.238714</td>\n",
       "      <td>-24.180878</td>\n",
       "      <td>16.942164</td>\n",
       "      <td>-0.473406</td>\n",
       "      <td>-0.474530</td>\n",
       "      <td>-1.079435</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>53</th>\n",
       "      <td>2021-01-08 13:40:00+05:30</td>\n",
       "      <td>714.75</td>\n",
       "      <td>714.80</td>\n",
       "      <td>712.30</td>\n",
       "      <td>714.20</td>\n",
       "      <td>388872</td>\n",
       "      <td>715.40</td>\n",
       "      <td>-1.996476e+06</td>\n",
       "      <td>-1921467</td>\n",
       "      <td>-0.126197</td>\n",
       "      <td>...</td>\n",
       "      <td>-74.834437</td>\n",
       "      <td>-1.682353</td>\n",
       "      <td>718.172408</td>\n",
       "      <td>-0.487669</td>\n",
       "      <td>6.400643</td>\n",
       "      <td>-18.064574</td>\n",
       "      <td>24.465217</td>\n",
       "      <td>-0.083940</td>\n",
       "      <td>-0.083975</td>\n",
       "      <td>-1.162469</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>54</th>\n",
       "      <td>2021-01-08 13:45:00+05:30</td>\n",
       "      <td>713.90</td>\n",
       "      <td>715.45</td>\n",
       "      <td>713.65</td>\n",
       "      <td>715.40</td>\n",
       "      <td>191115</td>\n",
       "      <td>712.15</td>\n",
       "      <td>-1.815978e+06</td>\n",
       "      <td>-1730352</td>\n",
       "      <td>-0.076549</td>\n",
       "      <td>...</td>\n",
       "      <td>-58.940397</td>\n",
       "      <td>-2.425588</td>\n",
       "      <td>717.958608</td>\n",
       "      <td>-0.237066</td>\n",
       "      <td>6.669585</td>\n",
       "      <td>-13.117742</td>\n",
       "      <td>19.787327</td>\n",
       "      <td>0.168020</td>\n",
       "      <td>0.167879</td>\n",
       "      <td>-0.996402</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>126</th>\n",
       "      <td>2021-01-11 13:30:00+05:30</td>\n",
       "      <td>692.50</td>\n",
       "      <td>694.50</td>\n",
       "      <td>692.25</td>\n",
       "      <td>693.60</td>\n",
       "      <td>265630</td>\n",
       "      <td>694.40</td>\n",
       "      <td>-3.525842e+06</td>\n",
       "      <td>-4387855</td>\n",
       "      <td>0.125970</td>\n",
       "      <td>...</td>\n",
       "      <td>-14.285714</td>\n",
       "      <td>-0.561029</td>\n",
       "      <td>692.446514</td>\n",
       "      <td>0.543596</td>\n",
       "      <td>-16.255492</td>\n",
       "      <td>-21.557824</td>\n",
       "      <td>5.302331</td>\n",
       "      <td>0.245700</td>\n",
       "      <td>0.245399</td>\n",
       "      <td>-4.013285</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>127</th>\n",
       "      <td>2021-01-11 13:35:00+05:30</td>\n",
       "      <td>693.65</td>\n",
       "      <td>694.95</td>\n",
       "      <td>693.30</td>\n",
       "      <td>694.40</td>\n",
       "      <td>272236</td>\n",
       "      <td>692.50</td>\n",
       "      <td>-3.435097e+06</td>\n",
       "      <td>-4115619</td>\n",
       "      <td>0.138003</td>\n",
       "      <td>...</td>\n",
       "      <td>-8.148148</td>\n",
       "      <td>-0.073382</td>\n",
       "      <td>693.166737</td>\n",
       "      <td>0.397600</td>\n",
       "      <td>-6.569904</td>\n",
       "      <td>-18.560240</td>\n",
       "      <td>11.990336</td>\n",
       "      <td>0.115340</td>\n",
       "      <td>0.115274</td>\n",
       "      <td>-3.902574</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>128</th>\n",
       "      <td>2021-01-11 13:40:00+05:30</td>\n",
       "      <td>694.40</td>\n",
       "      <td>694.55</td>\n",
       "      <td>692.25</td>\n",
       "      <td>692.50</td>\n",
       "      <td>149529</td>\n",
       "      <td>693.40</td>\n",
       "      <td>-3.552119e+06</td>\n",
       "      <td>-4265148</td>\n",
       "      <td>0.057863</td>\n",
       "      <td>...</td>\n",
       "      <td>-46.226415</td>\n",
       "      <td>0.227794</td>\n",
       "      <td>693.151627</td>\n",
       "      <td>0.086718</td>\n",
       "      <td>-5.934213</td>\n",
       "      <td>-16.035034</td>\n",
       "      <td>10.100821</td>\n",
       "      <td>-0.273618</td>\n",
       "      <td>-0.273993</td>\n",
       "      <td>-4.165513</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>129</th>\n",
       "      <td>2021-01-11 13:45:00+05:30</td>\n",
       "      <td>692.35</td>\n",
       "      <td>693.90</td>\n",
       "      <td>692.20</td>\n",
       "      <td>693.40</td>\n",
       "      <td>83542</td>\n",
       "      <td>693.65</td>\n",
       "      <td>-3.517720e+06</td>\n",
       "      <td>-4181606</td>\n",
       "      <td>0.008369</td>\n",
       "      <td>...</td>\n",
       "      <td>-36.470588</td>\n",
       "      <td>0.523235</td>\n",
       "      <td>693.162254</td>\n",
       "      <td>0.383641</td>\n",
       "      <td>-9.036984</td>\n",
       "      <td>-14.635424</td>\n",
       "      <td>5.598440</td>\n",
       "      <td>0.129964</td>\n",
       "      <td>0.129880</td>\n",
       "      <td>-4.040963</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>130</th>\n",
       "      <td>2021-01-11 13:50:00+05:30</td>\n",
       "      <td>693.50</td>\n",
       "      <td>693.80</td>\n",
       "      <td>693.30</td>\n",
       "      <td>693.65</td>\n",
       "      <td>8724</td>\n",
       "      <td>0.00</td>\n",
       "      <td>-3.514230e+06</td>\n",
       "      <td>-4172882</td>\n",
       "      <td>-0.009463</td>\n",
       "      <td>...</td>\n",
       "      <td>-30.588235</td>\n",
       "      <td>0.905882</td>\n",
       "      <td>693.193816</td>\n",
       "      <td>0.267418</td>\n",
       "      <td>-16.299265</td>\n",
       "      <td>-14.968192</td>\n",
       "      <td>-1.331072</td>\n",
       "      <td>0.036054</td>\n",
       "      <td>0.036048</td>\n",
       "      <td>-4.006366</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>81 rows × 90 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                         date    open    high     low   close  volume  \\\n",
       "50  2021-01-08 13:25:00+05:30  719.05  719.30  718.40  718.60   65754   \n",
       "51  2021-01-08 13:30:00+05:30  718.60  719.10  717.70  718.20   71103   \n",
       "52  2021-01-08 13:35:00+05:30  718.20  718.40  714.35  714.80  348034   \n",
       "53  2021-01-08 13:40:00+05:30  714.75  714.80  712.30  714.20  388872   \n",
       "54  2021-01-08 13:45:00+05:30  713.90  715.45  713.65  715.40  191115   \n",
       "..                        ...     ...     ...     ...     ...     ...   \n",
       "126 2021-01-11 13:30:00+05:30  692.50  694.50  692.25  693.60  265630   \n",
       "127 2021-01-11 13:35:00+05:30  693.65  694.95  693.30  694.40  272236   \n",
       "128 2021-01-11 13:40:00+05:30  694.40  694.55  692.25  692.50  149529   \n",
       "129 2021-01-11 13:45:00+05:30  692.35  693.90  692.20  693.40   83542   \n",
       "130 2021-01-11 13:50:00+05:30  693.50  693.80  693.30  693.65    8724   \n",
       "\n",
       "     next_close    volume_adi  volume_obv  volume_cmf  ...  momentum_wr  \\\n",
       "50       718.20 -1.907681e+06    -1113458   -0.151159  ...   -32.467532   \n",
       "51       714.80 -1.927996e+06    -1184561   -0.161644  ...   -42.857143   \n",
       "52       714.20 -2.198689e+06    -1532595   -0.272495  ...   -91.818182   \n",
       "53       715.40 -1.996476e+06    -1921467   -0.126197  ...   -74.834437   \n",
       "54       712.15 -1.815978e+06    -1730352   -0.076549  ...   -58.940397   \n",
       "..          ...           ...         ...         ...  ...          ...   \n",
       "126      694.40 -3.525842e+06    -4387855    0.125970  ...   -14.285714   \n",
       "127      692.50 -3.435097e+06    -4115619    0.138003  ...    -8.148148   \n",
       "128      693.40 -3.552119e+06    -4265148    0.057863  ...   -46.226415   \n",
       "129      693.65 -3.517720e+06    -4181606    0.008369  ...   -36.470588   \n",
       "130        0.00 -3.514230e+06    -4172882   -0.009463  ...   -30.588235   \n",
       "\n",
       "     momentum_ao  momentum_kama  momentum_roc  momentum_ppo  \\\n",
       "50     -0.979265     718.980482      0.048730    -24.696958   \n",
       "51     -0.808971     718.967298      0.139431    -25.535857   \n",
       "52     -1.012206     718.703349     -0.223339     -7.238714   \n",
       "53     -1.682353     718.172408     -0.487669      6.400643   \n",
       "54     -2.425588     717.958608     -0.237066      6.669585   \n",
       "..           ...            ...           ...           ...   \n",
       "126    -0.561029     692.446514      0.543596    -16.255492   \n",
       "127    -0.073382     693.166737      0.397600     -6.569904   \n",
       "128     0.227794     693.151627      0.086718     -5.934213   \n",
       "129     0.523235     693.162254      0.383641     -9.036984   \n",
       "130     0.905882     693.193816      0.267418    -16.299265   \n",
       "\n",
       "     momentum_ppo_signal  momentum_ppo_hist  others_dr  others_dlr  others_cr  \n",
       "50            -29.136560           4.439601  -0.055633   -0.055648  -0.553557  \n",
       "51            -28.416419           2.880563  -0.055664   -0.055679  -0.608912  \n",
       "52            -24.180878          16.942164  -0.473406   -0.474530  -1.079435  \n",
       "53            -18.064574          24.465217  -0.083940   -0.083975  -1.162469  \n",
       "54            -13.117742          19.787327   0.168020    0.167879  -0.996402  \n",
       "..                   ...                ...        ...         ...        ...  \n",
       "126           -21.557824           5.302331   0.245700    0.245399  -4.013285  \n",
       "127           -18.560240          11.990336   0.115340    0.115274  -3.902574  \n",
       "128           -16.035034          10.100821  -0.273618   -0.273993  -4.165513  \n",
       "129           -14.635424           5.598440   0.129964    0.129880  -4.040963  \n",
       "130           -14.968192          -1.331072   0.036054    0.036048  -4.006366  \n",
       "\n",
       "[81 rows x 90 columns]"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "insttkn = 895745\n",
    "interval='5minute'\n",
    "\n",
    "todaydt=datetime.date.today()\n",
    "hud_ago=todaydt-datetime.timedelta(days=3)\n",
    "to_date=datetime.date.isoformat(todaydt)\n",
    "from_date=datetime.date.isoformat(hud_ago)\n",
    "\n",
    "old_lst = kite.historical_data(insttkn, from_date, to_date, interval,continuous=False)\n",
    "\n",
    "print(len(old_lst))\n",
    "mydf = pd.DataFrame(old_lst)\n",
    "mydf['next_close'] = mydf['close'].shift(periods=-1)\n",
    "# mydf = ta.utils.dropna(mydf)\n",
    "mydf = ta.add_all_ta_features(mydf, open=\"open\", high=\"high\", low=\"low\", close=\"close\", volume=\"volume\")\n",
    "mydf = mydf.fillna(0)[50:]\n",
    "mydf.to_csv('mydf.csv', index=False)\n",
    "\n",
    "# train,test = train_test_split(mydf, test_size=0.02, shuffle=False)\n",
    "\n",
    "# train.to_csv('train.csv', index=False)\n",
    "# test.to_csv('test.csv', index=False)\n",
    "\n",
    "mydf"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.1"
  },
  "varInspector": {
   "cols": {
    "lenName": 16,
    "lenType": 16,
    "lenVar": 40
   },
   "kernels_config": {
    "python": {
     "delete_cmd_postfix": "",
     "delete_cmd_prefix": "del ",
     "library": "var_list.py",
     "varRefreshCmd": "print(var_dic_list())"
    },
    "r": {
     "delete_cmd_postfix": ") ",
     "delete_cmd_prefix": "rm(",
     "library": "var_list.r",
     "varRefreshCmd": "cat(var_dic_list()) "
    }
   },
   "types_to_exclude": [
    "module",
    "function",
    "builtin_function_or_method",
    "instance",
    "_Feature"
   ],
   "window_display": false
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}