Untitled122.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-12-05T20:48:46.636416Z",
     "start_time": "2020-12-05T20:48:27.235302Z"
    }
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/hemangjoshi37a_gmail_com/.local/lib/python3.8/site-packages/pandas/compat/__init__.py:97: UserWarning: Could not import the lzma module. Your installed Python is incomplete. Attempting to use lzma compression will result in a RuntimeError.\n",
      "  warnings.warn(msg)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "access token : URIKsT4Cgsg3yJxZoXYpVhKKUa48EEoQ\n",
      "request token : VqLPphSFCnnhnqH28fneLhnd6hYCc8J6\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "\n",
       "    <div class=\"bk-root\">\n",
       "        <a href=\"https://bokeh.org\" target=\"_blank\" class=\"bk-logo bk-logo-small bk-logo-notebook\"></a>\n",
       "        <span id=\"1001\">Loading BokehJS ...</span>\n",
       "    </div>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/javascript": [
       "\n",
       "(function(root) {\n",
       "  function now() {\n",
       "    return new Date();\n",
       "  }\n",
       "\n",
       "  var force = true;\n",
       "\n",
       "  if (typeof root._bokeh_onload_callbacks === \"undefined\" || force === true) {\n",
       "    root._bokeh_onload_callbacks = [];\n",
       "    root._bokeh_is_loading = undefined;\n",
       "  }\n",
       "\n",
       "  var JS_MIME_TYPE = 'application/javascript';\n",
       "  var HTML_MIME_TYPE = 'text/html';\n",
       "  var EXEC_MIME_TYPE = 'application/vnd.bokehjs_exec.v0+json';\n",
       "  var CLASS_NAME = 'output_bokeh rendered_html';\n",
       "\n",
       "  /**\n",
       "   * Render data to the DOM node\n",
       "   */\n",
       "  function render(props, node) {\n",
       "    var script = document.createElement(\"script\");\n",
       "    node.appendChild(script);\n",
       "  }\n",
       "\n",
       "  /**\n",
       "   * Handle when an output is cleared or removed\n",
       "   */\n",
       "  function handleClearOutput(event, handle) {\n",
       "    var cell = handle.cell;\n",
       "\n",
       "    var id = cell.output_area._bokeh_element_id;\n",
       "    var server_id = cell.output_area._bokeh_server_id;\n",
       "    // Clean up Bokeh references\n",
       "    if (id != null && id in Bokeh.index) {\n",
       "      Bokeh.index[id].model.document.clear();\n",
       "      delete Bokeh.index[id];\n",
       "    }\n",
       "\n",
       "    if (server_id !== undefined) {\n",
       "      // Clean up Bokeh references\n",
       "      var cmd = \"from bokeh.io.state import curstate; print(curstate().uuid_to_server['\" + server_id + \"'].get_sessions()[0].document.roots[0]._id)\";\n",
       "      cell.notebook.kernel.execute(cmd, {\n",
       "        iopub: {\n",
       "          output: function(msg) {\n",
       "            var id = msg.content.text.trim();\n",
       "            if (id in Bokeh.index) {\n",
       "              Bokeh.index[id].model.document.clear();\n",
       "              delete Bokeh.index[id];\n",
       "            }\n",
       "          }\n",
       "        }\n",
       "      });\n",
       "      // Destroy server and session\n",
       "      var cmd = \"import bokeh.io.notebook as ion; ion.destroy_server('\" + server_id + \"')\";\n",
       "      cell.notebook.kernel.execute(cmd);\n",
       "    }\n",
       "  }\n",
       "\n",
       "  /**\n",
       "   * Handle when a new output is added\n",
       "   */\n",
       "  function handleAddOutput(event, handle) {\n",
       "    var output_area = handle.output_area;\n",
       "    var output = handle.output;\n",
       "\n",
       "    // limit handleAddOutput to display_data with EXEC_MIME_TYPE content only\n",
       "    if ((output.output_type != \"display_data\") || (!output.data.hasOwnProperty(EXEC_MIME_TYPE))) {\n",
       "      return\n",
       "    }\n",
       "\n",
       "    var toinsert = output_area.element.find(\".\" + CLASS_NAME.split(' ')[0]);\n",
       "\n",
       "    if (output.metadata[EXEC_MIME_TYPE][\"id\"] !== undefined) {\n",
       "      toinsert[toinsert.length - 1].firstChild.textContent = output.data[JS_MIME_TYPE];\n",
       "      // store reference to embed id on output_area\n",
       "      output_area._bokeh_element_id = output.metadata[EXEC_MIME_TYPE][\"id\"];\n",
       "    }\n",
       "    if (output.metadata[EXEC_MIME_TYPE][\"server_id\"] !== undefined) {\n",
       "      var bk_div = document.createElement(\"div\");\n",
       "      bk_div.innerHTML = output.data[HTML_MIME_TYPE];\n",
       "      var script_attrs = bk_div.children[0].attributes;\n",
       "      for (var i = 0; i < script_attrs.length; i++) {\n",
       "        toinsert[toinsert.length - 1].firstChild.setAttribute(script_attrs[i].name, script_attrs[i].value);\n",
       "        toinsert[toinsert.length - 1].firstChild.textContent = bk_div.children[0].textContent\n",
       "      }\n",
       "      // store reference to server id on output_area\n",
       "      output_area._bokeh_server_id = output.metadata[EXEC_MIME_TYPE][\"server_id\"];\n",
       "    }\n",
       "  }\n",
       "\n",
       "  function register_renderer(events, OutputArea) {\n",
       "\n",
       "    function append_mime(data, metadata, element) {\n",
       "      // create a DOM node to render to\n",
       "      var toinsert = this.create_output_subarea(\n",
       "        metadata,\n",
       "        CLASS_NAME,\n",
       "        EXEC_MIME_TYPE\n",
       "      );\n",
       "      this.keyboard_manager.register_events(toinsert);\n",
       "      // Render to node\n",
       "      var props = {data: data, metadata: metadata[EXEC_MIME_TYPE]};\n",
       "      render(props, toinsert[toinsert.length - 1]);\n",
       "      element.append(toinsert);\n",
       "      return toinsert\n",
       "    }\n",
       "\n",
       "    /* Handle when an output is cleared or removed */\n",
       "    events.on('clear_output.CodeCell', handleClearOutput);\n",
       "    events.on('delete.Cell', handleClearOutput);\n",
       "\n",
       "    /* Handle when a new output is added */\n",
       "    events.on('output_added.OutputArea', handleAddOutput);\n",
       "\n",
       "    /**\n",
       "     * Register the mime type and append_mime function with output_area\n",
       "     */\n",
       "    OutputArea.prototype.register_mime_type(EXEC_MIME_TYPE, append_mime, {\n",
       "      /* Is output safe? */\n",
       "      safe: true,\n",
       "      /* Index of renderer in `output_area.display_order` */\n",
       "      index: 0\n",
       "    });\n",
       "  }\n",
       "\n",
       "  // register the mime type if in Jupyter Notebook environment and previously unregistered\n",
       "  if (root.Jupyter !== undefined) {\n",
       "    var events = require('base/js/events');\n",
       "    var OutputArea = require('notebook/js/outputarea').OutputArea;\n",
       "\n",
       "    if (OutputArea.prototype.mime_types().indexOf(EXEC_MIME_TYPE) == -1) {\n",
       "      register_renderer(events, OutputArea);\n",
       "    }\n",
       "  }\n",
       "\n",
       "  \n",
       "  if (typeof (root._bokeh_timeout) === \"undefined\" || force === true) {\n",
       "    root._bokeh_timeout = Date.now() + 5000;\n",
       "    root._bokeh_failed_load = false;\n",
       "  }\n",
       "\n",
       "  var NB_LOAD_WARNING = {'data': {'text/html':\n",
       "     \"<div style='background-color: #fdd'>\\n\"+\n",
       "     \"<p>\\n\"+\n",
       "     \"BokehJS does not appear to have successfully loaded. If loading BokehJS from CDN, this \\n\"+\n",
       "     \"may be due to a slow or bad network connection. Possible fixes:\\n\"+\n",
       "     \"</p>\\n\"+\n",
       "     \"<ul>\\n\"+\n",
       "     \"<li>re-rerun `output_notebook()` to attempt to load from CDN again, or</li>\\n\"+\n",
       "     \"<li>use INLINE resources instead, as so:</li>\\n\"+\n",
       "     \"</ul>\\n\"+\n",
       "     \"<code>\\n\"+\n",
       "     \"from bokeh.resources import INLINE\\n\"+\n",
       "     \"output_notebook(resources=INLINE)\\n\"+\n",
       "     \"</code>\\n\"+\n",
       "     \"</div>\"}};\n",
       "\n",
       "  function display_loaded() {\n",
       "    var el = document.getElementById(\"1001\");\n",
       "    if (el != null) {\n",
       "      el.textContent = \"BokehJS is loading...\";\n",
       "    }\n",
       "    if (root.Bokeh !== undefined) {\n",
       "      if (el != null) {\n",
       "        el.textContent = \"BokehJS \" + root.Bokeh.version + \" successfully loaded.\";\n",
       "      }\n",
       "    } else if (Date.now() < root._bokeh_timeout) {\n",
       "      setTimeout(display_loaded, 100)\n",
       "    }\n",
       "  }\n",
       "\n",
       "\n",
       "  function run_callbacks() {\n",
       "    try {\n",
       "      root._bokeh_onload_callbacks.forEach(function(callback) {\n",
       "        if (callback != null)\n",
       "          callback();\n",
       "      });\n",
       "    } finally {\n",
       "      delete root._bokeh_onload_callbacks\n",
       "    }\n",
       "    console.debug(\"Bokeh: all callbacks have finished\");\n",
       "  }\n",
       "\n",
       "  function load_libs(css_urls, js_urls, callback) {\n",
       "    if (css_urls == null) css_urls = [];\n",
       "    if (js_urls == null) js_urls = [];\n",
       "\n",
       "    root._bokeh_onload_callbacks.push(callback);\n",
       "    if (root._bokeh_is_loading > 0) {\n",
       "      console.debug(\"Bokeh: BokehJS is being loaded, scheduling callback at\", now());\n",
       "      return null;\n",
       "    }\n",
       "    if (js_urls == null || js_urls.length === 0) {\n",
       "      run_callbacks();\n",
       "      return null;\n",
       "    }\n",
       "    console.debug(\"Bokeh: BokehJS not loaded, scheduling load and callback at\", now());\n",
       "    root._bokeh_is_loading = css_urls.length + js_urls.length;\n",
       "\n",
       "    function on_load() {\n",
       "      root._bokeh_is_loading--;\n",
       "      if (root._bokeh_is_loading === 0) {\n",
       "        console.debug(\"Bokeh: all BokehJS libraries/stylesheets loaded\");\n",
       "        run_callbacks()\n",
       "      }\n",
       "    }\n",
       "\n",
       "    function on_error() {\n",
       "      console.error(\"failed to load \" + url);\n",
       "    }\n",
       "\n",
       "    for (var i = 0; i < css_urls.length; i++) {\n",
       "      var url = css_urls[i];\n",
       "      const element = document.createElement(\"link\");\n",
       "      element.onload = on_load;\n",
       "      element.onerror = on_error;\n",
       "      element.rel = \"stylesheet\";\n",
       "      element.type = \"text/css\";\n",
       "      element.href = url;\n",
       "      console.debug(\"Bokeh: injecting link tag for BokehJS stylesheet: \", url);\n",
       "      document.body.appendChild(element);\n",
       "    }\n",
       "\n",
       "    const hashes = {\"https://cdn.bokeh.org/bokeh/release/bokeh-2.2.3.min.js\": \"T2yuo9Oe71Cz/I4X9Ac5+gpEa5a8PpJCDlqKYO0CfAuEszu1JrXLl8YugMqYe3sM\", \"https://cdn.bokeh.org/bokeh/release/bokeh-widgets-2.2.3.min.js\": \"98GDGJ0kOMCUMUePhksaQ/GYgB3+NH9h996V88sh3aOiUNX3N+fLXAtry6xctSZ6\", \"https://cdn.bokeh.org/bokeh/release/bokeh-tables-2.2.3.min.js\": \"89bArO+nlbP3sgakeHjCo1JYxYR5wufVgA3IbUvDY+K7w4zyxJqssu7wVnfeKCq8\"};\n",
       "\n",
       "    for (var i = 0; i < js_urls.length; i++) {\n",
       "      var url = js_urls[i];\n",
       "      var element = document.createElement('script');\n",
       "      element.onload = on_load;\n",
       "      element.onerror = on_error;\n",
       "      element.async = false;\n",
       "      element.src = url;\n",
       "      if (url in hashes) {\n",
       "        element.crossOrigin = \"anonymous\";\n",
       "        element.integrity = \"sha384-\" + hashes[url];\n",
       "      }\n",
       "      console.debug(\"Bokeh: injecting script tag for BokehJS library: \", url);\n",
       "      document.head.appendChild(element);\n",
       "    }\n",
       "  };\n",
       "\n",
       "  function inject_raw_css(css) {\n",
       "    const element = document.createElement(\"style\");\n",
       "    element.appendChild(document.createTextNode(css));\n",
       "    document.body.appendChild(element);\n",
       "  }\n",
       "\n",
       "  \n",
       "  var js_urls = [\"https://cdn.bokeh.org/bokeh/release/bokeh-2.2.3.min.js\", \"https://cdn.bokeh.org/bokeh/release/bokeh-widgets-2.2.3.min.js\", \"https://cdn.bokeh.org/bokeh/release/bokeh-tables-2.2.3.min.js\"];\n",
       "  var css_urls = [];\n",
       "  \n",
       "\n",
       "  var inline_js = [\n",
       "    function(Bokeh) {\n",
       "      Bokeh.set_log_level(\"info\");\n",
       "    },\n",
       "    function(Bokeh) {\n",
       "    \n",
       "    \n",
       "    }\n",
       "  ];\n",
       "\n",
       "  function run_inline_js() {\n",
       "    \n",
       "    if (root.Bokeh !== undefined || force === true) {\n",
       "      \n",
       "    for (var i = 0; i < inline_js.length; i++) {\n",
       "      inline_js[i].call(root, root.Bokeh);\n",
       "    }\n",
       "    if (force === true) {\n",
       "        display_loaded();\n",
       "      }} else if (Date.now() < root._bokeh_timeout) {\n",
       "      setTimeout(run_inline_js, 100);\n",
       "    } else if (!root._bokeh_failed_load) {\n",
       "      console.log(\"Bokeh: BokehJS failed to load within specified timeout.\");\n",
       "      root._bokeh_failed_load = true;\n",
       "    } else if (force !== true) {\n",
       "      var cell = $(document.getElementById(\"1001\")).parents('.cell').data().cell;\n",
       "      cell.output_area.append_execute_result(NB_LOAD_WARNING)\n",
       "    }\n",
       "\n",
       "  }\n",
       "\n",
       "  if (root._bokeh_is_loading === 0) {\n",
       "    console.debug(\"Bokeh: BokehJS loaded, going straight to plotting\");\n",
       "    run_inline_js();\n",
       "  } else {\n",
       "    load_libs(css_urls, js_urls, function() {\n",
       "      console.debug(\"Bokeh: BokehJS plotting callback run at\", now());\n",
       "      run_inline_js();\n",
       "    });\n",
       "  }\n",
       "}(window));"
      ],
      "application/vnd.bokehjs_load.v0+json": "\n(function(root) {\n  function now() {\n    return new Date();\n  }\n\n  var force = true;\n\n  if (typeof root._bokeh_onload_callbacks === \"undefined\" || force === true) {\n    root._bokeh_onload_callbacks = [];\n    root._bokeh_is_loading = undefined;\n  }\n\n  \n\n  \n  if (typeof (root._bokeh_timeout) === \"undefined\" || force === true) {\n    root._bokeh_timeout = Date.now() + 5000;\n    root._bokeh_failed_load = false;\n  }\n\n  var NB_LOAD_WARNING = {'data': {'text/html':\n     \"<div style='background-color: #fdd'>\\n\"+\n     \"<p>\\n\"+\n     \"BokehJS does not appear to have successfully loaded. If loading BokehJS from CDN, this \\n\"+\n     \"may be due to a slow or bad network connection. Possible fixes:\\n\"+\n     \"</p>\\n\"+\n     \"<ul>\\n\"+\n     \"<li>re-rerun `output_notebook()` to attempt to load from CDN again, or</li>\\n\"+\n     \"<li>use INLINE resources instead, as so:</li>\\n\"+\n     \"</ul>\\n\"+\n     \"<code>\\n\"+\n     \"from bokeh.resources import INLINE\\n\"+\n     \"output_notebook(resources=INLINE)\\n\"+\n     \"</code>\\n\"+\n     \"</div>\"}};\n\n  function display_loaded() {\n    var el = document.getElementById(\"1001\");\n    if (el != null) {\n      el.textContent = \"BokehJS is loading...\";\n    }\n    if (root.Bokeh !== undefined) {\n      if (el != null) {\n        el.textContent = \"BokehJS \" + root.Bokeh.version + \" successfully loaded.\";\n      }\n    } else if (Date.now() < root._bokeh_timeout) {\n      setTimeout(display_loaded, 100)\n    }\n  }\n\n\n  function run_callbacks() {\n    try {\n      root._bokeh_onload_callbacks.forEach(function(callback) {\n        if (callback != null)\n          callback();\n      });\n    } finally {\n      delete root._bokeh_onload_callbacks\n    }\n    console.debug(\"Bokeh: all callbacks have finished\");\n  }\n\n  function load_libs(css_urls, js_urls, callback) {\n    if (css_urls == null) css_urls = [];\n    if (js_urls == null) js_urls = [];\n\n    root._bokeh_onload_callbacks.push(callback);\n    if (root._bokeh_is_loading > 0) {\n      console.debug(\"Bokeh: BokehJS is being loaded, scheduling callback at\", now());\n      return null;\n    }\n    if (js_urls == null || js_urls.length === 0) {\n      run_callbacks();\n      return null;\n    }\n    console.debug(\"Bokeh: BokehJS not loaded, scheduling load and callback at\", now());\n    root._bokeh_is_loading = css_urls.length + js_urls.length;\n\n    function on_load() {\n      root._bokeh_is_loading--;\n      if (root._bokeh_is_loading === 0) {\n        console.debug(\"Bokeh: all BokehJS libraries/stylesheets loaded\");\n        run_callbacks()\n      }\n    }\n\n    function on_error() {\n      console.error(\"failed to load \" + url);\n    }\n\n    for (var i = 0; i < css_urls.length; i++) {\n      var url = css_urls[i];\n      const element = document.createElement(\"link\");\n      element.onload = on_load;\n      element.onerror = on_error;\n      element.rel = \"stylesheet\";\n      element.type = \"text/css\";\n      element.href = url;\n      console.debug(\"Bokeh: injecting link tag for BokehJS stylesheet: \", url);\n      document.body.appendChild(element);\n    }\n\n    const hashes = {\"https://cdn.bokeh.org/bokeh/release/bokeh-2.2.3.min.js\": \"T2yuo9Oe71Cz/I4X9Ac5+gpEa5a8PpJCDlqKYO0CfAuEszu1JrXLl8YugMqYe3sM\", \"https://cdn.bokeh.org/bokeh/release/bokeh-widgets-2.2.3.min.js\": \"98GDGJ0kOMCUMUePhksaQ/GYgB3+NH9h996V88sh3aOiUNX3N+fLXAtry6xctSZ6\", \"https://cdn.bokeh.org/bokeh/release/bokeh-tables-2.2.3.min.js\": \"89bArO+nlbP3sgakeHjCo1JYxYR5wufVgA3IbUvDY+K7w4zyxJqssu7wVnfeKCq8\"};\n\n    for (var i = 0; i < js_urls.length; i++) {\n      var url = js_urls[i];\n      var element = document.createElement('script');\n      element.onload = on_load;\n      element.onerror = on_error;\n      element.async = false;\n      element.src = url;\n      if (url in hashes) {\n        element.crossOrigin = \"anonymous\";\n        element.integrity = \"sha384-\" + hashes[url];\n      }\n      console.debug(\"Bokeh: injecting script tag for BokehJS library: \", url);\n      document.head.appendChild(element);\n    }\n  };\n\n  function inject_raw_css(css) {\n    const element = document.createElement(\"style\");\n    element.appendChild(document.createTextNode(css));\n    document.body.appendChild(element);\n  }\n\n  \n  var js_urls = [\"https://cdn.bokeh.org/bokeh/release/bokeh-2.2.3.min.js\", \"https://cdn.bokeh.org/bokeh/release/bokeh-widgets-2.2.3.min.js\", \"https://cdn.bokeh.org/bokeh/release/bokeh-tables-2.2.3.min.js\"];\n  var css_urls = [];\n  \n\n  var inline_js = [\n    function(Bokeh) {\n      Bokeh.set_log_level(\"info\");\n    },\n    function(Bokeh) {\n    \n    \n    }\n  ];\n\n  function run_inline_js() {\n    \n    if (root.Bokeh !== undefined || force === true) {\n      \n    for (var i = 0; i < inline_js.length; i++) {\n      inline_js[i].call(root, root.Bokeh);\n    }\n    if (force === true) {\n        display_loaded();\n      }} else if (Date.now() < root._bokeh_timeout) {\n      setTimeout(run_inline_js, 100);\n    } else if (!root._bokeh_failed_load) {\n      console.log(\"Bokeh: BokehJS failed to load within specified timeout.\");\n      root._bokeh_failed_load = true;\n    } else if (force !== true) {\n      var cell = $(document.getElementById(\"1001\")).parents('.cell').data().cell;\n      cell.output_area.append_execute_result(NB_LOAD_WARNING)\n    }\n\n  }\n\n  if (root._bokeh_is_loading === 0) {\n    console.debug(\"Bokeh: BokehJS loaded, going straight to plotting\");\n    run_inline_js();\n  } else {\n    load_libs(css_urls, js_urls, function() {\n      console.debug(\"Bokeh: BokehJS plotting callback run at\", now());\n      run_inline_js();\n    });\n  }\n}(window));"
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "####################### AUTO LOGIN #####################\n",
    "import login\n",
    "from login import *\n",
    "import pandas as pd\n",
    "import joblib\n",
    "import ta\n",
    "import autokeras as ak\n",
    "import tensorflow as tf\n",
    "tf.get_logger().setLevel('ERROR')\n",
    "import os\n",
    "os.environ['TF_CPP_MIN_LOG_LEVEL'] = '3'\n",
    "pd.set_option('plotting.backend', 'pandas_bokeh')\n",
    "import pandas_bokeh\n",
    "pandas_bokeh.output_notebook()\n",
    "from sklearn.model_selection import train_test_split"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-01-04T08:17:31.770013Z",
     "start_time": "2021-01-04T08:17:06.371405Z"
    },
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "# mydf = joblib.load('./TATASTEEL_DF.p')\n",
    "mydf['next_close'] = mydf['close'].shift(periods=-1)\n",
    "mydf = ta.utils.dropna(mydf)\n",
    "mydf = ta.add_all_ta_features(mydf, open=\"open\", high=\"high\", low=\"low\", close=\"close\", volume=\"volume\")\n",
    "mydf = mydf.fillna(0)[500:]\n",
    "mydf"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-01-05T20:07:33.760591Z",
     "start_time": "2021-01-05T20:07:22.461148Z"
    }
   },
   "outputs": [],
   "source": [
    "train,test = train_test_split(mydf, test_size=0.02, shuffle=True)\n",
    "\n",
    "train.to_csv('train.csv', index=False)\n",
    "test.to_csv('eval.csv', index=False)\n",
    "\n",
    "# train_file_path = 'train.csv'\n",
    "# test_file_path = 'eval.csv'\n",
    "\n",
    "# xtest = test.loc[:, test.columns != 'next_close']\n",
    "# ytest = test['next_close']\n",
    "# xtrain = train.loc[:, train.columns != 'next_close']\n",
    "# ytrain = train['next_close']\n",
    "# print('xtrain : '+str(xtrain.shape))\n",
    "# print('ytrain : '+str(ytrain.shape))\n",
    "# print('xtest : '+str(xtest.shape))\n",
    "# print('ytest : '+str(ytest.shape))\n",
    "\n",
    "# xtrain.to_csv('xtrain.csv', index=False)\n",
    "# xtest.to_csv('xtest.csv', index=False)\n",
    "# ytrain.to_csv('ytrain.csv', index=False)\n",
    "# ytest.to_csv('ytest.csv', index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-01-04T02:29:46.668559Z",
     "start_time": "2021-01-04T02:29:46.665437Z"
    }
   },
   "outputs": [],
   "source": [
    "# reg = ak.StructuredDataRegressor(overwrite=True,max_trials=100)\n",
    "# # reg.fit(train_file_path,'next_close',epochs=100)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-01-04T04:16:12.346852Z",
     "start_time": "2021-01-04T04:16:10.557734Z"
    }
   },
   "outputs": [],
   "source": [
    "# model = reg.export_model()\n",
    "\n",
    "# print(type(model))\n",
    "\n",
    "# try:\n",
    "#     model.save(\"model_autokeras4\", save_format=\"tf\")\n",
    "# except:\n",
    "#     model.save(\"model_autokeras4.h5\")\n",
    "\n",
    "from tensorflow.keras.models import load_model\n",
    "model = load_model(\"model_autokeras4\", custom_objects=ak.CUSTOM_OBJECTS)\n",
    "# predicted_y = loaded_model.predict(tf.expand_dims(x_test, -1))\n",
    "# print(predicted_y)\n",
    "model.summary()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-01-04T04:24:52.985591Z",
     "start_time": "2021-01-04T04:24:52.983169Z"
    }
   },
   "outputs": [],
   "source": [
    "# predicted_y = reg.predict(test_file_path)\n",
    "# print(reg.evaluate(test_file_path, 'next_close'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-01-04T04:23:01.900169Z",
     "start_time": "2021-01-04T04:22:44.861267Z"
    },
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "history = model.fit(xtrain.to_numpy().astype('str'),\n",
    "                    ytrain,\n",
    "                    batch_size=1000,\n",
    "                    epochs=1,\n",
    "                    validation_data=(xtest.to_numpy().astype('str'), ytest))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-01-04T04:23:22.859915Z",
     "start_time": "2021-01-04T04:23:03.303465Z"
    }
   },
   "outputs": [],
   "source": [
    "history = model.fit(xtrain.to_numpy().astype('str'),\n",
    "                    ytrain,\n",
    "                    batch_size=500,\n",
    "                    epochs=1,\n",
    "                    validation_data=(xtest.to_numpy().astype('str'), ytest))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-01-04T04:32:25.525435Z",
     "start_time": "2021-01-04T04:31:57.952750Z"
    }
   },
   "outputs": [],
   "source": [
    "history = model.fit(xtrain.to_numpy().astype('str'),\n",
    "                    ytrain,\n",
    "                    batch_size=250,\n",
    "                    epochs=1,\n",
    "                    validation_data=(xtest.to_numpy().astype('str'), ytest))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-01-04T04:43:12.382882Z",
     "start_time": "2021-01-04T04:42:18.741020Z"
    }
   },
   "outputs": [],
   "source": [
    "history = model.fit(xtrain.to_numpy().astype('str'),\n",
    "                    ytrain,\n",
    "                    batch_size=500,\n",
    "                    epochs=5,\n",
    "                    validation_data=(xtest.to_numpy().astype('str'), ytest))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-01-04T05:03:05.006998Z",
     "start_time": "2021-01-04T05:03:04.005568Z"
    }
   },
   "outputs": [],
   "source": [
    "# pred = model.predict(xtest.to_numpy().astype('str')[-1].reshape((1,89)))\n",
    "pred = model.predict(xtest.to_numpy().astype('str'))\n",
    "preddf = pd.DataFrame({'pred':pred.reshape((pred.shape[0],)),'orig':ytest})\n",
    "preddf['diff'] = preddf['orig'] - preddf['pred']\n",
    "preddf"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-01-04T05:03:08.313731Z",
     "start_time": "2021-01-04T05:03:08.301108Z"
    }
   },
   "outputs": [],
   "source": [
    "pltdf = pd.DataFrame({'pred':preddf['pred'],'orig':preddf['orig']})\n",
    "# pltdf.plot()\n",
    "# pltdf\n",
    "preddf['diff'].describe()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-01-04T05:03:09.955062Z",
     "start_time": "2021-01-04T05:03:09.807148Z"
    }
   },
   "outputs": [],
   "source": [
    "predicted_y = pred\n",
    "pred = pd.DataFrame(predicted_y)\n",
    "orig = pd.DataFrame(test['next_close'])\n",
    "pred.reset_index(drop=True, inplace=True)\n",
    "orig.reset_index(drop=True, inplace=True)\n",
    "pred['next'] = pred.shift(periods=-1)\n",
    "pred['diff'] = pred[0] - pred['next']\n",
    "orig['next'] = orig['next_close'].shift(periods=-1)\n",
    "orig['diff'] = orig['next_close'] - orig['next']\n",
    "\n",
    "newdf = pd.concat([pred['diff'],orig['diff']], axis=1, ignore_index=True)\n",
    "newdf[0:500].plot()\n",
    "\n",
    "plotdf = abs(newdf[0] -  newdf[1])\n",
    "plotdf.plot()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-01-04T05:03:11.603914Z",
     "start_time": "2021-01-04T05:03:11.588918Z"
    }
   },
   "outputs": [],
   "source": [
    "pred"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-01-04T05:03:13.098006Z",
     "start_time": "2021-01-04T05:03:13.087489Z"
    }
   },
   "outputs": [],
   "source": [
    "orig"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-01-04T05:06:17.931827Z",
     "start_time": "2021-01-04T05:06:17.702265Z"
    }
   },
   "outputs": [],
   "source": [
    "decide = []\n",
    "\n",
    "truelist = []\n",
    "falselist = []\n",
    "\n",
    "for i in range(len(orig['diff'])):\n",
    "    if((pred['diff'][i]<0 and orig['diff'][i]<0) or (pred['diff'][i]>0 and orig['diff'][i]>0)):\n",
    "#         decide.append(abs(pred['diff'][i]-orig['diff'][i]))\n",
    "        decide.append(abs(pred['diff'][i]))\n",
    "        truelist.append(abs(pred['diff'][i]))\n",
    "    else:\n",
    "#         decide.append(-abs(pred['diff'][i]-orig['diff'][i]))\n",
    "        decide.append(-abs(pred['diff'][i]))\n",
    "        falselist.append(abs(pred['diff'][i]))\n",
    "    \n",
    "dcdf = pd.DataFrame(decide)\n",
    "dcdf.plot()\n",
    "\n",
    "print(dcdf.sum())\n",
    "print(len(truelist) - len(falselist))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-01-04T05:08:00.359488Z",
     "start_time": "2021-01-04T05:08:00.354025Z"
    }
   },
   "outputs": [],
   "source": [
    "(len(truelist) - len(falselist))/len(truelist) "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-01-04T07:59:04.540166Z",
     "start_time": "2021-01-04T07:59:04.493708Z"
    }
   },
   "outputs": [],
   "source": [
    "model.save('./hj.h5')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-01-04T08:04:38.979639Z",
     "start_time": "2021-01-04T08:04:38.971986Z"
    }
   },
   "outputs": [],
   "source": [
    "import pickle\n",
    "def load_pickle_model_from_file(name_file):\n",
    "    with open(name_file, 'rb') as file:\n",
    "        loaded_model = pickle.load(file)\n",
    "    return loaded_model\n",
    "\n",
    "def save_pickle_model_to_file(model, name_model):\n",
    "    with open('./'+name_model+'.pickle', 'wb') as file:.\n",
    "        pickle.dump(model, file)\n",
    "    print(str('./'+name_model+'.pickle')+' - model saved!')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-01-04T08:05:49.271030Z",
     "start_time": "2021-01-04T08:05:49.267709Z"
    }
   },
   "outputs": [],
   "source": [
    "save_pickle_model_to_file(model, 'name_model')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-12-05T20:56:36.164373Z",
     "start_time": "2020-12-05T20:49:30.771576Z"
    }
   },
   "outputs": [],
   "source": [
    "insttkn = 895745\n",
    "interval='minute'\n",
    "\n",
    "old_lst=[]\n",
    "todaydt=datetime.date.today()\n",
    "hud_ago=todaydt-datetime.timedelta(days=59)\n",
    "to_date=datetime.date.isoformat(todaydt)\n",
    "from_date=datetime.date.isoformat(hud_ago)\n",
    "\n",
    "for i2 in range(50):\n",
    "    new_lst = kite.historical_data(insttkn, from_date, to_date, interval,continuous=False)\n",
    "    old_lst = new_lst + old_lst\n",
    "    todaydt=todaydt-datetime.timedelta(days=60)\n",
    "    hud_ago=hud_ago-datetime.timedelta(days=60)\n",
    "    to_date=datetime.date.isoformat(todaydt)\n",
    "    from_date=datetime.date.isoformat(hud_ago)\n",
    "\n",
    "print(len(old_lst))\n",
    "mydf = pd.DataFrame(old_lst)\n",
    "mydf['next_close'] = mydf['close'].shift(periods=-1)\n",
    "mydf = ta.utils.dropna(mydf)\n",
    "mydf = ta.add_all_ta_features(mydf, open=\"open\", high=\"high\", low=\"low\", close=\"close\", volume=\"volume\")\n",
    "mydf = mydf.fillna(0)[500:]\n",
    "mydf.to_csv('mydf.csv', index=False)\n",
    "\n",
    "train,test = train_test_split(mydf, test_size=0.02, shuffle=False)\n",
    "\n",
    "train.to_csv('train.csv', index=False)\n",
    "test.to_csv('test.csv', index=False)\n",
    "\n",
    "mydf"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# interval='minute'\n",
    "\n",
    "# nfifty_tokens_list = [3861249,60417,1510401,4267265,4268801,81153,2714625,134657,140033,\n",
    "#                       177665,5215745,2800641,225537,232961,1207553,315393,1850625,340481,\n",
    "#                       341249,119553,345089,348929,356865,1270529,1346049,408065,415745,424961,\n",
    "#                       3001089,492033,2939649,519937,2815745,4598529,2977281,633601,3834113,\n",
    "#                       738561,5582849,779521,794369,857857,884737,895745,2953217,3465729,\n",
    "#                       897537,2952193,2889473,969473]\n",
    "\n",
    "# mergeddf = pd.DataFrame()\n",
    "\n",
    "# for i in range(len(nfifty_tokens_list)):\n",
    "#     insttkn = nfifty_tokens_list[i]\n",
    "\n",
    "#     old_lst=[]\n",
    "#     todaydt=datetime.date.today()\n",
    "#     hud_ago=todaydt-datetime.timedelta(days=59)\n",
    "#     to_date=datetime.date.isoformat(todaydt)\n",
    "#     from_date=datetime.date.isoformat(hud_ago)\n",
    "\n",
    "#     for i2 in range(35):\n",
    "#         new_lst = kite.historical_data(insttkn, from_date, to_date, interval,continuous=False)\n",
    "#         old_lst = new_lst + old_lst\n",
    "#         todaydt=todaydt-datetime.timedelta(days=60)\n",
    "#         hud_ago=hud_ago-datetime.timedelta(days=60)\n",
    "#         to_date=datetime.date.isoformat(todaydt)\n",
    "#         from_date=datetime.date.isoformat(hud_ago)\n",
    "    \n",
    "#     print(len(old_lst))\n",
    "    \n",
    "#     globals()['mydf'+str(nfifty_tokens_list[i])] = pd.DataFrame(old_lst)\n",
    "    \n",
    "#     globals()['mydf'+str(nfifty_tokens_list[i])]['next_close'] = globals()['mydf'+str(nfifty_tokens_list[i])]['close'].shift(periods=-1)\n",
    "#     globals()['mydf'+str(nfifty_tokens_list[i])] = globals()['mydf'+str(nfifty_tokens_list[i])][:-1].fillna(0)\n",
    "#     globals()['mydf'+str(nfifty_tokens_list[i])] = ta.add_all_ta_features(globals()['mydf'+str(nfifty_tokens_list[i])], open=\"open\", high=\"high\", low=\"low\", close=\"close\", volume=\"volume\")\n",
    "#     globals()['mydf'+str(nfifty_tokens_list[i])] = globals()['mydf'+str(nfifty_tokens_list[i])].fillna(0)[500:]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "nfifty_tokens_list = [3861249,60417,1510401,4267265,4268801,81153,2714625,134657,140033,\n",
    "                      177665,5215745,2800641,225537,232961,1207553,315393,1850625,340481,\n",
    "                      341249,119553,345089,348929,356865,1270529,1346049,408065,415745,424961,\n",
    "                      3001089,492033,2939649,519937,2815745,4598529,2977281,633601,3834113,\n",
    "                      738561,5582849,779521,794369,857857,884737,895745,2953217,3465729,\n",
    "                      897537,2952193,2889473,969473]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "106218\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/hemangjoshi37a_gmail_com/.local/lib/python3.8/site-packages/ta/trend.py:768: RuntimeWarning: invalid value encountered in double_scalars\n",
      "  dip[i] = 100 * (self._dip[i] / self._trs[i])\n",
      "/home/hemangjoshi37a_gmail_com/.local/lib/python3.8/site-packages/ta/trend.py:772: RuntimeWarning: invalid value encountered in double_scalars\n",
      "  din[i] = 100 * (self._din[i] / self._trs[i])\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "60479\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/hemangjoshi37a_gmail_com/.local/lib/python3.8/site-packages/ta/trend.py:768: RuntimeWarning: invalid value encountered in double_scalars\n",
      "  dip[i] = 100 * (self._dip[i] / self._trs[i])\n",
      "/home/hemangjoshi37a_gmail_com/.local/lib/python3.8/site-packages/ta/trend.py:772: RuntimeWarning: invalid value encountered in double_scalars\n",
      "  din[i] = 100 * (self._din[i] / self._trs[i])\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "106217\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/hemangjoshi37a_gmail_com/.local/lib/python3.8/site-packages/ta/trend.py:768: RuntimeWarning: invalid value encountered in double_scalars\n",
      "  dip[i] = 100 * (self._dip[i] / self._trs[i])\n",
      "/home/hemangjoshi37a_gmail_com/.local/lib/python3.8/site-packages/ta/trend.py:772: RuntimeWarning: invalid value encountered in double_scalars\n",
      "  din[i] = 100 * (self._din[i] / self._trs[i])\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "106215\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/hemangjoshi37a_gmail_com/.local/lib/python3.8/site-packages/ta/trend.py:768: RuntimeWarning: invalid value encountered in double_scalars\n",
      "  dip[i] = 100 * (self._dip[i] / self._trs[i])\n",
      "/home/hemangjoshi37a_gmail_com/.local/lib/python3.8/site-packages/ta/trend.py:772: RuntimeWarning: invalid value encountered in double_scalars\n",
      "  din[i] = 100 * (self._din[i] / self._trs[i])\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "106218\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/hemangjoshi37a_gmail_com/.local/lib/python3.8/site-packages/ta/trend.py:768: RuntimeWarning: invalid value encountered in double_scalars\n",
      "  dip[i] = 100 * (self._dip[i] / self._trs[i])\n",
      "/home/hemangjoshi37a_gmail_com/.local/lib/python3.8/site-packages/ta/trend.py:772: RuntimeWarning: invalid value encountered in double_scalars\n",
      "  din[i] = 100 * (self._din[i] / self._trs[i])\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "106218\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/hemangjoshi37a_gmail_com/.local/lib/python3.8/site-packages/ta/trend.py:768: RuntimeWarning: invalid value encountered in double_scalars\n",
      "  dip[i] = 100 * (self._dip[i] / self._trs[i])\n",
      "/home/hemangjoshi37a_gmail_com/.local/lib/python3.8/site-packages/ta/trend.py:772: RuntimeWarning: invalid value encountered in double_scalars\n",
      "  din[i] = 100 * (self._din[i] / self._trs[i])\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "106219\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/hemangjoshi37a_gmail_com/.local/lib/python3.8/site-packages/ta/trend.py:768: RuntimeWarning: invalid value encountered in double_scalars\n",
      "  dip[i] = 100 * (self._dip[i] / self._trs[i])\n",
      "/home/hemangjoshi37a_gmail_com/.local/lib/python3.8/site-packages/ta/trend.py:772: RuntimeWarning: invalid value encountered in double_scalars\n",
      "  din[i] = 100 * (self._din[i] / self._trs[i])\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "106220\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/hemangjoshi37a_gmail_com/.local/lib/python3.8/site-packages/ta/trend.py:768: RuntimeWarning: invalid value encountered in double_scalars\n",
      "  dip[i] = 100 * (self._dip[i] / self._trs[i])\n",
      "/home/hemangjoshi37a_gmail_com/.local/lib/python3.8/site-packages/ta/trend.py:772: RuntimeWarning: invalid value encountered in double_scalars\n",
      "  din[i] = 100 * (self._din[i] / self._trs[i])\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "106218\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/hemangjoshi37a_gmail_com/.local/lib/python3.8/site-packages/ta/trend.py:768: RuntimeWarning: invalid value encountered in double_scalars\n",
      "  dip[i] = 100 * (self._dip[i] / self._trs[i])\n",
      "/home/hemangjoshi37a_gmail_com/.local/lib/python3.8/site-packages/ta/trend.py:772: RuntimeWarning: invalid value encountered in double_scalars\n",
      "  din[i] = 100 * (self._din[i] / self._trs[i])\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "106220\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/hemangjoshi37a_gmail_com/.local/lib/python3.8/site-packages/ta/trend.py:768: RuntimeWarning: invalid value encountered in double_scalars\n",
      "  dip[i] = 100 * (self._dip[i] / self._trs[i])\n",
      "/home/hemangjoshi37a_gmail_com/.local/lib/python3.8/site-packages/ta/trend.py:772: RuntimeWarning: invalid value encountered in double_scalars\n",
      "  din[i] = 100 * (self._din[i] / self._trs[i])\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "106221\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/hemangjoshi37a_gmail_com/.local/lib/python3.8/site-packages/ta/trend.py:768: RuntimeWarning: invalid value encountered in double_scalars\n",
      "  dip[i] = 100 * (self._dip[i] / self._trs[i])\n",
      "/home/hemangjoshi37a_gmail_com/.local/lib/python3.8/site-packages/ta/trend.py:772: RuntimeWarning: invalid value encountered in double_scalars\n",
      "  din[i] = 100 * (self._din[i] / self._trs[i])\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "106221\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/hemangjoshi37a_gmail_com/.local/lib/python3.8/site-packages/ta/trend.py:768: RuntimeWarning: invalid value encountered in double_scalars\n",
      "  dip[i] = 100 * (self._dip[i] / self._trs[i])\n",
      "/home/hemangjoshi37a_gmail_com/.local/lib/python3.8/site-packages/ta/trend.py:772: RuntimeWarning: invalid value encountered in double_scalars\n",
      "  din[i] = 100 * (self._din[i] / self._trs[i])\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "106222\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/hemangjoshi37a_gmail_com/.local/lib/python3.8/site-packages/ta/trend.py:768: RuntimeWarning: invalid value encountered in double_scalars\n",
      "  dip[i] = 100 * (self._dip[i] / self._trs[i])\n",
      "/home/hemangjoshi37a_gmail_com/.local/lib/python3.8/site-packages/ta/trend.py:772: RuntimeWarning: invalid value encountered in double_scalars\n",
      "  din[i] = 100 * (self._din[i] / self._trs[i])\n"
     ]
    }
   ],
   "source": [
    "interval='5minute'\n",
    "\n",
    "mergeddf = pd.DataFrame()\n",
    "\n",
    "for i in range(len(nfifty_tokens_list)):\n",
    "    insttkn = nfifty_tokens_list[i]\n",
    "\n",
    "    old_lst=[]\n",
    "    todaydt=datetime.date.today()\n",
    "    hud_ago=todaydt-datetime.timedelta(days=59)\n",
    "    to_date=datetime.date.isoformat(todaydt)\n",
    "    from_date=datetime.date.isoformat(hud_ago)\n",
    "\n",
    "    for i2 in range(35):\n",
    "        new_lst = kite.historical_data(insttkn, from_date, to_date, interval,continuous=False)\n",
    "        old_lst = new_lst + old_lst\n",
    "        todaydt=todaydt-datetime.timedelta(days=60)\n",
    "        hud_ago=hud_ago-datetime.timedelta(days=60)\n",
    "        to_date=datetime.date.isoformat(todaydt)\n",
    "        from_date=datetime.date.isoformat(hud_ago)\n",
    "    \n",
    "    print(len(old_lst))\n",
    "    \n",
    "    globals()['mydf'+str(nfifty_tokens_list[i])] = pd.DataFrame(old_lst)\n",
    "    mydf = pd.DataFrame(old_lst)\n",
    "    mydf['next_close'] = mydf['close'].shift(periods=-1)\n",
    "    mydf = mydf[:-1].fillna(0)\n",
    "    mydf = ta.add_all_ta_features(mydf, open=\"open\", high=\"high\", low=\"low\", close=\"close\", volume=\"volume\")\n",
    "    mydf = mydf.fillna(0)[500:]\n",
    "    mydf.to_csv('./dfs2/mydf'+str(nfifty_tokens_list[i])+'.csv', index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/hemangjoshi37a_gmail_com/.local/lib/python3.8/site-packages/pandas/compat/__init__.py:97: UserWarning: Could not import the lzma module. Your installed Python is incomplete. Attempting to use lzma compression will result in a RuntimeError.\n",
      "  warnings.warn(msg)\n"
     ]
    }
   ],
   "source": [
    "#### COMBINE ALL CSV FILES ############\n",
    "import os\n",
    "import glob\n",
    "import pandas as pd\n",
    "os.chdir(\"/home/hemangjoshi37a_gmail_com/pfiles/dfs3\")\n",
    "\n",
    "extension = 'csv'\n",
    "all_filenames = [i for i in glob.glob('*.{}'.format(extension))]\n",
    "\n",
    "combined_csv = pd.concat([pd.read_csv(f) for f in all_filenames ])\n",
    "combined_csv.to_csv( \"combined_csv.csv\", index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "3861249\n",
      "60417\n",
      "1510401\n",
      "4267265\n",
      "4268801\n",
      "81153\n",
      "2714625\n",
      "134657\n",
      "140033\n",
      "177665\n",
      "5215745\n",
      "2800641\n",
      "225537\n",
      "232961\n",
      "1207553\n",
      "315393\n",
      "1850625\n",
      "340481\n",
      "341249\n",
      "119553\n",
      "345089\n",
      "348929\n",
      "356865\n",
      "1270529\n",
      "1346049\n",
      "408065\n",
      "415745\n",
      "424961\n",
      "3001089\n",
      "492033\n",
      "2939649\n",
      "519937\n",
      "2815745\n",
      "4598529\n",
      "2977281\n",
      "633601\n",
      "3834113\n",
      "738561\n",
      "5582849\n",
      "779521\n",
      "794369\n",
      "857857\n",
      "884737\n",
      "895745\n",
      "2953217\n",
      "3465729\n",
      "897537\n",
      "2952193\n",
      "2889473\n",
      "969473\n"
     ]
    }
   ],
   "source": [
    "for ii in nfifty_tokens_list:\n",
    "    df1 = pd.read_csv('/home/hemangjoshi37a_gmail_com/pfiles/dfs2/mydf'+str(ii)+'.csv')\n",
    "    df1['bullbear']=0\n",
    "    for ind in df1.index:\n",
    "         if(df1['next_close'][ind]>df1['close'][ind]):\n",
    "                df1.iloc[ind, df1.columns.get_loc('bullbear')]=1\n",
    "         if(df1['next_close'][ind]<df1['close'][ind]):\n",
    "                df1.iloc[ind, df1.columns.get_loc('bullbear')]=-1\n",
    "    del df1['next_close']\n",
    "    df1.to_csv('./dfs3/mydf'+str(ii)+'.csv', index=False)\n",
    "    df1=pd.DataFrame()\n",
    "    print(ii)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>predict</th>\n",
       "      <th>p-1</th>\n",
       "      <th>p0</th>\n",
       "      <th>p1</th>\n",
       "      <th>predict0</th>\n",
       "      <th>predict0.1</th>\n",
       "      <th>p-10</th>\n",
       "      <th>p00</th>\n",
       "      <th>p10</th>\n",
       "      <th>date</th>\n",
       "      <th>...</th>\n",
       "      <th>momentum_ao</th>\n",
       "      <th>momentum_kama</th>\n",
       "      <th>momentum_roc</th>\n",
       "      <th>momentum_ppo</th>\n",
       "      <th>momentum_ppo_signal</th>\n",
       "      <th>momentum_ppo_hist</th>\n",
       "      <th>others_dr</th>\n",
       "      <th>others_dlr</th>\n",
       "      <th>others_cr</th>\n",
       "      <th>bullbear</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1</td>\n",
       "      <td>0.255243</td>\n",
       "      <td>0.015290</td>\n",
       "      <td>0.729468</td>\n",
       "      <td>1211.815662</td>\n",
       "      <td>1</td>\n",
       "      <td>0.299202</td>\n",
       "      <td>0.249407</td>\n",
       "      <td>0.451392</td>\n",
       "      <td>2015-04-22 13:25:00+05:30</td>\n",
       "      <td>...</td>\n",
       "      <td>1.040588</td>\n",
       "      <td>1212.988970</td>\n",
       "      <td>-0.098905</td>\n",
       "      <td>-31.284405</td>\n",
       "      <td>-26.134756</td>\n",
       "      <td>-5.149649</td>\n",
       "      <td>-0.071726</td>\n",
       "      <td>-0.071752</td>\n",
       "      <td>-8.071293</td>\n",
       "      <td>-1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1</td>\n",
       "      <td>0.105430</td>\n",
       "      <td>0.012522</td>\n",
       "      <td>0.882048</td>\n",
       "      <td>1210.233417</td>\n",
       "      <td>1</td>\n",
       "      <td>0.283552</td>\n",
       "      <td>0.197892</td>\n",
       "      <td>0.518556</td>\n",
       "      <td>2015-04-22 13:30:00+05:30</td>\n",
       "      <td>...</td>\n",
       "      <td>0.475382</td>\n",
       "      <td>1212.785747</td>\n",
       "      <td>-0.244656</td>\n",
       "      <td>-23.416115</td>\n",
       "      <td>-25.591028</td>\n",
       "      <td>2.174913</td>\n",
       "      <td>-0.090753</td>\n",
       "      <td>-0.090794</td>\n",
       "      <td>-8.154721</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>1</td>\n",
       "      <td>0.379543</td>\n",
       "      <td>0.025739</td>\n",
       "      <td>0.594718</td>\n",
       "      <td>1211.134855</td>\n",
       "      <td>1</td>\n",
       "      <td>0.355435</td>\n",
       "      <td>0.238500</td>\n",
       "      <td>0.406064</td>\n",
       "      <td>2015-04-22 13:35:00+05:30</td>\n",
       "      <td>...</td>\n",
       "      <td>0.049500</td>\n",
       "      <td>1212.710212</td>\n",
       "      <td>-0.193594</td>\n",
       "      <td>-17.546216</td>\n",
       "      <td>-23.982066</td>\n",
       "      <td>6.435850</td>\n",
       "      <td>0.045418</td>\n",
       "      <td>0.045407</td>\n",
       "      <td>-8.113007</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>1</td>\n",
       "      <td>0.461270</td>\n",
       "      <td>0.026133</td>\n",
       "      <td>0.512597</td>\n",
       "      <td>1211.492326</td>\n",
       "      <td>-1</td>\n",
       "      <td>0.382438</td>\n",
       "      <td>0.248062</td>\n",
       "      <td>0.369500</td>\n",
       "      <td>2015-04-22 13:40:00+05:30</td>\n",
       "      <td>...</td>\n",
       "      <td>-0.356441</td>\n",
       "      <td>1212.645670</td>\n",
       "      <td>-0.150818</td>\n",
       "      <td>-4.371426</td>\n",
       "      <td>-20.059938</td>\n",
       "      <td>15.688512</td>\n",
       "      <td>0.001651</td>\n",
       "      <td>0.001651</td>\n",
       "      <td>-8.111490</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>-1</td>\n",
       "      <td>0.727872</td>\n",
       "      <td>0.020882</td>\n",
       "      <td>0.251247</td>\n",
       "      <td>1210.879491</td>\n",
       "      <td>-1</td>\n",
       "      <td>0.445155</td>\n",
       "      <td>0.230079</td>\n",
       "      <td>0.324766</td>\n",
       "      <td>2015-04-22 13:45:00+05:30</td>\n",
       "      <td>...</td>\n",
       "      <td>-0.813824</td>\n",
       "      <td>1212.626607</td>\n",
       "      <td>-0.135982</td>\n",
       "      <td>-7.767964</td>\n",
       "      <td>-17.601543</td>\n",
       "      <td>9.833579</td>\n",
       "      <td>0.016508</td>\n",
       "      <td>0.016506</td>\n",
       "      <td>-8.096322</td>\n",
       "      <td>-1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5191715</th>\n",
       "      <td>1</td>\n",
       "      <td>0.153162</td>\n",
       "      <td>0.003570</td>\n",
       "      <td>0.843268</td>\n",
       "      <td>25279.700829</td>\n",
       "      <td>1</td>\n",
       "      <td>0.380959</td>\n",
       "      <td>0.186415</td>\n",
       "      <td>0.432625</td>\n",
       "      <td>2021-01-08 09:40:00+05:30</td>\n",
       "      <td>...</td>\n",
       "      <td>171.677059</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.719166</td>\n",
       "      <td>24.562028</td>\n",
       "      <td>18.060399</td>\n",
       "      <td>6.501630</td>\n",
       "      <td>-0.512618</td>\n",
       "      <td>-0.513937</td>\n",
       "      <td>135.123083</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5191716</th>\n",
       "      <td>-1</td>\n",
       "      <td>0.881354</td>\n",
       "      <td>0.001258</td>\n",
       "      <td>0.117387</td>\n",
       "      <td>25294.370224</td>\n",
       "      <td>-1</td>\n",
       "      <td>0.495133</td>\n",
       "      <td>0.186547</td>\n",
       "      <td>0.318320</td>\n",
       "      <td>2021-01-08 09:45:00+05:30</td>\n",
       "      <td>...</td>\n",
       "      <td>177.443235</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>1.196357</td>\n",
       "      <td>19.138593</td>\n",
       "      <td>18.276037</td>\n",
       "      <td>0.862555</td>\n",
       "      <td>0.277447</td>\n",
       "      <td>0.277063</td>\n",
       "      <td>135.775426</td>\n",
       "      <td>-1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5191717</th>\n",
       "      <td>-1</td>\n",
       "      <td>0.531656</td>\n",
       "      <td>0.005615</td>\n",
       "      <td>0.462729</td>\n",
       "      <td>25298.819357</td>\n",
       "      <td>-1</td>\n",
       "      <td>0.396069</td>\n",
       "      <td>0.233925</td>\n",
       "      <td>0.370006</td>\n",
       "      <td>2021-01-08 09:50:00+05:30</td>\n",
       "      <td>...</td>\n",
       "      <td>183.331471</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.967737</td>\n",
       "      <td>11.542094</td>\n",
       "      <td>16.929249</td>\n",
       "      <td>-5.387155</td>\n",
       "      <td>-0.181818</td>\n",
       "      <td>-0.181984</td>\n",
       "      <td>135.346744</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5191718</th>\n",
       "      <td>1</td>\n",
       "      <td>0.467975</td>\n",
       "      <td>0.005056</td>\n",
       "      <td>0.526968</td>\n",
       "      <td>25325.966781</td>\n",
       "      <td>-1</td>\n",
       "      <td>0.426049</td>\n",
       "      <td>0.197550</td>\n",
       "      <td>0.376401</td>\n",
       "      <td>2021-01-08 09:55:00+05:30</td>\n",
       "      <td>...</td>\n",
       "      <td>188.601176</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>1.123593</td>\n",
       "      <td>7.416548</td>\n",
       "      <td>15.026709</td>\n",
       "      <td>-7.610160</td>\n",
       "      <td>0.107112</td>\n",
       "      <td>0.107054</td>\n",
       "      <td>135.598828</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5191719</th>\n",
       "      <td>1</td>\n",
       "      <td>0.462989</td>\n",
       "      <td>0.000930</td>\n",
       "      <td>0.536081</td>\n",
       "      <td>25331.686508</td>\n",
       "      <td>-1</td>\n",
       "      <td>0.409088</td>\n",
       "      <td>0.216006</td>\n",
       "      <td>0.374906</td>\n",
       "      <td>2021-01-08 10:00:00+05:30</td>\n",
       "      <td>...</td>\n",
       "      <td>190.740000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>1.488893</td>\n",
       "      <td>20.061763</td>\n",
       "      <td>16.033719</td>\n",
       "      <td>4.028043</td>\n",
       "      <td>0.448755</td>\n",
       "      <td>0.447751</td>\n",
       "      <td>136.656089</td>\n",
       "      <td>-1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5191720 rows × 99 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "         predict       p-1        p0        p1      predict0  predict0.1  \\\n",
       "0              1  0.255243  0.015290  0.729468   1211.815662           1   \n",
       "1              1  0.105430  0.012522  0.882048   1210.233417           1   \n",
       "2              1  0.379543  0.025739  0.594718   1211.134855           1   \n",
       "3              1  0.461270  0.026133  0.512597   1211.492326          -1   \n",
       "4             -1  0.727872  0.020882  0.251247   1210.879491          -1   \n",
       "...          ...       ...       ...       ...           ...         ...   \n",
       "5191715        1  0.153162  0.003570  0.843268  25279.700829           1   \n",
       "5191716       -1  0.881354  0.001258  0.117387  25294.370224          -1   \n",
       "5191717       -1  0.531656  0.005615  0.462729  25298.819357          -1   \n",
       "5191718        1  0.467975  0.005056  0.526968  25325.966781          -1   \n",
       "5191719        1  0.462989  0.000930  0.536081  25331.686508          -1   \n",
       "\n",
       "             p-10       p00       p10                       date  ...  \\\n",
       "0        0.299202  0.249407  0.451392  2015-04-22 13:25:00+05:30  ...   \n",
       "1        0.283552  0.197892  0.518556  2015-04-22 13:30:00+05:30  ...   \n",
       "2        0.355435  0.238500  0.406064  2015-04-22 13:35:00+05:30  ...   \n",
       "3        0.382438  0.248062  0.369500  2015-04-22 13:40:00+05:30  ...   \n",
       "4        0.445155  0.230079  0.324766  2015-04-22 13:45:00+05:30  ...   \n",
       "...           ...       ...       ...                        ...  ...   \n",
       "5191715  0.380959  0.186415  0.432625  2021-01-08 09:40:00+05:30  ...   \n",
       "5191716  0.495133  0.186547  0.318320  2021-01-08 09:45:00+05:30  ...   \n",
       "5191717  0.396069  0.233925  0.370006  2021-01-08 09:50:00+05:30  ...   \n",
       "5191718  0.426049  0.197550  0.376401  2021-01-08 09:55:00+05:30  ...   \n",
       "5191719  0.409088  0.216006  0.374906  2021-01-08 10:00:00+05:30  ...   \n",
       "\n",
       "         momentum_ao  momentum_kama  momentum_roc  momentum_ppo  \\\n",
       "0           1.040588    1212.988970     -0.098905    -31.284405   \n",
       "1           0.475382    1212.785747     -0.244656    -23.416115   \n",
       "2           0.049500    1212.710212     -0.193594    -17.546216   \n",
       "3          -0.356441    1212.645670     -0.150818     -4.371426   \n",
       "4          -0.813824    1212.626607     -0.135982     -7.767964   \n",
       "...              ...            ...           ...           ...   \n",
       "5191715   171.677059       0.000000      0.719166     24.562028   \n",
       "5191716   177.443235       0.000000      1.196357     19.138593   \n",
       "5191717   183.331471       0.000000      0.967737     11.542094   \n",
       "5191718   188.601176       0.000000      1.123593      7.416548   \n",
       "5191719   190.740000       0.000000      1.488893     20.061763   \n",
       "\n",
       "         momentum_ppo_signal  momentum_ppo_hist  others_dr  others_dlr  \\\n",
       "0                 -26.134756          -5.149649  -0.071726   -0.071752   \n",
       "1                 -25.591028           2.174913  -0.090753   -0.090794   \n",
       "2                 -23.982066           6.435850   0.045418    0.045407   \n",
       "3                 -20.059938          15.688512   0.001651    0.001651   \n",
       "4                 -17.601543           9.833579   0.016508    0.016506   \n",
       "...                      ...                ...        ...         ...   \n",
       "5191715            18.060399           6.501630  -0.512618   -0.513937   \n",
       "5191716            18.276037           0.862555   0.277447    0.277063   \n",
       "5191717            16.929249          -5.387155  -0.181818   -0.181984   \n",
       "5191718            15.026709          -7.610160   0.107112    0.107054   \n",
       "5191719            16.033719           4.028043   0.448755    0.447751   \n",
       "\n",
       "          others_cr  bullbear  \n",
       "0         -8.071293        -1  \n",
       "1         -8.154721         1  \n",
       "2         -8.113007         1  \n",
       "3         -8.111490         1  \n",
       "4         -8.096322        -1  \n",
       "...             ...       ...  \n",
       "5191715  135.123083         1  \n",
       "5191716  135.775426        -1  \n",
       "5191717  135.346744         1  \n",
       "5191718  135.598828         1  \n",
       "5191719  136.656089        -1  \n",
       "\n",
       "[5191720 rows x 99 columns]"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "ipf = pd.read_csv('./dfs3/predicted_final.csv')\n",
    "ipf"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>predict</th>\n",
       "      <th>p-1</th>\n",
       "      <th>p0</th>\n",
       "      <th>p1</th>\n",
       "      <th>predict0</th>\n",
       "      <th>predict0.1</th>\n",
       "      <th>p-10</th>\n",
       "      <th>p00</th>\n",
       "      <th>p10</th>\n",
       "      <th>volume</th>\n",
       "      <th>...</th>\n",
       "      <th>momentum_stoch_signal</th>\n",
       "      <th>momentum_wr</th>\n",
       "      <th>momentum_ao</th>\n",
       "      <th>momentum_roc</th>\n",
       "      <th>momentum_ppo</th>\n",
       "      <th>momentum_ppo_signal</th>\n",
       "      <th>momentum_ppo_hist</th>\n",
       "      <th>others_dr</th>\n",
       "      <th>others_dlr</th>\n",
       "      <th>next_close</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1</td>\n",
       "      <td>0.255243</td>\n",
       "      <td>0.015290</td>\n",
       "      <td>0.729468</td>\n",
       "      <td>1211.815662</td>\n",
       "      <td>1</td>\n",
       "      <td>0.299202</td>\n",
       "      <td>0.249407</td>\n",
       "      <td>0.451392</td>\n",
       "      <td>21712</td>\n",
       "      <td>...</td>\n",
       "      <td>38.220548</td>\n",
       "      <td>-76.027397</td>\n",
       "      <td>1.040588</td>\n",
       "      <td>-0.098905</td>\n",
       "      <td>-31.284405</td>\n",
       "      <td>-26.134756</td>\n",
       "      <td>-5.149649</td>\n",
       "      <td>-0.071726</td>\n",
       "      <td>-0.071752</td>\n",
       "      <td>1210.98</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1</td>\n",
       "      <td>0.105430</td>\n",
       "      <td>0.012522</td>\n",
       "      <td>0.882048</td>\n",
       "      <td>1210.233417</td>\n",
       "      <td>1</td>\n",
       "      <td>0.283552</td>\n",
       "      <td>0.197892</td>\n",
       "      <td>0.518556</td>\n",
       "      <td>39644</td>\n",
       "      <td>...</td>\n",
       "      <td>18.758551</td>\n",
       "      <td>-93.521127</td>\n",
       "      <td>0.475382</td>\n",
       "      <td>-0.244656</td>\n",
       "      <td>-23.416115</td>\n",
       "      <td>-25.591028</td>\n",
       "      <td>2.174913</td>\n",
       "      <td>-0.090753</td>\n",
       "      <td>-0.090794</td>\n",
       "      <td>1211.53</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>1</td>\n",
       "      <td>0.379543</td>\n",
       "      <td>0.025739</td>\n",
       "      <td>0.594718</td>\n",
       "      <td>1211.134855</td>\n",
       "      <td>1</td>\n",
       "      <td>0.355435</td>\n",
       "      <td>0.238500</td>\n",
       "      <td>0.406064</td>\n",
       "      <td>38402</td>\n",
       "      <td>...</td>\n",
       "      <td>17.474436</td>\n",
       "      <td>-78.028169</td>\n",
       "      <td>0.049500</td>\n",
       "      <td>-0.193594</td>\n",
       "      <td>-17.546216</td>\n",
       "      <td>-23.982066</td>\n",
       "      <td>6.435850</td>\n",
       "      <td>0.045418</td>\n",
       "      <td>0.045407</td>\n",
       "      <td>1211.55</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>1</td>\n",
       "      <td>0.461270</td>\n",
       "      <td>0.026133</td>\n",
       "      <td>0.512597</td>\n",
       "      <td>1211.492326</td>\n",
       "      <td>-1</td>\n",
       "      <td>0.382438</td>\n",
       "      <td>0.248062</td>\n",
       "      <td>0.369500</td>\n",
       "      <td>71066</td>\n",
       "      <td>...</td>\n",
       "      <td>16.995305</td>\n",
       "      <td>-77.464789</td>\n",
       "      <td>-0.356441</td>\n",
       "      <td>-0.150818</td>\n",
       "      <td>-4.371426</td>\n",
       "      <td>-20.059938</td>\n",
       "      <td>15.688512</td>\n",
       "      <td>0.001651</td>\n",
       "      <td>0.001651</td>\n",
       "      <td>1211.75</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>-1</td>\n",
       "      <td>0.727872</td>\n",
       "      <td>0.020882</td>\n",
       "      <td>0.251247</td>\n",
       "      <td>1210.879491</td>\n",
       "      <td>-1</td>\n",
       "      <td>0.445155</td>\n",
       "      <td>0.230079</td>\n",
       "      <td>0.324766</td>\n",
       "      <td>18010</td>\n",
       "      <td>...</td>\n",
       "      <td>24.225352</td>\n",
       "      <td>-71.830986</td>\n",
       "      <td>-0.813824</td>\n",
       "      <td>-0.135982</td>\n",
       "      <td>-7.767964</td>\n",
       "      <td>-17.601543</td>\n",
       "      <td>9.833579</td>\n",
       "      <td>0.016508</td>\n",
       "      <td>0.016506</td>\n",
       "      <td>1209.33</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5191715</th>\n",
       "      <td>1</td>\n",
       "      <td>0.153162</td>\n",
       "      <td>0.003570</td>\n",
       "      <td>0.843268</td>\n",
       "      <td>25279.700829</td>\n",
       "      <td>1</td>\n",
       "      <td>0.380959</td>\n",
       "      <td>0.186415</td>\n",
       "      <td>0.432625</td>\n",
       "      <td>2929</td>\n",
       "      <td>...</td>\n",
       "      <td>79.620021</td>\n",
       "      <td>-39.563514</td>\n",
       "      <td>171.677059</td>\n",
       "      <td>0.719166</td>\n",
       "      <td>24.562028</td>\n",
       "      <td>18.060399</td>\n",
       "      <td>6.501630</td>\n",
       "      <td>-0.512618</td>\n",
       "      <td>-0.513937</td>\n",
       "      <td>25300.00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5191716</th>\n",
       "      <td>-1</td>\n",
       "      <td>0.881354</td>\n",
       "      <td>0.001258</td>\n",
       "      <td>0.117387</td>\n",
       "      <td>25294.370224</td>\n",
       "      <td>-1</td>\n",
       "      <td>0.495133</td>\n",
       "      <td>0.186547</td>\n",
       "      <td>0.318320</td>\n",
       "      <td>1028</td>\n",
       "      <td>...</td>\n",
       "      <td>76.474847</td>\n",
       "      <td>-23.894796</td>\n",
       "      <td>177.443235</td>\n",
       "      <td>1.196357</td>\n",
       "      <td>19.138593</td>\n",
       "      <td>18.276037</td>\n",
       "      <td>0.862555</td>\n",
       "      <td>0.277447</td>\n",
       "      <td>0.277063</td>\n",
       "      <td>25254.00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5191717</th>\n",
       "      <td>-1</td>\n",
       "      <td>0.531656</td>\n",
       "      <td>0.005615</td>\n",
       "      <td>0.462729</td>\n",
       "      <td>25298.819357</td>\n",
       "      <td>-1</td>\n",
       "      <td>0.396069</td>\n",
       "      <td>0.233925</td>\n",
       "      <td>0.370006</td>\n",
       "      <td>467</td>\n",
       "      <td>...</td>\n",
       "      <td>67.450103</td>\n",
       "      <td>-34.191382</td>\n",
       "      <td>183.331471</td>\n",
       "      <td>0.967737</td>\n",
       "      <td>11.542094</td>\n",
       "      <td>16.929249</td>\n",
       "      <td>-5.387155</td>\n",
       "      <td>-0.181818</td>\n",
       "      <td>-0.181984</td>\n",
       "      <td>25281.05</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5191718</th>\n",
       "      <td>1</td>\n",
       "      <td>0.467975</td>\n",
       "      <td>0.005056</td>\n",
       "      <td>0.526968</td>\n",
       "      <td>25325.966781</td>\n",
       "      <td>-1</td>\n",
       "      <td>0.426049</td>\n",
       "      <td>0.197550</td>\n",
       "      <td>0.376401</td>\n",
       "      <td>876</td>\n",
       "      <td>...</td>\n",
       "      <td>71.259093</td>\n",
       "      <td>-28.136542</td>\n",
       "      <td>188.601176</td>\n",
       "      <td>1.123593</td>\n",
       "      <td>7.416548</td>\n",
       "      <td>15.026709</td>\n",
       "      <td>-7.610160</td>\n",
       "      <td>0.107112</td>\n",
       "      <td>0.107054</td>\n",
       "      <td>25394.50</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5191719</th>\n",
       "      <td>1</td>\n",
       "      <td>0.462989</td>\n",
       "      <td>0.000930</td>\n",
       "      <td>0.536081</td>\n",
       "      <td>25331.686508</td>\n",
       "      <td>-1</td>\n",
       "      <td>0.409088</td>\n",
       "      <td>0.216006</td>\n",
       "      <td>0.374906</td>\n",
       "      <td>3909</td>\n",
       "      <td>...</td>\n",
       "      <td>76.196893</td>\n",
       "      <td>-9.081398</td>\n",
       "      <td>190.740000</td>\n",
       "      <td>1.488893</td>\n",
       "      <td>20.061763</td>\n",
       "      <td>16.033719</td>\n",
       "      <td>4.028043</td>\n",
       "      <td>0.448755</td>\n",
       "      <td>0.447751</td>\n",
       "      <td>0.00</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5191720 rows × 82 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "         predict       p-1        p0        p1      predict0  predict0.1  \\\n",
       "0              1  0.255243  0.015290  0.729468   1211.815662           1   \n",
       "1              1  0.105430  0.012522  0.882048   1210.233417           1   \n",
       "2              1  0.379543  0.025739  0.594718   1211.134855           1   \n",
       "3              1  0.461270  0.026133  0.512597   1211.492326          -1   \n",
       "4             -1  0.727872  0.020882  0.251247   1210.879491          -1   \n",
       "...          ...       ...       ...       ...           ...         ...   \n",
       "5191715        1  0.153162  0.003570  0.843268  25279.700829           1   \n",
       "5191716       -1  0.881354  0.001258  0.117387  25294.370224          -1   \n",
       "5191717       -1  0.531656  0.005615  0.462729  25298.819357          -1   \n",
       "5191718        1  0.467975  0.005056  0.526968  25325.966781          -1   \n",
       "5191719        1  0.462989  0.000930  0.536081  25331.686508          -1   \n",
       "\n",
       "             p-10       p00       p10  volume  ...  momentum_stoch_signal  \\\n",
       "0        0.299202  0.249407  0.451392   21712  ...              38.220548   \n",
       "1        0.283552  0.197892  0.518556   39644  ...              18.758551   \n",
       "2        0.355435  0.238500  0.406064   38402  ...              17.474436   \n",
       "3        0.382438  0.248062  0.369500   71066  ...              16.995305   \n",
       "4        0.445155  0.230079  0.324766   18010  ...              24.225352   \n",
       "...           ...       ...       ...     ...  ...                    ...   \n",
       "5191715  0.380959  0.186415  0.432625    2929  ...              79.620021   \n",
       "5191716  0.495133  0.186547  0.318320    1028  ...              76.474847   \n",
       "5191717  0.396069  0.233925  0.370006     467  ...              67.450103   \n",
       "5191718  0.426049  0.197550  0.376401     876  ...              71.259093   \n",
       "5191719  0.409088  0.216006  0.374906    3909  ...              76.196893   \n",
       "\n",
       "         momentum_wr  momentum_ao  momentum_roc  momentum_ppo  \\\n",
       "0         -76.027397     1.040588     -0.098905    -31.284405   \n",
       "1         -93.521127     0.475382     -0.244656    -23.416115   \n",
       "2         -78.028169     0.049500     -0.193594    -17.546216   \n",
       "3         -77.464789    -0.356441     -0.150818     -4.371426   \n",
       "4         -71.830986    -0.813824     -0.135982     -7.767964   \n",
       "...              ...          ...           ...           ...   \n",
       "5191715   -39.563514   171.677059      0.719166     24.562028   \n",
       "5191716   -23.894796   177.443235      1.196357     19.138593   \n",
       "5191717   -34.191382   183.331471      0.967737     11.542094   \n",
       "5191718   -28.136542   188.601176      1.123593      7.416548   \n",
       "5191719    -9.081398   190.740000      1.488893     20.061763   \n",
       "\n",
       "         momentum_ppo_signal  momentum_ppo_hist  others_dr  others_dlr  \\\n",
       "0                 -26.134756          -5.149649  -0.071726   -0.071752   \n",
       "1                 -25.591028           2.174913  -0.090753   -0.090794   \n",
       "2                 -23.982066           6.435850   0.045418    0.045407   \n",
       "3                 -20.059938          15.688512   0.001651    0.001651   \n",
       "4                 -17.601543           9.833579   0.016508    0.016506   \n",
       "...                      ...                ...        ...         ...   \n",
       "5191715            18.060399           6.501630  -0.512618   -0.513937   \n",
       "5191716            18.276037           0.862555   0.277447    0.277063   \n",
       "5191717            16.929249          -5.387155  -0.181818   -0.181984   \n",
       "5191718            15.026709          -7.610160   0.107112    0.107054   \n",
       "5191719            16.033719           4.028043   0.448755    0.447751   \n",
       "\n",
       "         next_close  \n",
       "0           1210.98  \n",
       "1           1211.53  \n",
       "2           1211.55  \n",
       "3           1211.75  \n",
       "4           1209.33  \n",
       "...             ...  \n",
       "5191715    25300.00  \n",
       "5191716    25254.00  \n",
       "5191717    25281.05  \n",
       "5191718    25394.50  \n",
       "5191719        0.00  \n",
       "\n",
       "[5191720 rows x 82 columns]"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# ipf = pd.read_csv('./dfs3/predicted_final.csv')\n",
    "ipf['next_close'] = ipf['close'].shift(periods=-1)\n",
    "ipf = ipf.fillna(0)\n",
    "ipf = ipf.drop(columns =['others_cr','close','high','low','open','trend_ichimoku_conv',\n",
    "       'momentum_kama','trend_ema_fast','volatility_kcc','volume_vwap','volatility_kch','date',\n",
    "       'volatility_kcl','bullbear','trend_sma_fast','trend_ichimoku_a','volatility_dcm','volatility_dcl'],\n",
    "               axis = 1)\n",
    "\n",
    "ipf"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Index(['predict', 'p-1', 'p0', 'p1', 'predict0', 'predict0.1', 'p-10', 'p00',\n",
       "       'p10', 'volume', 'volume_adi', 'volume_obv', 'volume_cmf', 'volume_fi',\n",
       "       'volume_mfi', 'volume_em', 'volume_sma_em', 'volume_vpt', 'volume_nvi',\n",
       "       'volatility_atr', 'volatility_bbm', 'volatility_bbh', 'volatility_bbl',\n",
       "       'volatility_bbw', 'volatility_bbp', 'volatility_bbhi',\n",
       "       'volatility_bbli', 'volatility_kcw', 'volatility_kcp',\n",
       "       'volatility_kchi', 'volatility_kcli', 'volatility_dch',\n",
       "       'volatility_dcw', 'volatility_dcp', 'volatility_ui', 'trend_macd',\n",
       "       'trend_macd_signal', 'trend_macd_diff', 'trend_sma_slow',\n",
       "       'trend_ema_slow', 'trend_adx', 'trend_adx_pos', 'trend_adx_neg',\n",
       "       'trend_vortex_ind_pos', 'trend_vortex_ind_neg', 'trend_vortex_ind_diff',\n",
       "       'trend_trix', 'trend_mass_index', 'trend_cci', 'trend_dpo', 'trend_kst',\n",
       "       'trend_kst_sig', 'trend_kst_diff', 'trend_ichimoku_base',\n",
       "       'trend_ichimoku_b', 'trend_visual_ichimoku_a',\n",
       "       'trend_visual_ichimoku_b', 'trend_aroon_up', 'trend_aroon_down',\n",
       "       'trend_aroon_ind', 'trend_psar_up', 'trend_psar_down',\n",
       "       'trend_psar_up_indicator', 'trend_psar_down_indicator', 'trend_stc',\n",
       "       'momentum_rsi', 'momentum_stoch_rsi', 'momentum_stoch_rsi_k',\n",
       "       'momentum_stoch_rsi_d', 'momentum_tsi', 'momentum_uo', 'momentum_stoch',\n",
       "       'momentum_stoch_signal', 'momentum_wr', 'momentum_ao', 'momentum_roc',\n",
       "       'momentum_ppo', 'momentum_ppo_signal', 'momentum_ppo_hist', 'others_dr',\n",
       "       'others_dlr', 'next_close'],\n",
       "      dtype='object')"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "ipf.keys()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "ipf.to_csv('./dfs3/predicted_final2.csv', index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "ipf[500:1000].to_csv('./dfs3/temp.csv', index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.1"
  },
  "varInspector": {
   "cols": {
    "lenName": 16,
    "lenType": 16,
    "lenVar": 40
   },
   "kernels_config": {
    "python": {
     "delete_cmd_postfix": "",
     "delete_cmd_prefix": "del ",
     "library": "var_list.py",
     "varRefreshCmd": "print(var_dic_list())"
    },
    "r": {
     "delete_cmd_postfix": ") ",
     "delete_cmd_prefix": "rm(",
     "library": "var_list.r",
     "varRefreshCmd": "cat(var_dic_list()) "
    }
   },
   "types_to_exclude": [
    "module",
    "function",
    "builtin_function_or_method",
    "instance",
    "_Feature"
   ],
   "window_display": false
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}