SBT_08 Python Classes and web parsing.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Классы в Python"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Foo\n",
      "1 2\n",
      "123\n"
     ]
    }
   ],
   "source": [
    "class Foo(object):\n",
    "    def __init__(self):\n",
    "        self.x = 1\n",
    "        self.y = 2\n",
    "        print \"Foo\"\n",
    "    \n",
    "    def __my_method(self, z):\n",
    "        print z\n",
    "        \n",
    "    def f(self, z):\n",
    "        self.__my_method(z)\n",
    "\n",
    "foo = Foo()\n",
    "print foo.x, foo.y\n",
    "foo.f(123)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<__main__.Foo instance at 0x0000000003981848>\n"
     ]
    }
   ],
   "source": [
    "print foo"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<type 'instance'>\n"
     ]
    }
   ],
   "source": [
    "print type(foo)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Декораторы в Python"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "import time\n",
    "\n",
    "\n",
    "def timer(f):\n",
    "    def tmp(*args, **kwargs):\n",
    "        t = time.time()\n",
    "        res = f(*args, **kwargs)\n",
    "        print \"Время выполнения функции %s: %f\" % (str(f), (time.time()-t))\n",
    "        return res\n",
    "\n",
    "    return tmp\n",
    "\n",
    "\n",
    "def memoized(f):\n",
    "    memory = {}\n",
    "\n",
    "    def wrapper(*args, **kwargs):\n",
    "        key = (tuple(args), hash(tuple(sorted(kwargs.items()))))\n",
    "        if not key in memory:\n",
    "            memory[key] = f(*args, **kwargs)\n",
    "        return memory[key]\n",
    "    return wrapper"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Время выполнения функции <function wrapper at 0x0000000003A5BF98>: 0.009000\n",
      "Время выполнения функции <function wrapper at 0x0000000003A5BF98>: 0.000000\n",
      "Время выполнения функции <function wrapper at 0x0000000003A5BF98>: 0.000000\n",
      "Время выполнения функции <function wrapper at 0x0000000003A5BF98>: 0.000000\n",
      "Время выполнения функции <function wrapper at 0x0000000003A5BF98>: 0.000000\n",
      "Время выполнения функции <function wrapper at 0x0000000003A5BF98>: 0.000000\n",
      "Время выполнения функции <function wrapper at 0x0000000003A5BF98>: 0.000000\n",
      "Время выполнения функции <function wrapper at 0x0000000003A5BF98>: 0.000000\n",
      "Время выполнения функции <function wrapper at 0x0000000003A5BF98>: 0.000000\n",
      "Время выполнения функции <function wrapper at 0x0000000003A5BF98>: 0.000000\n",
      "Время выполнения функции <function wrapper at 0x0000000003A5BF98>: 0.000000\n",
      "Время выполнения функции <function wrapper at 0x0000000003A5BF98>: 0.000000\n",
      "Время выполнения функции <function wrapper at 0x0000000003A5BF98>: 0.000000\n",
      "Время выполнения функции <function wrapper at 0x0000000003A5BF98>: 0.000000\n",
      "Время выполнения функции <function wrapper at 0x0000000003A5BF98>: 0.000000\n",
      "Время выполнения функции <function wrapper at 0x0000000003A5BF98>: 0.000000\n",
      "Время выполнения функции <function wrapper at 0x0000000003A5BF98>: 0.000000\n",
      "Время выполнения функции <function wrapper at 0x0000000003A5BF98>: 0.000000\n",
      "Время выполнения функции <function wrapper at 0x0000000003A5BF98>: 0.000000\n",
      "Время выполнения функции <function wrapper at 0x0000000003A5BF98>: 0.000000\n",
      "Время выполнения функции <function wrapper at 0x0000000003A5BF98>: 0.000000\n",
      "Время выполнения функции <function wrapper at 0x0000000003A5BF98>: 0.000000\n",
      "Время выполнения функции <function wrapper at 0x0000000003A5BF98>: 0.000000\n",
      "Время выполнения функции <function wrapper at 0x0000000003A5BF98>: 0.000000\n",
      "Время выполнения функции <function wrapper at 0x0000000003A5BF98>: 0.000000\n",
      "Время выполнения функции <function wrapper at 0x0000000003A5BF98>: 0.000000\n",
      "Время выполнения функции <function wrapper at 0x0000000003A5BF98>: 0.000000\n",
      "Время выполнения функции <function wrapper at 0x0000000003A5BF98>: 0.000000\n",
      "Время выполнения функции <function wrapper at 0x0000000003A5BF98>: 0.000000\n",
      "Время выполнения функции <function wrapper at 0x0000000003A5BF98>: 0.000000\n",
      "Время выполнения функции <function wrapper at 0x0000000003A5BF98>: 0.000000\n",
      "Время выполнения функции <function wrapper at 0x0000000003A5BF98>: 0.000000\n",
      "Время выполнения функции <function wrapper at 0x0000000003A5BF98>: 0.000000\n",
      "Время выполнения функции <function wrapper at 0x0000000003A5BF98>: 0.000000\n",
      "Время выполнения функции <function wrapper at 0x0000000003A5BF98>: 0.000000\n",
      "Время выполнения функции <function wrapper at 0x0000000003A5BF98>: 0.000000\n",
      "Время выполнения функции <function wrapper at 0x0000000003A5BF98>: 0.000000\n",
      "Время выполнения функции <function wrapper at 0x0000000003A5BF98>: 0.000000\n",
      "Время выполнения функции <function wrapper at 0x0000000003A5BF98>: 0.000000\n",
      "Время выполнения функции <function wrapper at 0x0000000003A5BF98>: 0.000000\n",
      "Время выполнения функции <function wrapper at 0x0000000003A5BF98>: 0.000000\n",
      "Время выполнения функции <function wrapper at 0x0000000003A5BF98>: 0.000000\n",
      "Время выполнения функции <function wrapper at 0x0000000003A5BF98>: 0.000000\n",
      "Время выполнения функции <function wrapper at 0x0000000003A5BF98>: 0.000000\n",
      "Время выполнения функции <function wrapper at 0x0000000003A5BF98>: 0.000000\n",
      "Время выполнения функции <function wrapper at 0x0000000003A5BF98>: 0.000000\n",
      "Время выполнения функции <function wrapper at 0x0000000003A5BF98>: 0.000000\n",
      "Время выполнения функции <function wrapper at 0x0000000003A5BF98>: 0.000000\n",
      "Время выполнения функции <function wrapper at 0x0000000003A5BF98>: 0.000000\n",
      "Время выполнения функции <function wrapper at 0x0000000003A5BF98>: 0.000000\n"
     ]
    }
   ],
   "source": [
    "@timer\n",
    "@memoized\n",
    "def sqr(x):\n",
    "    return x ** 19000\n",
    "\n",
    "#sqr = timer(sqr)\n",
    "\n",
    "for k in range(50):\n",
    "    sqr(3 ** 7)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Foo\n",
      "<property object at 0x0000000003A398B8>\n"
     ]
    }
   ],
   "source": [
    "class Foo:\n",
    "    def __init__(self):\n",
    "        print \"Foo\"\n",
    "\n",
    "    @property\n",
    "    def x(self):\n",
    "        return 1\n",
    "    \n",
    "foo = Foo()\n",
    "print Foo.x"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Beautiful Soup, urllib и chardet: парсим веб-страницы"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "from collections import deque\n",
    "import codecs\n",
    "import os\n",
    "import time\n",
    "import urllib2\n",
    "import chardet\n",
    "\n",
    "site = \"http://simple.wikipedia.org\"\n",
    "\n",
    "def read_url(name):\n",
    "    html = urllib2.urlopen(name).read()\n",
    "    encoding = chardet.detect(html)['encoding']\n",
    "    if encoding != 'unicode':\n",
    "        html = html.decode(encoding)\n",
    "    return html\n",
    "\n",
    "\n",
    "def parse_url(name):\n",
    "    html = read_url(name)\n",
    "    soup = bs4.BeautifulSoup(html)\n",
    "\n",
    "    wiki = \"/wiki/\"\n",
    "\n",
    "    return [x for x in [hit.attrs.get(\"href\",\"\") for hit in soup.findAll('a')] if x[:len(wiki)] == wiki]\n",
    "\n",
    "def save_html(link, link_id, docs_folder):\n",
    "        html = read_url(link)\n",
    "        out_file = codecs.open(os.path.join(docs_folder, str(link_id) + \".html\"), \"w\", \"utf-8\")\n",
    "        print >> out_file, html\n",
    "\n",
    "\n",
    "def crawl(start_link=\"/wiki/Main_Page\"):\n",
    "    queue = deque([start_link])\n",
    "    id_queue = deque([0])\n",
    "\n",
    "    cur_id = 0\n",
    "\n",
    "    visited = dict()\n",
    "\n",
    "    links_id_map = codecs.open(\"link_id_map.txt\", \"w\", \"utf-8\")\n",
    "    graph = codecs.open(\"graph.txt\", \"w\", \"utf-8\")\n",
    "    visited[start_link] = 0\n",
    "\n",
    "    while queue:\n",
    "        link = site + queue.popleft()\n",
    "        print link\n",
    "        link_id = id_queue.popleft()\n",
    "\n",
    "        try:\n",
    "            try:\n",
    "                save_html(link, link_id, \"./docs/\")\n",
    "            except:\n",
    "                time.sleep(60)\n",
    "                save_html(link, link_id, \"./docs/\")\n",
    "\n",
    "            print >> links_id_map, link_id, link\n",
    "\n",
    "\n",
    "            links = parse_url(link)\n",
    "\n",
    "            links_ids = []\n",
    "            for lnk in links:\n",
    "                if lnk in visited.keys():\n",
    "                    links_ids.append(visited[lnk])\n",
    "                else:\n",
    "                    cur_id += 1\n",
    "                    visited[lnk] = cur_id\n",
    "                    queue.append(lnk)\n",
    "                    id_queue.append(cur_id)\n",
    "\n",
    "            print >> graph, str(link_id) + \":\" + \",\".join([str(i) for i in links_ids])\n",
    "        except:\n",
    "            pass"
   ]
  }
 ],
 "metadata": {
  "anaconda-cloud": {},
  "kernelspec": {
   "display_name": "Python [default]",
   "language": "python",
   "name": "python2"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.12"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}