file_mapping_and_entry_points.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# File mapping and Entry points\n",
    "\n",
    "This notebook creates the tables below from a mix of hand-input data (based on recordings) and computationally-created data via a survey tool.\n",
    "\n",
    "**Important**: This file assumes previous data preparation steps have been run, and won't work if the file below isn't present. - see the readme for more info."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Setup - import library and load data from local file\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "\n",
    "fm = pd.read_csv(\"data/file_mapping_responses.csv\")\n",
    "fm = fm.iloc[:,1:11] #subset the data to just strings and remove the notes columns"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Spread of cards matched to term across different terms\n",
    "\n",
    "This function takes in processed data from the file above, and identifies which cards were matched to which terms, and how many times. `NaN` results mean there were no matches and is equivalent to \"0\". \n",
    "\n",
    "GO Term was reasonably consistent, everything else was much more varied"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>homo_sapiens.gff|100287102</th>\n",
       "      <th>homo_sapiens.gff|HGNC:37102</th>\n",
       "      <th>homo_sapiens.gff|DDX11L1</th>\n",
       "      <th>flybase_d_melanogaster.gaf|FBgn0043467</th>\n",
       "      <th>flybase_d_melanogaster.gaf|GO:0048149</th>\n",
       "      <th>ncbi_homo_sapiens.gene_info|9606</th>\n",
       "      <th>ncbi_homo_sapiens.gene_info|1</th>\n",
       "      <th>ncbi_homo_sapiens.gene_info|A1BG</th>\n",
       "      <th>ncbi_homo_sapiens.gene_info|MIM:138670</th>\n",
       "      <th>ncbi_homo_sapiens.gene_info|HGNC:HGNC:5</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>identifier</th>\n",
       "      <td>15.0</td>\n",
       "      <td>9.0</td>\n",
       "      <td>5.0</td>\n",
       "      <td>13.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>9.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>6.0</td>\n",
       "      <td>4.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>gene</th>\n",
       "      <td>14.0</td>\n",
       "      <td>6.0</td>\n",
       "      <td>7.0</td>\n",
       "      <td>5.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1.0</td>\n",
       "      <td>6.0</td>\n",
       "      <td>6.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>3.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>accession</th>\n",
       "      <td>2.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>4.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>symbol</th>\n",
       "      <td>1.0</td>\n",
       "      <td>4.0</td>\n",
       "      <td>8.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>15.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>dataset/database</th>\n",
       "      <td>1.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>4.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>5.0</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>h. sapiens</th>\n",
       "      <td>1.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>3.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>organism</th>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>8.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>name</th>\n",
       "      <td>1.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>15.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>7.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>2.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>go: protein binding</th>\n",
       "      <td>NaN</td>\n",
       "      <td>1.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>location</th>\n",
       "      <td>NaN</td>\n",
       "      <td>1.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Q9H4C3_HUMAN</th>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>protein</th>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>GO Term</th>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1.0</td>\n",
       "      <td>18.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>d. melanogaster</th>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>3.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>label</th>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>database</th>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Chromosome location</th>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>pathway</th>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>accesssion</th>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>taxon id</th>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>3.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>human</th>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>5.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>chromosome</th>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>3.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2.0</td>\n",
       "      <td>2.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>BRCA1_HUMAN</th>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>links to other dbs</th>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1.0</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>disease</th>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1.0</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                     homo_sapiens.gff|100287102  homo_sapiens.gff|HGNC:37102  \\\n",
       "identifier                                 15.0                          9.0   \n",
       "gene                                       14.0                          6.0   \n",
       "accession                                   2.0                          3.0   \n",
       "symbol                                      1.0                          4.0   \n",
       "dataset/database                            1.0                          3.0   \n",
       "h. sapiens                                  1.0                          2.0   \n",
       "organism                                    1.0                          1.0   \n",
       "name                                        1.0                          2.0   \n",
       "go: protein binding                         NaN                          1.0   \n",
       "location                                    NaN                          1.0   \n",
       "Q9H4C3_HUMAN                                NaN                          NaN   \n",
       "protein                                     NaN                          NaN   \n",
       "GO Term                                     NaN                          NaN   \n",
       "d. melanogaster                             NaN                          NaN   \n",
       "label                                       NaN                          NaN   \n",
       "database                                    NaN                          NaN   \n",
       "Chromosome location                         NaN                          NaN   \n",
       "pathway                                     NaN                          NaN   \n",
       "accesssion                                  NaN                          NaN   \n",
       "taxon id                                    NaN                          NaN   \n",
       "human                                       NaN                          NaN   \n",
       "chromosome                                  NaN                          NaN   \n",
       "BRCA1_HUMAN                                 NaN                          NaN   \n",
       "links to other dbs                          NaN                          NaN   \n",
       "disease                                     NaN                          NaN   \n",
       "\n",
       "                     homo_sapiens.gff|DDX11L1  \\\n",
       "identifier                                5.0   \n",
       "gene                                      7.0   \n",
       "accession                                 1.0   \n",
       "symbol                                    8.0   \n",
       "dataset/database                          1.0   \n",
       "h. sapiens                                NaN   \n",
       "organism                                  1.0   \n",
       "name                                     15.0   \n",
       "go: protein binding                       NaN   \n",
       "location                                  NaN   \n",
       "Q9H4C3_HUMAN                              1.0   \n",
       "protein                                   NaN   \n",
       "GO Term                                   NaN   \n",
       "d. melanogaster                           NaN   \n",
       "label                                     NaN   \n",
       "database                                  NaN   \n",
       "Chromosome location                       NaN   \n",
       "pathway                                   NaN   \n",
       "accesssion                                NaN   \n",
       "taxon id                                  NaN   \n",
       "human                                     NaN   \n",
       "chromosome                                NaN   \n",
       "BRCA1_HUMAN                               NaN   \n",
       "links to other dbs                        NaN   \n",
       "disease                                   NaN   \n",
       "\n",
       "                     flybase_d_melanogaster.gaf|FBgn0043467  \\\n",
       "identifier                                             13.0   \n",
       "gene                                                    5.0   \n",
       "accession                                               4.0   \n",
       "symbol                                                  3.0   \n",
       "dataset/database                                        4.0   \n",
       "h. sapiens                                              NaN   \n",
       "organism                                                1.0   \n",
       "name                                                    1.0   \n",
       "go: protein binding                                     NaN   \n",
       "location                                                NaN   \n",
       "Q9H4C3_HUMAN                                            NaN   \n",
       "protein                                                 2.0   \n",
       "GO Term                                                 1.0   \n",
       "d. melanogaster                                         3.0   \n",
       "label                                                   1.0   \n",
       "database                                                1.0   \n",
       "Chromosome location                                     NaN   \n",
       "pathway                                                 NaN   \n",
       "accesssion                                              NaN   \n",
       "taxon id                                                NaN   \n",
       "human                                                   NaN   \n",
       "chromosome                                              NaN   \n",
       "BRCA1_HUMAN                                             NaN   \n",
       "links to other dbs                                      NaN   \n",
       "disease                                                 NaN   \n",
       "\n",
       "                     flybase_d_melanogaster.gaf|GO:0048149  \\\n",
       "identifier                                             2.0   \n",
       "gene                                                   NaN   \n",
       "accession                                              1.0   \n",
       "symbol                                                 NaN   \n",
       "dataset/database                                       1.0   \n",
       "h. sapiens                                             NaN   \n",
       "organism                                               NaN   \n",
       "name                                                   NaN   \n",
       "go: protein binding                                    2.0   \n",
       "location                                               NaN   \n",
       "Q9H4C3_HUMAN                                           NaN   \n",
       "protein                                                NaN   \n",
       "GO Term                                               18.0   \n",
       "d. melanogaster                                        NaN   \n",
       "label                                                  NaN   \n",
       "database                                               NaN   \n",
       "Chromosome location                                    1.0   \n",
       "pathway                                                1.0   \n",
       "accesssion                                             1.0   \n",
       "taxon id                                               NaN   \n",
       "human                                                  NaN   \n",
       "chromosome                                             NaN   \n",
       "BRCA1_HUMAN                                            NaN   \n",
       "links to other dbs                                     NaN   \n",
       "disease                                                NaN   \n",
       "\n",
       "                     ncbi_homo_sapiens.gene_info|9606  \\\n",
       "identifier                                        3.0   \n",
       "gene                                              1.0   \n",
       "accession                                         NaN   \n",
       "symbol                                            2.0   \n",
       "dataset/database                                  NaN   \n",
       "h. sapiens                                        3.0   \n",
       "organism                                          8.0   \n",
       "name                                              1.0   \n",
       "go: protein binding                               NaN   \n",
       "location                                          NaN   \n",
       "Q9H4C3_HUMAN                                      NaN   \n",
       "protein                                           NaN   \n",
       "GO Term                                           NaN   \n",
       "d. melanogaster                                   NaN   \n",
       "label                                             NaN   \n",
       "database                                          NaN   \n",
       "Chromosome location                               NaN   \n",
       "pathway                                           NaN   \n",
       "accesssion                                        NaN   \n",
       "taxon id                                          3.0   \n",
       "human                                             5.0   \n",
       "chromosome                                        NaN   \n",
       "BRCA1_HUMAN                                       NaN   \n",
       "links to other dbs                                NaN   \n",
       "disease                                           NaN   \n",
       "\n",
       "                     ncbi_homo_sapiens.gene_info|1  \\\n",
       "identifier                                     9.0   \n",
       "gene                                           6.0   \n",
       "accession                                      NaN   \n",
       "symbol                                         1.0   \n",
       "dataset/database                               NaN   \n",
       "h. sapiens                                     NaN   \n",
       "organism                                       1.0   \n",
       "name                                           1.0   \n",
       "go: protein binding                            NaN   \n",
       "location                                       NaN   \n",
       "Q9H4C3_HUMAN                                   NaN   \n",
       "protein                                        1.0   \n",
       "GO Term                                        NaN   \n",
       "d. melanogaster                                NaN   \n",
       "label                                          NaN   \n",
       "database                                       NaN   \n",
       "Chromosome location                            NaN   \n",
       "pathway                                        NaN   \n",
       "accesssion                                     NaN   \n",
       "taxon id                                       NaN   \n",
       "human                                          NaN   \n",
       "chromosome                                     3.0   \n",
       "BRCA1_HUMAN                                    NaN   \n",
       "links to other dbs                             NaN   \n",
       "disease                                        NaN   \n",
       "\n",
       "                     ncbi_homo_sapiens.gene_info|A1BG  \\\n",
       "identifier                                        2.0   \n",
       "gene                                              6.0   \n",
       "accession                                         1.0   \n",
       "symbol                                           15.0   \n",
       "dataset/database                                  NaN   \n",
       "h. sapiens                                        1.0   \n",
       "organism                                          1.0   \n",
       "name                                              7.0   \n",
       "go: protein binding                               NaN   \n",
       "location                                          NaN   \n",
       "Q9H4C3_HUMAN                                      NaN   \n",
       "protein                                           NaN   \n",
       "GO Term                                           NaN   \n",
       "d. melanogaster                                   NaN   \n",
       "label                                             NaN   \n",
       "database                                          NaN   \n",
       "Chromosome location                               NaN   \n",
       "pathway                                           NaN   \n",
       "accesssion                                        NaN   \n",
       "taxon id                                          NaN   \n",
       "human                                             NaN   \n",
       "chromosome                                        NaN   \n",
       "BRCA1_HUMAN                                       1.0   \n",
       "links to other dbs                                NaN   \n",
       "disease                                           NaN   \n",
       "\n",
       "                     ncbi_homo_sapiens.gene_info|MIM:138670  \\\n",
       "identifier                                              6.0   \n",
       "gene                                                    2.0   \n",
       "accession                                               2.0   \n",
       "symbol                                                  1.0   \n",
       "dataset/database                                        5.0   \n",
       "h. sapiens                                              NaN   \n",
       "organism                                                1.0   \n",
       "name                                                    1.0   \n",
       "go: protein binding                                     NaN   \n",
       "location                                                NaN   \n",
       "Q9H4C3_HUMAN                                            NaN   \n",
       "protein                                                 NaN   \n",
       "GO Term                                                 NaN   \n",
       "d. melanogaster                                         NaN   \n",
       "label                                                   NaN   \n",
       "database                                                NaN   \n",
       "Chromosome location                                     NaN   \n",
       "pathway                                                 NaN   \n",
       "accesssion                                              NaN   \n",
       "taxon id                                                NaN   \n",
       "human                                                   NaN   \n",
       "chromosome                                              2.0   \n",
       "BRCA1_HUMAN                                             NaN   \n",
       "links to other dbs                                      1.0   \n",
       "disease                                                 1.0   \n",
       "\n",
       "                     ncbi_homo_sapiens.gene_info|HGNC:HGNC:5  \n",
       "identifier                                               4.0  \n",
       "gene                                                     3.0  \n",
       "accession                                                1.0  \n",
       "symbol                                                   1.0  \n",
       "dataset/database                                         1.0  \n",
       "h. sapiens                                               NaN  \n",
       "organism                                                 1.0  \n",
       "name                                                     2.0  \n",
       "go: protein binding                                      NaN  \n",
       "location                                                 NaN  \n",
       "Q9H4C3_HUMAN                                             NaN  \n",
       "protein                                                  NaN  \n",
       "GO Term                                                  NaN  \n",
       "d. melanogaster                                          NaN  \n",
       "label                                                    NaN  \n",
       "database                                                 1.0  \n",
       "Chromosome location                                      NaN  \n",
       "pathway                                                  NaN  \n",
       "accesssion                                               NaN  \n",
       "taxon id                                                 NaN  \n",
       "human                                                    NaN  \n",
       "chromosome                                               2.0  \n",
       "BRCA1_HUMAN                                              NaN  \n",
       "links to other dbs                                       NaN  \n",
       "disease                                                  NaN  "
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#\"split a multi choice answer, returning a series with a column \n",
    "# header and the responses with the header stripped out.\"\n",
    "def count_unique_answers( a_series ):\n",
    "    # variable to store the final response in as we build it\n",
    "    tokenise = list()\n",
    "    split_answer = pd.DataFrame(a_series.str.split(\"|\", expand=True))\n",
    "    for i in split_answer:   \n",
    "        tokenise.extend(split_answer[i])\n",
    "        \n",
    "    response = {}\n",
    "    for i in tokenise:\n",
    "        if not pd.isnull(i):  \n",
    "            if i in response:\n",
    "                response[i] = response[i]+1\n",
    "            else:\n",
    "                response[i] = 1\n",
    "    return response\n",
    "\n",
    "result_counts = {}\n",
    "\n",
    "for i in fm: \n",
    "    if (fm[i].dtype == np.object):\n",
    "        result_counts[fm[i].name] = count_unique_answers(fm[i])\n",
    "        \n",
    "results = pd.DataFrame.from_dict(result_counts)\n",
    "results\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Same view as above, but with the axes flipped. "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>identifier</th>\n",
       "      <th>gene</th>\n",
       "      <th>accession</th>\n",
       "      <th>symbol</th>\n",
       "      <th>dataset/database</th>\n",
       "      <th>h. sapiens</th>\n",
       "      <th>organism</th>\n",
       "      <th>name</th>\n",
       "      <th>go: protein binding</th>\n",
       "      <th>location</th>\n",
       "      <th>...</th>\n",
       "      <th>database</th>\n",
       "      <th>Chromosome location</th>\n",
       "      <th>pathway</th>\n",
       "      <th>accesssion</th>\n",
       "      <th>taxon id</th>\n",
       "      <th>human</th>\n",
       "      <th>chromosome</th>\n",
       "      <th>BRCA1_HUMAN</th>\n",
       "      <th>links to other dbs</th>\n",
       "      <th>disease</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>homo_sapiens.gff|100287102</th>\n",
       "      <td>15.0</td>\n",
       "      <td>14.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>homo_sapiens.gff|HGNC:37102</th>\n",
       "      <td>9.0</td>\n",
       "      <td>6.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>4.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>homo_sapiens.gff|DDX11L1</th>\n",
       "      <td>5.0</td>\n",
       "      <td>7.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>8.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1.0</td>\n",
       "      <td>15.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>flybase_d_melanogaster.gaf|FBgn0043467</th>\n",
       "      <td>13.0</td>\n",
       "      <td>5.0</td>\n",
       "      <td>4.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>4.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>1.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>flybase_d_melanogaster.gaf|GO:0048149</th>\n",
       "      <td>2.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ncbi_homo_sapiens.gene_info|9606</th>\n",
       "      <td>3.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>3.0</td>\n",
       "      <td>8.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>3.0</td>\n",
       "      <td>5.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ncbi_homo_sapiens.gene_info|1</th>\n",
       "      <td>9.0</td>\n",
       "      <td>6.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>3.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ncbi_homo_sapiens.gene_info|A1BG</th>\n",
       "      <td>2.0</td>\n",
       "      <td>6.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>15.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>7.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ncbi_homo_sapiens.gene_info|MIM:138670</th>\n",
       "      <td>6.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>5.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ncbi_homo_sapiens.gene_info|HGNC:HGNC:5</th>\n",
       "      <td>4.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>1.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>10 rows × 25 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                         identifier  gene  accession  symbol  \\\n",
       "homo_sapiens.gff|100287102                     15.0  14.0        2.0     1.0   \n",
       "homo_sapiens.gff|HGNC:37102                     9.0   6.0        3.0     4.0   \n",
       "homo_sapiens.gff|DDX11L1                        5.0   7.0        1.0     8.0   \n",
       "flybase_d_melanogaster.gaf|FBgn0043467         13.0   5.0        4.0     3.0   \n",
       "flybase_d_melanogaster.gaf|GO:0048149           2.0   NaN        1.0     NaN   \n",
       "ncbi_homo_sapiens.gene_info|9606                3.0   1.0        NaN     2.0   \n",
       "ncbi_homo_sapiens.gene_info|1                   9.0   6.0        NaN     1.0   \n",
       "ncbi_homo_sapiens.gene_info|A1BG                2.0   6.0        1.0    15.0   \n",
       "ncbi_homo_sapiens.gene_info|MIM:138670          6.0   2.0        2.0     1.0   \n",
       "ncbi_homo_sapiens.gene_info|HGNC:HGNC:5         4.0   3.0        1.0     1.0   \n",
       "\n",
       "                                         dataset/database  h. sapiens  \\\n",
       "homo_sapiens.gff|100287102                            1.0         1.0   \n",
       "homo_sapiens.gff|HGNC:37102                           3.0         2.0   \n",
       "homo_sapiens.gff|DDX11L1                              1.0         NaN   \n",
       "flybase_d_melanogaster.gaf|FBgn0043467                4.0         NaN   \n",
       "flybase_d_melanogaster.gaf|GO:0048149                 1.0         NaN   \n",
       "ncbi_homo_sapiens.gene_info|9606                      NaN         3.0   \n",
       "ncbi_homo_sapiens.gene_info|1                         NaN         NaN   \n",
       "ncbi_homo_sapiens.gene_info|A1BG                      NaN         1.0   \n",
       "ncbi_homo_sapiens.gene_info|MIM:138670                5.0         NaN   \n",
       "ncbi_homo_sapiens.gene_info|HGNC:HGNC:5               1.0         NaN   \n",
       "\n",
       "                                         organism  name  go: protein binding  \\\n",
       "homo_sapiens.gff|100287102                    1.0   1.0                  NaN   \n",
       "homo_sapiens.gff|HGNC:37102                   1.0   2.0                  1.0   \n",
       "homo_sapiens.gff|DDX11L1                      1.0  15.0                  NaN   \n",
       "flybase_d_melanogaster.gaf|FBgn0043467        1.0   1.0                  NaN   \n",
       "flybase_d_melanogaster.gaf|GO:0048149         NaN   NaN                  2.0   \n",
       "ncbi_homo_sapiens.gene_info|9606              8.0   1.0                  NaN   \n",
       "ncbi_homo_sapiens.gene_info|1                 1.0   1.0                  NaN   \n",
       "ncbi_homo_sapiens.gene_info|A1BG              1.0   7.0                  NaN   \n",
       "ncbi_homo_sapiens.gene_info|MIM:138670        1.0   1.0                  NaN   \n",
       "ncbi_homo_sapiens.gene_info|HGNC:HGNC:5       1.0   2.0                  NaN   \n",
       "\n",
       "                                         location  ...  database  \\\n",
       "homo_sapiens.gff|100287102                    NaN  ...       NaN   \n",
       "homo_sapiens.gff|HGNC:37102                   1.0  ...       NaN   \n",
       "homo_sapiens.gff|DDX11L1                      NaN  ...       NaN   \n",
       "flybase_d_melanogaster.gaf|FBgn0043467        NaN  ...       1.0   \n",
       "flybase_d_melanogaster.gaf|GO:0048149         NaN  ...       NaN   \n",
       "ncbi_homo_sapiens.gene_info|9606              NaN  ...       NaN   \n",
       "ncbi_homo_sapiens.gene_info|1                 NaN  ...       NaN   \n",
       "ncbi_homo_sapiens.gene_info|A1BG              NaN  ...       NaN   \n",
       "ncbi_homo_sapiens.gene_info|MIM:138670        NaN  ...       NaN   \n",
       "ncbi_homo_sapiens.gene_info|HGNC:HGNC:5       NaN  ...       1.0   \n",
       "\n",
       "                                         Chromosome location  pathway  \\\n",
       "homo_sapiens.gff|100287102                               NaN      NaN   \n",
       "homo_sapiens.gff|HGNC:37102                              NaN      NaN   \n",
       "homo_sapiens.gff|DDX11L1                                 NaN      NaN   \n",
       "flybase_d_melanogaster.gaf|FBgn0043467                   NaN      NaN   \n",
       "flybase_d_melanogaster.gaf|GO:0048149                    1.0      1.0   \n",
       "ncbi_homo_sapiens.gene_info|9606                         NaN      NaN   \n",
       "ncbi_homo_sapiens.gene_info|1                            NaN      NaN   \n",
       "ncbi_homo_sapiens.gene_info|A1BG                         NaN      NaN   \n",
       "ncbi_homo_sapiens.gene_info|MIM:138670                   NaN      NaN   \n",
       "ncbi_homo_sapiens.gene_info|HGNC:HGNC:5                  NaN      NaN   \n",
       "\n",
       "                                         accesssion  taxon id  human  \\\n",
       "homo_sapiens.gff|100287102                      NaN       NaN    NaN   \n",
       "homo_sapiens.gff|HGNC:37102                     NaN       NaN    NaN   \n",
       "homo_sapiens.gff|DDX11L1                        NaN       NaN    NaN   \n",
       "flybase_d_melanogaster.gaf|FBgn0043467          NaN       NaN    NaN   \n",
       "flybase_d_melanogaster.gaf|GO:0048149           1.0       NaN    NaN   \n",
       "ncbi_homo_sapiens.gene_info|9606                NaN       3.0    5.0   \n",
       "ncbi_homo_sapiens.gene_info|1                   NaN       NaN    NaN   \n",
       "ncbi_homo_sapiens.gene_info|A1BG                NaN       NaN    NaN   \n",
       "ncbi_homo_sapiens.gene_info|MIM:138670          NaN       NaN    NaN   \n",
       "ncbi_homo_sapiens.gene_info|HGNC:HGNC:5         NaN       NaN    NaN   \n",
       "\n",
       "                                         chromosome  BRCA1_HUMAN  \\\n",
       "homo_sapiens.gff|100287102                      NaN          NaN   \n",
       "homo_sapiens.gff|HGNC:37102                     NaN          NaN   \n",
       "homo_sapiens.gff|DDX11L1                        NaN          NaN   \n",
       "flybase_d_melanogaster.gaf|FBgn0043467          NaN          NaN   \n",
       "flybase_d_melanogaster.gaf|GO:0048149           NaN          NaN   \n",
       "ncbi_homo_sapiens.gene_info|9606                NaN          NaN   \n",
       "ncbi_homo_sapiens.gene_info|1                   3.0          NaN   \n",
       "ncbi_homo_sapiens.gene_info|A1BG                NaN          1.0   \n",
       "ncbi_homo_sapiens.gene_info|MIM:138670          2.0          NaN   \n",
       "ncbi_homo_sapiens.gene_info|HGNC:HGNC:5         2.0          NaN   \n",
       "\n",
       "                                         links to other dbs  disease  \n",
       "homo_sapiens.gff|100287102                              NaN      NaN  \n",
       "homo_sapiens.gff|HGNC:37102                             NaN      NaN  \n",
       "homo_sapiens.gff|DDX11L1                                NaN      NaN  \n",
       "flybase_d_melanogaster.gaf|FBgn0043467                  NaN      NaN  \n",
       "flybase_d_melanogaster.gaf|GO:0048149                   NaN      NaN  \n",
       "ncbi_homo_sapiens.gene_info|9606                        NaN      NaN  \n",
       "ncbi_homo_sapiens.gene_info|1                           NaN      NaN  \n",
       "ncbi_homo_sapiens.gene_info|A1BG                        NaN      NaN  \n",
       "ncbi_homo_sapiens.gene_info|MIM:138670                  1.0      1.0  \n",
       "ncbi_homo_sapiens.gene_info|HGNC:HGNC:5                 NaN      NaN  \n",
       "\n",
       "[10 rows x 25 columns]"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "results = results.transpose()\n",
    "results.head(10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "homo_sapiens.gff|100287102                  NaN\n",
       "homo_sapiens.gff|HGNC:37102                 NaN\n",
       "homo_sapiens.gff|DDX11L1                    NaN\n",
       "flybase_d_melanogaster.gaf|FBgn0043467      1.0\n",
       "flybase_d_melanogaster.gaf|GO:0048149      18.0\n",
       "ncbi_homo_sapiens.gene_info|9606            NaN\n",
       "ncbi_homo_sapiens.gene_info|1               NaN\n",
       "ncbi_homo_sapiens.gene_info|A1BG            NaN\n",
       "ncbi_homo_sapiens.gene_info|MIM:138670      NaN\n",
       "ncbi_homo_sapiens.gene_info|HGNC:HGNC:5     NaN\n",
       "Name: GO Term, dtype: float64"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# GO Term is quite consistent in the matching\n",
    "results[\"GO Term\"]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "homo_sapiens.gff|100287102                  1.0\n",
       "homo_sapiens.gff|HGNC:37102                 4.0\n",
       "homo_sapiens.gff|DDX11L1                    8.0\n",
       "flybase_d_melanogaster.gaf|FBgn0043467      3.0\n",
       "flybase_d_melanogaster.gaf|GO:0048149       NaN\n",
       "ncbi_homo_sapiens.gene_info|9606            2.0\n",
       "ncbi_homo_sapiens.gene_info|1               1.0\n",
       "ncbi_homo_sapiens.gene_info|A1BG           15.0\n",
       "ncbi_homo_sapiens.gene_info|MIM:138670      1.0\n",
       "ncbi_homo_sapiens.gene_info|HGNC:HGNC:5     1.0\n",
       "Name: symbol, dtype: float64"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Symbol is the next best and is already a mess - it's used in a lot of different places.\n",
    "results[\"symbol\"]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Variation within a given term\n",
    "When a term is matched to cards, how many different options are given in total, and per person?"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "                                                                                     terms  \\\n",
      "homo_sapiens.gff|100287102               [accession, identifier, gene, dataset/database...   \n",
      "homo_sapiens.gff|HGNC:37102              [dataset/database, h. sapiens, gene, accession...   \n",
      "homo_sapiens.gff|DDX11L1                 [Q9H4C3_HUMAN, gene, name, symbol, identifier,...   \n",
      "flybase_d_melanogaster.gaf|FBgn0043467   [GO Term, accession, gene, identifier, name, s...   \n",
      "flybase_d_melanogaster.gaf|GO:0048149    [Chromosome location, GO Term, accesssion, dat...   \n",
      "ncbi_homo_sapiens.gene_info|9606         [h. sapiens, human, identifier, taxon id, orga...   \n",
      "ncbi_homo_sapiens.gene_info|1            [chromosome, gene, identifier, protein, name, ...   \n",
      "ncbi_homo_sapiens.gene_info|A1BG         [BRCA1_HUMAN, gene, name, symbol, organism, id...   \n",
      "ncbi_homo_sapiens.gene_info|MIM:138670   [accession, chromosome, dataset/database, iden...   \n",
      "ncbi_homo_sapiens.gene_info|HGNC:HGNC:5  [chromosome, database, accession, identifier, ...   \n",
      "\n",
      "                                        count  \n",
      "homo_sapiens.gff|100287102                  8  \n",
      "homo_sapiens.gff|HGNC:37102                10  \n",
      "homo_sapiens.gff|DDX11L1                    8  \n",
      "flybase_d_melanogaster.gaf|FBgn0043467     12  \n",
      "flybase_d_melanogaster.gaf|GO:0048149       8  \n",
      "ncbi_homo_sapiens.gene_info|9606            8  \n",
      "ncbi_homo_sapiens.gene_info|1               7  \n",
      "ncbi_homo_sapiens.gene_info|A1BG            8  \n",
      "ncbi_homo_sapiens.gene_info|MIM:138670     10  \n",
      "ncbi_homo_sapiens.gene_info|HGNC:HGNC:5     9  \n",
      "                                                                                     terms  \\\n",
      "homo_sapiens.gff|100287102               [[accession, identifier], [gene], [gene, datas...   \n",
      "homo_sapiens.gff|HGNC:37102              [[dataset/database], [dataset/database, h. sap...   \n",
      "homo_sapiens.gff|DDX11L1                 [[Q9H4C3_HUMAN], [gene], [gene, name], [gene, ...   \n",
      "flybase_d_melanogaster.gaf|FBgn0043467   [[GO Term], [accession, gene, identifier], [ac...   \n",
      "flybase_d_melanogaster.gaf|GO:0048149    [[Chromosome location], [GO Term], [GO Term, a...   \n",
      "ncbi_homo_sapiens.gene_info|9606         [[h. sapiens], [human], [identifier], [identif...   \n",
      "ncbi_homo_sapiens.gene_info|1            [[chromosome], [gene], [gene, identifier], [ge...   \n",
      "ncbi_homo_sapiens.gene_info|A1BG         [[BRCA1_HUMAN, gene], [gene, name, symbol], [g...   \n",
      "ncbi_homo_sapiens.gene_info|MIM:138670   [[accession], [chromosome], [chromosome, datas...   \n",
      "ncbi_homo_sapiens.gene_info|HGNC:HGNC:5  [[chromosome], [chromosome, database, accessio...   \n",
      "\n",
      "                                        count  \n",
      "homo_sapiens.gff|100287102                  8  \n",
      "homo_sapiens.gff|HGNC:37102                13  \n",
      "homo_sapiens.gff|DDX11L1                   12  \n",
      "flybase_d_melanogaster.gaf|FBgn0043467     13  \n",
      "flybase_d_melanogaster.gaf|GO:0048149       7  \n",
      "ncbi_homo_sapiens.gene_info|9606            9  \n",
      "ncbi_homo_sapiens.gene_info|1               6  \n",
      "ncbi_homo_sapiens.gene_info|A1BG           10  \n",
      "ncbi_homo_sapiens.gene_info|MIM:138670     10  \n",
      "ncbi_homo_sapiens.gene_info|HGNC:HGNC:5     6  \n"
     ]
    }
   ],
   "source": [
    "#\"split a multi choice answer, returning a series with a column \n",
    "# header and the responses with the header stripped out.\"\n",
    "def count_variation( a_series ):\n",
    "    # variable to store the final response in as we build it\n",
    "    split_answer = a_series.str.split(\"|\")\n",
    "    split_answer = split_answer.sort_values()\n",
    "    \n",
    "    term_combinations = list()\n",
    "    for terms_list in split_answer:\n",
    "        if isinstance(terms_list, list):\n",
    "            #Record unique combinations of terms\n",
    "            if terms_list not in term_combinations:\n",
    "                term_combinations.append(terms_list)\n",
    "    return term_combinations\n",
    "\n",
    "#\"split a multi choice answer, returning a series with a column \n",
    "# header and the responses with the header stripped out.\"\n",
    "def count_unique_terms( a_series ):\n",
    "    # variable to store the final response in as we build it\n",
    "    split_answer = a_series.str.split(\"|\")\n",
    "    split_answer = split_answer.sort_values()\n",
    "    \n",
    "    unique_terms = list()\n",
    "    for terms_list in split_answer:\n",
    "        if isinstance(terms_list, list):\n",
    "            #Record unique terms regardless of combo\n",
    "            for individual_term in terms_list:\n",
    "                if individual_term not in unique_terms:\n",
    "                    unique_terms.append(individual_term)            \n",
    "\n",
    "    return unique_terms\n",
    "\n",
    "term_combinations = {}\n",
    "unique_results_per_term = {}\n",
    "for i in fm: \n",
    "    if (fm[i].dtype == np.object):\n",
    "        \n",
    "        #column name\n",
    "        col = fm[i].name\n",
    "        \n",
    "        #calculate number of different term combos per matched field\n",
    "        term_combos = count_variation(fm[i])\n",
    "        term_combinations[col] = {\"terms\": term_combos, \"count\": len(term_combos)}\n",
    "        \n",
    "        #calculate number of different unique terms per field\n",
    "        unique_terms = count_unique_terms(fm[i])\n",
    "        unique_results_per_term[col] = {\"terms\": unique_terms, \"count\": len(unique_terms)}\n",
    "\n",
    "\n",
    "print(pd.DataFrame(unique_results_per_term).transpose())\n",
    "print(pd.DataFrame(term_combinations).transpose())"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Entry points\n",
    "We asked participants if they were thinking of doing research or an experiment themselves, which card would they consider the entry point to this set of items?"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>term_selected</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>homologue</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>accession|identifier|DOI|pubmed ID</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>disease|flymine|ensembl|uniprot</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>gene</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>identifier|dataset|database|publication</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                             term_selected\n",
       "0                                homologue\n",
       "1       accession|identifier|DOI|pubmed ID\n",
       "2          disease|flymine|ensembl|uniprot\n",
       "3                                     gene\n",
       "4  identifier|dataset|database|publication"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#Data prep to load the data and subset for usage\n",
    "entry_points = pd.read_csv(\"data/entry_point.csv\")\n",
    " #subset the data to just the columns that answer this question\n",
    "entry_terms = entry_points.iloc[:,1:2]\n",
    "#let's take a peek at the data\n",
    "entry_terms.head(5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "                 term_selected\n",
      "gene                         7\n",
      "identifier                   5\n",
      "publication                  4\n",
      "database                     3\n",
      "protein                      2\n",
      "disease                      2\n",
      "organism                     2\n",
      "accession                    2\n",
      "pubmed ID                    1\n",
      "pathway                      1\n",
      "name                         1\n",
      "DOI                          1\n",
      "homologue                    1\n",
      "flymine                      1\n",
      "ensembl                      1\n",
      "dataset                      1\n",
      "d. melanogaster              1\n",
      "chromosome                   1\n",
      "uniprot                      1\n"
     ]
    }
   ],
   "source": [
    "#\"Split out responses to the entry point question and return a count of how many\n",
    "# times each term is used\"\n",
    "def count_entry_point_unique_answers( a_series ):\n",
    "    # variable to store the final response in as we build it\n",
    "    tokenise = list()\n",
    "    split_answer = pd.DataFrame(a_series.str.split(\"|\", expand=True))\n",
    "    for i in split_answer:   \n",
    "        tokenise.extend(split_answer[i])\n",
    "        \n",
    "    response = {}\n",
    "    for i in tokenise:\n",
    "        if not pd.isnull(i):  \n",
    "            if i in response:\n",
    "                response[i] = response[i]+1\n",
    "            else:\n",
    "                response[i] = 1\n",
    "    return response\n",
    "\n",
    "\n",
    "#we'll store unique counts in this var\n",
    "result_counts = {}\n",
    "\n",
    "# split up the terms and count them\n",
    "for i in entry_terms: \n",
    "    if (entry_terms[i].dtype == np.object):\n",
    "        result_counts[entry_terms[i].name] = count_entry_point_unique_answers(entry_terms[i])\n",
    "\n",
    "entry_df = pd.DataFrame(result_counts)\n",
    "\n",
    "#make sure it's in order from big to small\n",
    "entry_df.sort_values(by=['term_selected'], inplace=True, ascending=False)\n",
    "\n",
    "#show the results to the world as a nice lil table\n",
    "print(entry_df)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Entry point types\n",
    "Counts the number of different response types (mostly information or bio centric)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "#subset the data \n",
    "entry_point_type = entry_points[[\"participant_id\",\"bio_or_info\"]]\n",
    "#we'll store unique counts in this var\n",
    "entry_point_type_result_counts = {}\n",
    "\n",
    "# split up the terms and count them\n",
    "for i in entry_point_type: \n",
    "    if (entry_point_type[i].dtype == np.object):\n",
    "        entry_point_type_result_counts[entry_point_type[i].name] = count_entry_point_unique_answers(entry_point_type[i])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>info</th>\n",
       "      <th>bio</th>\n",
       "      <th>both</th>\n",
       "      <th>unclear</th>\n",
       "      <th>money</th>\n",
       "      <th>circuits</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>bio_or_info</th>\n",
       "      <td>12</td>\n",
       "      <td>5</td>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "             info  bio  both  unclear  money  circuits\n",
       "bio_or_info    12    5     2        1      1         1"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Let's print each type as a nice table, sorted big to small. \n",
    "result_types = pd.DataFrame(entry_point_type_result_counts[\"bio_or_info\"], index=[0])\n",
    "result_types.rename(index = {0:'bio_or_info'}, inplace = True)\n",
    "result_types.sort_values(by=[\"bio_or_info\"], inplace=True, axis=1, ascending=False)\n",
    "result_types\n"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}