replace_wt_w_sy_count_python2.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import os\n",
    "import numpy as np\n",
    "from enrich2.variant import WILD_TYPE_VARIANT\n",
    "import glob\n",
    "#reference:\n",
    "#https://pandas.pydata.org/pandas-docs/stable/user_guide/io.html#hdf5-pytables"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "#If you get a dependency error that tables is not installed, try sudo apt-get install python3-tables, that solved it for me"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "results_path = \"/home/henrike/Documents/PD_AS/projects/Sofie_Mave/results/enrich_full_dhfr/\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['/home/henrike/Documents/PD_AS/projects/Sofie_Mave/results/enrich_full_dhfr/hdf5_manipulation_test/2B0_lib.h5',\n",
       " '/home/henrike/Documents/PD_AS/projects/Sofie_Mave/results/enrich_full_dhfr/hdf5_manipulation_test/2A3_lib.h5',\n",
       " '/home/henrike/Documents/PD_AS/projects/Sofie_Mave/results/enrich_full_dhfr/hdf5_manipulation_test/2B2_lib.h5',\n",
       " '/home/henrike/Documents/PD_AS/projects/Sofie_Mave/results/enrich_full_dhfr/hdf5_manipulation_test/2A0_lib.h5',\n",
       " '/home/henrike/Documents/PD_AS/projects/Sofie_Mave/results/enrich_full_dhfr/hdf5_manipulation_test/2C2_lib.h5',\n",
       " '/home/henrike/Documents/PD_AS/projects/Sofie_Mave/results/enrich_full_dhfr/hdf5_manipulation_test/2B1_lib.h5',\n",
       " '/home/henrike/Documents/PD_AS/projects/Sofie_Mave/results/enrich_full_dhfr/hdf5_manipulation_test/2B3_lib.h5',\n",
       " '/home/henrike/Documents/PD_AS/projects/Sofie_Mave/results/enrich_full_dhfr/hdf5_manipulation_test/2C1_lib.h5',\n",
       " '/home/henrike/Documents/PD_AS/projects/Sofie_Mave/results/enrich_full_dhfr/hdf5_manipulation_test/2A2_lib.h5',\n",
       " '/home/henrike/Documents/PD_AS/projects/Sofie_Mave/results/enrich_full_dhfr/hdf5_manipulation_test/2A1_lib.h5',\n",
       " '/home/henrike/Documents/PD_AS/projects/Sofie_Mave/results/enrich_full_dhfr/hdf5_manipulation_test/2C0_lib.h5',\n",
       " '/home/henrike/Documents/PD_AS/projects/Sofie_Mave/results/enrich_full_dhfr/hdf5_manipulation_test/2C3_lib.h5']"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#list all hdf5 files\n",
    "listing = glob.glob(os.path.join(results_path, 'hdf5_manipulation_test', '*h5'))\n",
    "listing"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['/home/henrike/Documents/PD_AS/projects/Sofie_Mave/results/enrich_full_dhfr/tile3_all_reps_minvarcount10_corr_sy',\n",
       " '/home/henrike/Documents/PD_AS/projects/Sofie_Mave/results/enrich_full_dhfr/tile5_all_reps_minvarcount10_corr_sy',\n",
       " '/home/henrike/Documents/PD_AS/projects/Sofie_Mave/results/enrich_full_dhfr/tile4_all_reps_minvarcount10_corr_sy',\n",
       " '/home/henrike/Documents/PD_AS/projects/Sofie_Mave/results/enrich_full_dhfr/tile1_all_reps_minvarcount10_corr_sy',\n",
       " '/home/henrike/Documents/PD_AS/projects/Sofie_Mave/results/enrich_full_dhfr/tile2_all_reps_minvarcount10_corr_sy']"
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "folders = glob.glob(os.path.join(results_path, 'tile*_all_reps_minvarcount10_corr_sy'))\n",
    "folders"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "/home/henrike/Documents/PD_AS/projects/Sofie_Mave/results/enrich_full_dhfr/tile3_all_reps_minvarcount10_corr_sy/3C0_lib.h5\n",
      "('sy:', 8327)\n",
      "/home/henrike/Documents/PD_AS/projects/Sofie_Mave/results/enrich_full_dhfr/tile3_all_reps_minvarcount10_corr_sy/3C2_lib.h5\n",
      "('sy:', 10170)\n",
      "/home/henrike/Documents/PD_AS/projects/Sofie_Mave/results/enrich_full_dhfr/tile3_all_reps_minvarcount10_corr_sy/3B0_lib.h5\n",
      "('sy:', 8946)\n",
      "/home/henrike/Documents/PD_AS/projects/Sofie_Mave/results/enrich_full_dhfr/tile3_all_reps_minvarcount10_corr_sy/3A0_lib.h5\n",
      "('sy:', 9995)\n",
      "/home/henrike/Documents/PD_AS/projects/Sofie_Mave/results/enrich_full_dhfr/tile3_all_reps_minvarcount10_corr_sy/3B3_lib.h5\n",
      "('sy:', 17304)\n",
      "/home/henrike/Documents/PD_AS/projects/Sofie_Mave/results/enrich_full_dhfr/tile3_all_reps_minvarcount10_corr_sy/3C3_lib.h5\n",
      "('sy:', 7061)\n",
      "/home/henrike/Documents/PD_AS/projects/Sofie_Mave/results/enrich_full_dhfr/tile3_all_reps_minvarcount10_corr_sy/3B2_lib.h5\n",
      "('sy:', 12684)\n",
      "/home/henrike/Documents/PD_AS/projects/Sofie_Mave/results/enrich_full_dhfr/tile3_all_reps_minvarcount10_corr_sy/3A3_lib.h5\n",
      "('sy:', 6212)\n",
      "/home/henrike/Documents/PD_AS/projects/Sofie_Mave/results/enrich_full_dhfr/tile3_all_reps_minvarcount10_corr_sy/3B1_lib.h5\n",
      "('sy:', 716)\n",
      "/home/henrike/Documents/PD_AS/projects/Sofie_Mave/results/enrich_full_dhfr/tile3_all_reps_minvarcount10_corr_sy/3A1_lib.h5\n",
      "('sy:', 4163)\n",
      "/home/henrike/Documents/PD_AS/projects/Sofie_Mave/results/enrich_full_dhfr/tile3_all_reps_minvarcount10_corr_sy/3A2_lib.h5\n",
      "('sy:', 4522)\n",
      "/home/henrike/Documents/PD_AS/projects/Sofie_Mave/results/enrich_full_dhfr/tile3_all_reps_minvarcount10_corr_sy/3C1_lib.h5\n",
      "('sy:', 3249)\n",
      "/home/henrike/Documents/PD_AS/projects/Sofie_Mave/results/enrich_full_dhfr/tile5_all_reps_minvarcount10_corr_sy/5C3_lib.h5\n",
      "('sy:', 21140)\n",
      "/home/henrike/Documents/PD_AS/projects/Sofie_Mave/results/enrich_full_dhfr/tile5_all_reps_minvarcount10_corr_sy/5A0_lib.h5\n",
      "('sy:', 18834)\n",
      "/home/henrike/Documents/PD_AS/projects/Sofie_Mave/results/enrich_full_dhfr/tile5_all_reps_minvarcount10_corr_sy/5A2_lib.h5\n",
      "('sy:', 11617)\n",
      "/home/henrike/Documents/PD_AS/projects/Sofie_Mave/results/enrich_full_dhfr/tile5_all_reps_minvarcount10_corr_sy/5C2_lib.h5\n",
      "('sy:', 23822)\n",
      "/home/henrike/Documents/PD_AS/projects/Sofie_Mave/results/enrich_full_dhfr/tile5_all_reps_minvarcount10_corr_sy/5C1_lib.h5\n",
      "('sy:', 22399)\n",
      "/home/henrike/Documents/PD_AS/projects/Sofie_Mave/results/enrich_full_dhfr/tile5_all_reps_minvarcount10_corr_sy/5C0_lib.h5\n",
      "('sy:', 20517)\n",
      "/home/henrike/Documents/PD_AS/projects/Sofie_Mave/results/enrich_full_dhfr/tile5_all_reps_minvarcount10_corr_sy/5B3_lib.h5\n",
      "('sy:', 20048)\n",
      "/home/henrike/Documents/PD_AS/projects/Sofie_Mave/results/enrich_full_dhfr/tile5_all_reps_minvarcount10_corr_sy/5A1_lib.h5\n",
      "('sy:', 26457)\n",
      "/home/henrike/Documents/PD_AS/projects/Sofie_Mave/results/enrich_full_dhfr/tile5_all_reps_minvarcount10_corr_sy/5B0_lib.h5\n",
      "('sy:', 23154)\n",
      "/home/henrike/Documents/PD_AS/projects/Sofie_Mave/results/enrich_full_dhfr/tile5_all_reps_minvarcount10_corr_sy/5A3_lib.h5\n",
      "('sy:', 24222)\n",
      "/home/henrike/Documents/PD_AS/projects/Sofie_Mave/results/enrich_full_dhfr/tile5_all_reps_minvarcount10_corr_sy/5B1_lib.h5\n",
      "('sy:', 22694)\n",
      "/home/henrike/Documents/PD_AS/projects/Sofie_Mave/results/enrich_full_dhfr/tile5_all_reps_minvarcount10_corr_sy/5B2_lib.h5\n",
      "('sy:', 21268)\n",
      "/home/henrike/Documents/PD_AS/projects/Sofie_Mave/results/enrich_full_dhfr/tile4_all_reps_minvarcount10_corr_sy/4A3_lib.h5\n",
      "('sy:', 6978)\n",
      "/home/henrike/Documents/PD_AS/projects/Sofie_Mave/results/enrich_full_dhfr/tile4_all_reps_minvarcount10_corr_sy/4B2_lib.h5\n",
      "('sy:', 7125)\n",
      "/home/henrike/Documents/PD_AS/projects/Sofie_Mave/results/enrich_full_dhfr/tile4_all_reps_minvarcount10_corr_sy/4C3_lib.h5\n",
      "('sy:', 8806)\n",
      "/home/henrike/Documents/PD_AS/projects/Sofie_Mave/results/enrich_full_dhfr/tile4_all_reps_minvarcount10_corr_sy/4A1_lib.h5\n",
      "('sy:', 2464)\n",
      "/home/henrike/Documents/PD_AS/projects/Sofie_Mave/results/enrich_full_dhfr/tile4_all_reps_minvarcount10_corr_sy/4B3_lib.h5\n",
      "('sy:', 6951)\n",
      "/home/henrike/Documents/PD_AS/projects/Sofie_Mave/results/enrich_full_dhfr/tile4_all_reps_minvarcount10_corr_sy/4C1_lib.h5\n",
      "('sy:', 5076)\n",
      "/home/henrike/Documents/PD_AS/projects/Sofie_Mave/results/enrich_full_dhfr/tile4_all_reps_minvarcount10_corr_sy/4B1_lib.h5\n",
      "('sy:', 2436)\n",
      "/home/henrike/Documents/PD_AS/projects/Sofie_Mave/results/enrich_full_dhfr/tile4_all_reps_minvarcount10_corr_sy/4C2_lib.h5\n",
      "('sy:', 11162)\n",
      "/home/henrike/Documents/PD_AS/projects/Sofie_Mave/results/enrich_full_dhfr/tile4_all_reps_minvarcount10_corr_sy/4C0_lib.h5\n",
      "('sy:', 5076)\n",
      "/home/henrike/Documents/PD_AS/projects/Sofie_Mave/results/enrich_full_dhfr/tile4_all_reps_minvarcount10_corr_sy/4A2_lib.h5\n",
      "('sy:', 8760)\n",
      "/home/henrike/Documents/PD_AS/projects/Sofie_Mave/results/enrich_full_dhfr/tile4_all_reps_minvarcount10_corr_sy/4A0_lib.h5\n",
      "('sy:', 9665)\n",
      "/home/henrike/Documents/PD_AS/projects/Sofie_Mave/results/enrich_full_dhfr/tile4_all_reps_minvarcount10_corr_sy/4B0_lib.h5\n",
      "('sy:', 3860)\n",
      "/home/henrike/Documents/PD_AS/projects/Sofie_Mave/results/enrich_full_dhfr/tile1_all_reps_minvarcount10_corr_sy/1B2_lib.h5\n",
      "('sy:', 5787)\n",
      "/home/henrike/Documents/PD_AS/projects/Sofie_Mave/results/enrich_full_dhfr/tile1_all_reps_minvarcount10_corr_sy/1A3_lib.h5\n",
      "('sy:', 2535)\n",
      "/home/henrike/Documents/PD_AS/projects/Sofie_Mave/results/enrich_full_dhfr/tile1_all_reps_minvarcount10_corr_sy/1C3_lib.h5\n",
      "('sy:', 2805)\n",
      "/home/henrike/Documents/PD_AS/projects/Sofie_Mave/results/enrich_full_dhfr/tile1_all_reps_minvarcount10_corr_sy/1A1_lib.h5\n",
      "('sy:', 2717)\n",
      "/home/henrike/Documents/PD_AS/projects/Sofie_Mave/results/enrich_full_dhfr/tile1_all_reps_minvarcount10_corr_sy/1A0_alllanes_lib.h5\n",
      "('sy:', 12336)\n",
      "/home/henrike/Documents/PD_AS/projects/Sofie_Mave/results/enrich_full_dhfr/tile1_all_reps_minvarcount10_corr_sy/1C0_lib.h5\n",
      "('sy:', 12935)\n",
      "/home/henrike/Documents/PD_AS/projects/Sofie_Mave/results/enrich_full_dhfr/tile1_all_reps_minvarcount10_corr_sy/1B1_lib.h5\n",
      "('sy:', 3837)\n",
      "/home/henrike/Documents/PD_AS/projects/Sofie_Mave/results/enrich_full_dhfr/tile1_all_reps_minvarcount10_corr_sy/1A2_lib.h5\n",
      "('sy:', 5165)\n",
      "/home/henrike/Documents/PD_AS/projects/Sofie_Mave/results/enrich_full_dhfr/tile1_all_reps_minvarcount10_corr_sy/1B3_lib.h5\n",
      "('sy:', 3740)\n",
      "/home/henrike/Documents/PD_AS/projects/Sofie_Mave/results/enrich_full_dhfr/tile1_all_reps_minvarcount10_corr_sy/1C1_lib.h5\n",
      "('sy:', 8986)\n",
      "/home/henrike/Documents/PD_AS/projects/Sofie_Mave/results/enrich_full_dhfr/tile1_all_reps_minvarcount10_corr_sy/1B0_lib.h5\n",
      "('sy:', 13342)\n",
      "/home/henrike/Documents/PD_AS/projects/Sofie_Mave/results/enrich_full_dhfr/tile1_all_reps_minvarcount10_corr_sy/1C2_lib.h5\n",
      "('sy:', 6412)\n",
      "/home/henrike/Documents/PD_AS/projects/Sofie_Mave/results/enrich_full_dhfr/tile2_all_reps_minvarcount10_corr_sy/2B0_lib.h5\n",
      "('sy:', 5739)\n",
      "/home/henrike/Documents/PD_AS/projects/Sofie_Mave/results/enrich_full_dhfr/tile2_all_reps_minvarcount10_corr_sy/2A3_lib.h5\n",
      "('sy:', 3513)\n",
      "/home/henrike/Documents/PD_AS/projects/Sofie_Mave/results/enrich_full_dhfr/tile2_all_reps_minvarcount10_corr_sy/2B2_lib.h5\n",
      "('sy:', 24116)\n",
      "/home/henrike/Documents/PD_AS/projects/Sofie_Mave/results/enrich_full_dhfr/tile2_all_reps_minvarcount10_corr_sy/2A0_lib.h5\n",
      "('sy:', 6839)\n",
      "/home/henrike/Documents/PD_AS/projects/Sofie_Mave/results/enrich_full_dhfr/tile2_all_reps_minvarcount10_corr_sy/2C2_lib.h5\n",
      "('sy:', 4250)\n",
      "/home/henrike/Documents/PD_AS/projects/Sofie_Mave/results/enrich_full_dhfr/tile2_all_reps_minvarcount10_corr_sy/2B1_lib.h5\n",
      "('sy:', 5555)\n",
      "/home/henrike/Documents/PD_AS/projects/Sofie_Mave/results/enrich_full_dhfr/tile2_all_reps_minvarcount10_corr_sy/2B3_lib.h5\n",
      "('sy:', 3117)\n",
      "/home/henrike/Documents/PD_AS/projects/Sofie_Mave/results/enrich_full_dhfr/tile2_all_reps_minvarcount10_corr_sy/2C1_lib.h5\n",
      "('sy:', 6095)\n",
      "/home/henrike/Documents/PD_AS/projects/Sofie_Mave/results/enrich_full_dhfr/tile2_all_reps_minvarcount10_corr_sy/2A2_lib.h5\n",
      "('sy:', 5237)\n",
      "/home/henrike/Documents/PD_AS/projects/Sofie_Mave/results/enrich_full_dhfr/tile2_all_reps_minvarcount10_corr_sy/2A1_lib.h5\n",
      "('sy:', 4770)\n",
      "/home/henrike/Documents/PD_AS/projects/Sofie_Mave/results/enrich_full_dhfr/tile2_all_reps_minvarcount10_corr_sy/2C0_lib.h5\n",
      "('sy:', 5041)\n",
      "/home/henrike/Documents/PD_AS/projects/Sofie_Mave/results/enrich_full_dhfr/tile2_all_reps_minvarcount10_corr_sy/2C3_lib.h5\n",
      "('sy:', 3393)\n"
     ]
    }
   ],
   "source": [
    "for folder in folders:\n",
    "    listing = glob.glob(os.path.join(folder, '*h5'))\n",
    "    for h_file in listing:\n",
    "        print(h_file)\n",
    "        my_store = pd.HDFStore(h_file)\n",
    "\n",
    "        #get the synonymous to WT counts\n",
    "        syn_counts = int(my_store.select('/main/synonymous/counts', where=\"index in ['_sy']\")['count'])\n",
    "        print('sy:',syn_counts)\n",
    "\n",
    "        #rm wt counts from /main/variants/counts and /main/synonymous/counts\n",
    "        my_store.remove('/main/synonymous/counts', where=\"index in ['_wt']\")\n",
    "        my_store.remove('/main/variants/counts', where=\"index in ['_wt']\")\n",
    "        my_store.remove('/raw/variants/counts', where=\"index in ['_wt']\")\n",
    "\n",
    "        #re-add a wt var with the syn_counts\n",
    "        my_df = pd.DataFrame(data=syn_counts, index=['_wt'], columns=['count'], dtype=np.int32, copy=False)\n",
    "        my_store.append('/main/synonymous/counts', my_df)\n",
    "        my_store.append('/main/variants/counts', my_df)\n",
    "        my_store.append('/raw/variants/counts', my_df)\n",
    "\n",
    "        #close the storage to save\n",
    "        my_store.close()\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#verify"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [],
   "source": [
    "my_store = pd.HDFStore('/home/henrike/Documents/PD_AS/projects/Sofie_Mave/results/enrich_full_dhfr/tile1_all_reps_minvarcount10_corr_sy/1A1_lib.h5')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>count</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>c.68T&gt;G (p.Leu23Trp)</th>\n",
       "      <td>8939</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>c.68T&gt;A (p.Leu23Tyr), c.69G&gt;T (p.Leu23Tyr)</th>\n",
       "      <td>8639</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>c.67T&gt;G (p.Leu23Glu), c.68T&gt;A (p.Leu23Glu), c.69G&gt;A (p.Leu23Glu)</th>\n",
       "      <td>4561</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>c.94T&gt;G (p.Phe32Asp), c.95T&gt;A (p.Phe32Asp)</th>\n",
       "      <td>2830</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>c.22A&gt;T (p.Ile8Phe), c.24C&gt;T (p.Ile8Phe)</th>\n",
       "      <td>2611</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                    count\n",
       "c.68T>G (p.Leu23Trp)                                 8939\n",
       "c.68T>A (p.Leu23Tyr), c.69G>T (p.Leu23Tyr)           8639\n",
       "c.67T>G (p.Leu23Glu), c.68T>A (p.Leu23Glu), c.6...   4561\n",
       "c.94T>G (p.Phe32Asp), c.95T>A (p.Phe32Asp)           2830\n",
       "c.22A>T (p.Ile8Phe), c.24C>T (p.Ile8Phe)             2611"
      ]
     },
     "execution_count": 23,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "my_store[\"/main/variants/counts\"].head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>count</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>_wt</th>\n",
       "      <td>2717</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "     count\n",
       "_wt   2717"
      ]
     },
     "execution_count": 24,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "my_store.select('/main/synonymous/counts', where=\"index in ['_wt']\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>count</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>_sy</th>\n",
       "      <td>2717</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "     count\n",
       "_sy   2717"
      ]
     },
     "execution_count": 25,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "my_store.select('/main/synonymous/counts', where=\"index in ['_sy']\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>count</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>_wt</th>\n",
       "      <td>2717</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "     count\n",
       "_wt   2717"
      ]
     },
     "execution_count": 26,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "my_store.select('/main/variants/counts', where=\"index in ['_wt']\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['/raw/filter',\n",
       " '/raw/overlap_mismatches',\n",
       " '/raw/variants/counts',\n",
       " '/main/variants/counts',\n",
       " '/main/synonymous/counts']"
      ]
     },
     "execution_count": 27,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "my_store.keys()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>count</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>_wt</th>\n",
       "      <td>2717</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "     count\n",
       "_wt   2717"
      ]
     },
     "execution_count": 28,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "my_store.select('/raw/variants/counts', where=\"index in ['_wt']\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>resolved</th>\n",
       "      <th>unresolved</th>\n",
       "      <th>first</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>28</th>\n",
       "      <td>11392</td>\n",
       "      <td>4562</td>\n",
       "      <td>15954</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>29</th>\n",
       "      <td>9526</td>\n",
       "      <td>3532</td>\n",
       "      <td>2637</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>30</th>\n",
       "      <td>8458</td>\n",
       "      <td>3931</td>\n",
       "      <td>4831</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>31</th>\n",
       "      <td>10815</td>\n",
       "      <td>6861</td>\n",
       "      <td>4324</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>32</th>\n",
       "      <td>11410</td>\n",
       "      <td>4774</td>\n",
       "      <td>2179</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "    resolved  unresolved  first\n",
       "28     11392        4562  15954\n",
       "29      9526        3532   2637\n",
       "30      8458        3931   4831\n",
       "31     10815        6861   4324\n",
       "32     11410        4774   2179"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "my_store['/raw/overlap_mismatches'].head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#old: trying it out on one storage"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#open the hdf5 storage\n",
    "#if we open the experiment level one tile2_all_reps_exp.h5 the counts in /main/variants/counts are all NA\n",
    "#this is a known bug that was reported before.\n",
    "#I might have to change the wt counts for each selection anyway. Or for each lib?\n",
    "#Alan said:\n",
    "#1. Delete the selection and experiment HDF5 files.\n",
    "#2. For each SeqLib HDF5 file, replace the '_wt' count with the '_sy' count in the /main/counts table\n",
    "my_store = pd.HDFStore(os.path.join(results_path, \"hdf5_manipulation_test/2B0_lib.h5\"))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "my_store.keys()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#this shows the /main/variants/counts table\n",
    "#an equivalent method is my_store.get('/main/variants/counts')\n",
    "my_store[\"/main/variants/counts\"].head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "my_store[\"/main/synonymous/counts\"].head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#does not work :(\n",
    "#my_store[\"/main/variants/counts\"]['_wt']\n",
    "#neither does this\n",
    "#my_store[\"/main/synonymous/counts/_wt\"]\n",
    "#this type of selection using where works though!\n",
    "c = my_store.select('/main/synonymous/counts', where=\"index in ['_sy']\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#I'll try to remove and re-add the _wt count\n",
    "my_store.remove('/main/synonymous/counts', where=\"index in ['_wt']\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#display result\n",
    "my_store[\"/main/synonymous/counts\"].head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#now re-add the wt var with the sy count\n",
    "my_df = pd.DataFrame(data=int(c['count']), index=['_wt'], columns=['count'], dtype=np.int32, copy=False)\n",
    "my_df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "my_store.append('/main/synonymous/counts', my_df)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#display result\n",
    "my_store[\"/main/synonymous/counts\"].head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "my_store.select('/main/synonymous/counts', where=\"index in ['_wt']\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "my_store.select('/main/synonymous/counts', where=\"index in ['_sy']\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#close the storage to save\n",
    "my_store.close()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 2",
   "language": "python",
   "name": "python2"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.18"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}