Dual Model - A Divorce Story-Optimized.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# run this block *first* every time your kernel starts/restarts\n",
    "import os\n",
    "os.environ[\"CUDA_VISIBLE_DEVICES\"] = \"6\"  # either 3 or 6"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "GENOMES = { \"mouse\" : \"/users/kcochran/genomes/mm10_no_alt_analysis_set_ENCODE.fasta\",\n",
    "            \"human\" : \"/users/kcochran/genomes/GRCh38_no_alt_analysis_set_GCA_000001405.15.fasta\" }\n",
    "\n",
    "ROOT = \"/users/kcochran/projects/cs197_cross_species_domain_adaptation/\"\n",
    "DATA_DIR = ROOT + \"data/\"\n",
    "\n",
    "EPSILON = 0.00001\n",
    "SPECIES = [\"mouse\", \"human\"]\n",
    "\n",
    "ACCESSIBILITY_FILE = \"/users/angezhao/cs197_covariates/data/human/accessibility/reads.bigWig\"\n",
    "POS_BIND = \"/users/angezhao/cs197_covariates/data/human/CTCF/chr3toY_pos_shuf.bed.gz\"\n",
    "NEG_BIND = \"/users/angezhao/cs197_covariates/data/human/ctcf/chr3toY_neg_shuf.bed.gz\"\n",
    "#WINDOWS = np.array(mouse_false_negatives_windows)\n",
    "train_species = \"mouse\"\n",
    "tf = \"CTCF\"\n",
    "\n",
    "TFS = [\"CTCF\", \"CEBPA\", \"HNF4A\", \"RXRA\"]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Data Loaders"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {
    "collapsed": false,
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'1.8.1'"
      ]
     },
     "execution_count": 27,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import gzip\n",
    "import random\n",
    "import numpy as np\n",
    "from pyfaidx import Fasta\n",
    "from torch.utils.data import Dataset\n",
    "import pyBigWig\n",
    "import torch\n",
    "torch.device('cuda')\n",
    "# from torch.utils.data import Dataset\n",
    "#import pytorch\n",
    "torch.__version__"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def expand_window(start, end, target_len):\n",
    "    midpoint = (start + end) / 2\n",
    "    if not midpoint.is_integer() and target_len % 2 == 0:\n",
    "        midpoint = midpoint - 0.5\n",
    "    if midpoint.is_integer() and target_len % 2 != 0:\n",
    "        midpoint = midpoint - 0.5\n",
    "    new_start = midpoint - target_len / 2\n",
    "    new_end = midpoint + target_len / 2\n",
    "    \n",
    "    assert new_start.is_integer(), new_start\n",
    "    assert new_end.is_integer(), new_end\n",
    "    assert new_start >= 0\n",
    "    assert new_end - new_start == target_len, (new_end, new_start, target_len)\n",
    "    \n",
    "    return int(new_start), int(new_end)\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "class TrainGenerator(Dataset):\n",
    "    # must also load in accessibility here\n",
    "    # see profile model starter code for train_generator\n",
    "    # remove the control track, replace the bit w/ not control track with accessibility\n",
    "    # print out shapes\n",
    "\n",
    "    letter_dict = {\n",
    "        'a':[1,0,0,0],'c':[0,1,0,0],'g':[0,0,1,0],'t':[0,0,0,1],\n",
    "        'n':[0,0,0,0],'A':[1,0,0,0],'C':[0,1,0,0],'G':[0,0,1,0],\n",
    "        'T':[0,0,0,1],'N':[0,0,0,0]\n",
    "    }\n",
    "\n",
    "    def __init__(self, species, tf):\n",
    "        self.posfile = DATA_DIR + species + \"/\" + tf + \"/chr3toY_pos_shuf.bed.gz\"\n",
    "        self.negfile = DATA_DIR + species + \"/\" + tf + \"/chr3toY_neg_shuf_run1_1E.bed.gz\"\n",
    "        #set accessibility according to correct species\n",
    "        ACCESSIBILITY_FILE = \"/users/angezhao/cs197_covariates/data/\" + species + \"/accessibility/reads.bigWig\"\n",
    "        self.accessFile = ACCESSIBILITY_FILE\n",
    "        self.converter = Fasta(GENOMES[species])\n",
    "        self.batchsize = 400\n",
    "        self.halfbatchsize = self.batchsize // 2\n",
    "        self.current_epoch = 1\n",
    "\n",
    "        self.get_coords()\n",
    "        self.on_epoch_end()\n",
    "\n",
    "        self.profiles, self.logcounts_sum, self.logcounts_mean, self.logcounts_max = [0,0,0,1]#self.get_profiles_and_logcounts(self.coords)\n",
    "\n",
    "    def __len__(self):\n",
    "        return self.steps_per_epoch\n",
    "\n",
    "\n",
    "    def get_coords(self):\n",
    "        with gzip.open(self.posfile) as posf:\n",
    "            pos_coords_tmp = [line.decode().split()[:3] for line in posf]  # expecting bed file format\n",
    "            self.pos_coords = [(coord[0], int(coord[1]), int(coord[2])) for coord in pos_coords_tmp]  # no strand consideration\n",
    "        with gzip.open(self.negfile) as negf:\n",
    "            neg_coords_tmp = [line.decode().split()[:3] for line in negf]\n",
    "            self.neg_coords = [(coord[0], int(coord[1]), int(coord[2])) for coord in neg_coords_tmp]\n",
    "            \n",
    "        # Load in all accessibility for positive coords\n",
    "        self.pos_profiles = []\n",
    "        self.neg_profiles = []\n",
    "        with pyBigWig.open(self.accessFile) as accessibility_reader:\n",
    "            for chrom, start, end in self.pos_coords:\n",
    "                #print (chrom, \" is chrom. \", start, \" is start. \", end, \" is end.\")\n",
    "                prof_start, prof_end = expand_window(start, end,\n",
    "                                                 500 + 2 * 0)\n",
    "                pos_profile = np.array(\n",
    "                    accessibility_reader.values(chrom, prof_start, prof_end))\n",
    "                pos_profile[np.isnan(pos_profile)] = EPSILON\n",
    "                profile = np.array(pos_profile)\n",
    "\n",
    "                self.pos_profiles.append(profile)\n",
    "             \n",
    "            # load in accessibility for negative coords\n",
    "            for chrom, start, end in self.neg_coords:\n",
    "                #print (chrom, \" is chrom. \", start, \" is start. \", end, \" is end.\")\n",
    "                prof_start, prof_end = expand_window(start, end,\n",
    "                                                 500 + 2 * 0)\n",
    "                neg_profile = np.array(\n",
    "                    accessibility_reader.values(chrom, prof_start, prof_end))\n",
    "                neg_profile[np.isnan(neg_profile)] = EPSILON\n",
    "                profile = np.array(neg_profile)\n",
    "\n",
    "                self.neg_profiles.append(profile)\n",
    "        \n",
    "            \n",
    "        self.steps_per_epoch = int(len(self.pos_coords) / self.halfbatchsize)\n",
    "        print(self.steps_per_epoch)\n",
    "                \n",
    "    def get_profiles_and_logcounts(self, coords):\n",
    "        #print(\"inside profiles\")\n",
    "        profiles = []\n",
    "        logcounts_sum = []\n",
    "        logcounts_mean = []\n",
    "        logcounts_max = []\n",
    "\n",
    "        with pyBigWig.open(self.accessFile) as accessibility_reader:\n",
    "            for chrom, start, end in coords:\n",
    "                #print (chrom, \" is chrom. \", start, \" is start. \", end, \" is end.\")\n",
    "                prof_start, prof_end = expand_window(start, end,\n",
    "                                                 500 + 2 * 0)\n",
    "                pos_profile = np.array(\n",
    "                    accessibility_reader.values(chrom, prof_start, prof_end))\n",
    "                pos_profile[np.isnan(pos_profile)] = EPSILON\n",
    "                profile = np.array(pos_profile)\n",
    "\n",
    "                profiles.append(profile)\n",
    "                logcounts_sum.append(np.array(np.sum(pos_profile)))\n",
    "                logcounts_mean.append(np.array(np.mean(pos_profile)))\n",
    "                logcounts_max.append(np.array(np.max(pos_profile)))\n",
    "                \n",
    "        #print(\"Finished going thorugh it all. \")        \n",
    "        profiles = np.array(profiles)\n",
    "        logcounts_sum = np.array(logcounts_sum)\n",
    "        logcounts_mean = np.array(logcounts_mean)\n",
    "        logcounts_max = np.array(logcounts_max)\n",
    "\n",
    "        return profiles, logcounts_sum, logcounts_mean, logcounts_max\n",
    "    \n",
    "    def convert(self, coords):\n",
    "        seqs_onehot = []\n",
    "        #with pyBigWig.open(self.accessFile) as acc_bw_reader:\n",
    "        for coord in coords:\n",
    "            chrom, start, stop = coord\n",
    "            #prof_start, prof_end = expand_window(start, stop,\n",
    "             #                                500 + 2 * 0)\n",
    "            #pos_profile = np.array(\n",
    "            #    acc_bw_reader.values(chrom, prof_start, prof_end))\n",
    "            seq = self.converter[chrom][start:stop].seq\n",
    "            seq_onehot = np.array([self.letter_dict.get(x,[0,0,0,0]) for x in seq])\n",
    "            #print(seq_onehot, \"convert traingenerator\");\n",
    "            seqs_onehot.append(seq_onehot)\n",
    "\n",
    "        seqs_onehot = np.array(seqs_onehot)\n",
    "        return seqs_onehot\n",
    "\n",
    "    def addAccess(self, coords, seqs_onehot):\n",
    "        with pyBigWig.open(self.accessFile) as acc_bw_reader:\n",
    "            for coord in coords:\n",
    "                chrom, start, stop = coord\n",
    "                seq = self.converter[chrom][start:stop].seq\n",
    "                \n",
    "        return seqs_onehot\n",
    "\n",
    "    def __getitem__(self, batch_index):\t\n",
    "        # First, get chunk of coordinates\n",
    "        pos_coords_batch = self.pos_coords[batch_index * self.halfbatchsize : (batch_index + 1) * self.halfbatchsize]\n",
    "        neg_coords_batch = self.neg_coords[batch_index * self.halfbatchsize : (batch_index + 1) * self.halfbatchsize]\n",
    "        pos_access_batch = self.pos_profiles[batch_index * self.halfbatchsize : (batch_index + 1) * self.halfbatchsize]\n",
    "        neg_access_batch = self.neg_profiles[batch_index * self.halfbatchsize : (batch_index + 1) * self.halfbatchsize]\n",
    "\n",
    "        # if train_steps calculation is off, lists of coords may be empty\n",
    "        assert len(pos_coords_batch) > 0, len(pos_coords_batch)\n",
    "        assert len(neg_coords_batch) > 0, len(neg_coords_batch)\n",
    "\n",
    "        # Second, convert the coordinates into one-hot encoded sequences\n",
    "        pos_onehot = self.convert(pos_coords_batch)\n",
    "        neg_onehot = self.convert(neg_coords_batch)\n",
    "        \n",
    "        #add in access\n",
    "        \n",
    "        # seqdataloader returns empty array if coords are empty list or not in genome\n",
    "        assert pos_onehot.shape[0] > 0, pos_onehot.shape[0]\n",
    "        assert neg_onehot.shape[0] > 0, neg_onehot.shape[0]\n",
    "\n",
    "        # Third, combine bound and unbound sites into one large array, and create label vector\n",
    "        # We don't need to shuffle here because all these examples will correspond\n",
    "        # to a simultaneous gradient update for the whole batch\n",
    "        all_seqs = np.concatenate((pos_onehot, neg_onehot))\n",
    "        labels = np.concatenate((np.ones(pos_onehot.shape[0],), np.zeros(neg_onehot.shape[0],)))\n",
    "        \n",
    "        all_seqs = torch.tensor(all_seqs, dtype=torch.float).permute(0, 2, 1)\n",
    "        labels = torch.tensor(labels, dtype=torch.float)\n",
    "        assert all_seqs.shape[0] == self.batchsize, all_seqs.shape[0]\n",
    "        \n",
    "        #Getting Accessibility for these coordinate\n",
    "        #profiles1, logcounts_sum1, logcounts_mean1, logcounts_max1 = self.get_profiles_and_logcounts(pos_coords_batch)\n",
    "        #profiles, logcounts_sum, logcounts_mean, logcounts_max = self.get_profiles_and_logcounts(neg_coords_batch)\n",
    "        profiles = np.concatenate((pos_access_batch, neg_access_batch))\n",
    "        profiles = torch.tensor(profiles, dtype=torch.float)\n",
    "        #self.logcounts_sum = logcounts_sum1 + logcounts_sum\n",
    "        #self.logcounts_mean = logcounts_max1 + logcounts_max\n",
    "\n",
    "        return all_seqs, labels, profiles\n",
    "\n",
    "\n",
    "    def on_epoch_end(self):\n",
    "        # switch to next set of negative examples\n",
    "        prev_epoch = self.current_epoch\n",
    "        next_epoch = prev_epoch + 1\n",
    "\n",
    "        # update file where we will retrieve unbound site coordinates from\n",
    "        prev_negfile = self.negfile\n",
    "        next_negfile = prev_negfile.replace(str(prev_epoch) + \"E\", str(next_epoch) + \"E\")\n",
    "        self.negfile = next_negfile\n",
    "\n",
    "        # load in new unbound site coordinates\n",
    "        with gzip.open(self.negfile) as negf:\n",
    "            neg_coords_tmp = [line.decode().split()[:3] for line in negf]\n",
    "            self.neg_coords = [(coord[0], int(coord[1]), int(coord[2])) for coord in neg_coords_tmp]\n",
    "\n",
    "        # then shuffle positive examples\n",
    "        random.shuffle(self.pos_coords)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "class ValGenerator(Dataset):\n",
    "    letter_dict = {\n",
    "        'a':[1,0,0,0],'c':[0,1,0,0],'g':[0,0,1,0],'t':[0,0,0,1],\n",
    "        'n':[0,0,0,0],'A':[1,0,0,0],'C':[0,1,0,0],'G':[0,0,1,0],\n",
    "        'T':[0,0,0,1],'N':[0,0,0,0]}\n",
    "\n",
    "    def __init__(self, species, tf, return_labels = True):\n",
    "        self.valfile = DATA_DIR + species + \"/\" + tf + \"/chr1_random_1m.bed.gz\"\n",
    "        self.converter = Fasta(GENOMES[species])\n",
    "        self.batchsize = 1000  # arbitrarily large number that will fit into memory\n",
    "        self.return_labels = return_labels\n",
    "        ACCESSIBILITY_FILE = \"/users/angezhao/cs197_covariates/data/\" + species + \"/accessibility/reads.bigWig\"\n",
    "        self.accessFile = ACCESSIBILITY_FILE\n",
    "        self.get_coords_and_labels()\n",
    "        \n",
    "\n",
    "\n",
    "    def __len__(self):\n",
    "        return self.steps_per_epoch\n",
    "\n",
    "\n",
    "    def get_coords_and_labels(self):\n",
    "        with gzip.open(self.valfile) as f:\n",
    "            coords_tmp = [line.decode().split()[:4] for line in f]  # expecting bed file format\n",
    "        \n",
    "        self.labels = [int(coord[3]) for coord in coords_tmp]\n",
    "        self.coords = [(coord[0], int(coord[1]), int(coord[2])) for coord in coords_tmp]  # no strand consideration\n",
    "        \n",
    "        self.steps_per_epoch = int(len(self.coords) / self.batchsize)\n",
    "        # Load in all accessibility for positive coords\n",
    "        self.profiles = []\n",
    "\n",
    "        with pyBigWig.open(self.accessFile) as accessibility_reader:\n",
    "            for chrom, start, end in self.coords:\n",
    "                #print (chrom, \" is chrom. \", start, \" is start. \", end, \" is end.\")\n",
    "                prof_start, prof_end = expand_window(start, end,\n",
    "                                                 500 + 2 * 0)\n",
    "                pos_profile = np.array(\n",
    "                    accessibility_reader.values(chrom, prof_start, prof_end))\n",
    "                pos_profile[np.isnan(pos_profile)] = EPSILON\n",
    "                profile = np.array(pos_profile)\n",
    "\n",
    "                self.profiles.append(profile)\n",
    "        \n",
    "\n",
    "    def convert(self, coords):\n",
    "        seqs_onehot = []\n",
    "        \n",
    "        for coord in coords:\n",
    "            chrom, start, stop = coord\n",
    "            seq = self.converter[chrom][start:stop].seq\n",
    "            seq_onehot = np.array([self.letter_dict.get(x,[0,0,0,0]) for x in seq])\n",
    "            seqs_onehot.append(seq_onehot)\n",
    "\n",
    "        seqs_onehot = np.array(seqs_onehot)\n",
    "        return seqs_onehot\n",
    "\n",
    "    def get_profiles_and_logcounts(self, coords):\n",
    "        #print(\"inside profiles\")\n",
    "        profiles = []\n",
    "        logcounts_sum = []\n",
    "        logcounts_mean = []\n",
    "        logcounts_max = []\n",
    "\n",
    "        with pyBigWig.open(self.accessFile) as accessibility_reader:\n",
    "            for chrom, start, end in coords:\n",
    "                #print (chrom, \" is chrom. \", start, \" is start. \", end, \" is end.\")\n",
    "                prof_start, prof_end = expand_window(start, end,\n",
    "                                                 500 + 2 * 0)\n",
    "                pos_profile = np.array(\n",
    "                    accessibility_reader.values(chrom, prof_start, prof_end))\n",
    "                pos_profile[np.isnan(pos_profile)] = EPSILON\n",
    "                profile = np.array(pos_profile)\n",
    "\n",
    "                profiles.append(profile)\n",
    "                logcounts_sum.append(np.array(np.sum(pos_profile)))\n",
    "                logcounts_mean.append(np.array(np.mean(pos_profile)))\n",
    "                logcounts_max.append(np.array(np.max(pos_profile)))\n",
    "                \n",
    "        #print(\"Finished going thorugh it all. \")        \n",
    "        profiles = np.array(profiles)\n",
    "        logcounts_sum = np.array(logcounts_sum)\n",
    "        logcounts_mean = np.array(logcounts_mean)\n",
    "        logcounts_max = np.array(logcounts_max)\n",
    "\n",
    "        return profiles, logcounts_sum, logcounts_mean, logcounts_max\n",
    "\n",
    "    def __getitem__(self, batch_index):\t\n",
    "        # First, get chunk of coordinates\n",
    "        batch_start = batch_index * self.batchsize\n",
    "        batch_end = (batch_index + 1) * self.batchsize\n",
    "        coords_batch = self.coords[batch_start : batch_end]\n",
    "        profiles = self.profiles[batch_start : batch_end]\n",
    "        # if train_steps calculation is off, lists of coords may be empty\n",
    "        assert len(coords_batch) > 0, len(coords_batch)\n",
    "\n",
    "        # Second, convert the coordinates into one-hot encoded sequences\n",
    "        onehot = self.convert(coords_batch)\n",
    "\n",
    "        # array will be empty if coords are not found in the genome\n",
    "        assert onehot.shape[0] > 0, onehot.shape[0]\n",
    "\n",
    "        onehot = torch.tensor(onehot, dtype=torch.float).permute(0, 2, 1)\n",
    "        profiles = torch.tensor(profiles, dtype=torch.float)\n",
    "        \n",
    "        if self.return_labels:\n",
    "            labels = self.labels[batch_start : batch_end]\n",
    "            labels = torch.tensor(labels, dtype=torch.float)\n",
    "            return onehot, labels, profiles\n",
    "        else:\n",
    "            return onehot"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Model Training And Evaluation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# Performance metric functions\n",
    "\n",
    "from sklearn.metrics import average_precision_score, roc_auc_score, confusion_matrix, log_loss\n",
    "\n",
    "\n",
    "def print_metrics(preds, labels):\n",
    "    preds = np.array(preds)\n",
    "    labels = np.array(labels)\n",
    "    preds = preds.squeeze()\n",
    "\n",
    "    # this is the binary cross-entropy loss, same as in training\n",
    "    print(\"Loss:\\t\", log_loss(labels, preds))\n",
    "    print(\"auROC:\\t\", roc_auc_score(labels, preds))\n",
    "    auPRC = average_precision_score(labels, preds)\n",
    "    print(\"auPRC:\\t\", auPRC)\n",
    "    print_confusion_matrix(preds, labels)\n",
    "    return auPRC\n",
    "\n",
    "def print_confusion_matrix(preds, labels):\n",
    "    npthresh = np.vectorize(lambda t: 1 if t >= 0.5 else 0)\n",
    "    preds_binarized = npthresh(preds)\n",
    "    conf_matrix = confusion_matrix(labels, preds_binarized)\n",
    "    print(\"Confusion Matrix (at t = 0.5):\\n\", conf_matrix)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import torch\n",
    "from torch.utils.data import DataLoader\n",
    "\n",
    "\n",
    "\n",
    "class BasicModel(torch.nn.Module):\n",
    "    def __init__(self):\n",
    "        super(BasicModel, self).__init__()\n",
    "        self.input_seq_len = 500\n",
    "        num_conv_filters = 240\n",
    "        lstm_hidden_units = 32\n",
    "        fc_layer1_units = 1024\n",
    "        fc_layer2_units = 512\n",
    "        \n",
    "        #How do we know which species of data we should use???\n",
    "        #Our added on data.\n",
    "        #self.access_data = Generator(train_species, tf, \"val\")\n",
    "        #inits for our Access\n",
    "        self.convAccess = torch.nn.Conv1d(1, 10, kernel_size=10, padding=0)\n",
    "        self.lstmAccess = torch.nn.LSTM(input_size=10,\n",
    "                                  hidden_size=32,\n",
    "                                  batch_first=True)\n",
    "        # Defining the layers to go into our model\n",
    "        # (see the forward function for how they fit together)\n",
    "        self.conv = torch.nn.Conv1d(4, num_conv_filters, kernel_size=20, padding=0)\n",
    "        # self.conv = torch.nn.Conv1d(1, num_conv_filters, kernel_size=5, padding=0) -> accessibility\n",
    "        self.relu = torch.nn.ReLU()\n",
    "        self.maxpool = torch.nn.MaxPool1d(15, stride=15, padding=0)\n",
    "        self.lstm = torch.nn.LSTM(input_size=num_conv_filters,\n",
    "                                  hidden_size=lstm_hidden_units,\n",
    "                                  batch_first=True)\n",
    "        self.fc1 = torch.nn.Linear(in_features=lstm_hidden_units,\n",
    "                                   out_features=fc_layer1_units)\n",
    "        self.dropout = torch.nn.Dropout(p=0.5)\n",
    "        self.fc2 = torch.nn.Linear(in_features=fc_layer1_units,\n",
    "                                   out_features=fc_layer2_units)\n",
    "        self.fc_final = torch.nn.Linear(in_features=fc_layer2_units,\n",
    "                                        out_features=1)\n",
    "        self.sigmoid = torch.nn.Sigmoid()\n",
    "\n",
    "        # The loss function we'll use -- binary cross-entropy\n",
    "        # (this is the standard loss to use for binary classification)\n",
    "        self.loss = torch.nn.BCELoss()\n",
    "\n",
    "        # We'll store performance metrics during training in these lists\n",
    "        self.train_loss_by_epoch = []\n",
    "        self.source_val_loss_by_epoch = []\n",
    "        self.source_val_auprc_by_epoch = []\n",
    "        self.target_val_loss_by_epoch = []\n",
    "        self.target_val_auprc_by_epoch = []\n",
    "\n",
    "        # We'll record the best model we've seen yet each epoch\n",
    "        self.best_state_so_far = self.state_dict()\n",
    "        self.best_auprc_so_far = 1\n",
    "\n",
    "\n",
    "    def forward(self, X, Y):\n",
    "        #print(\"Inside forward\")\n",
    "        #print(X.traingen.profiles)\n",
    "        #print(\"Y shape \", Y.shape)\n",
    "        Y = Y.permute(1,0,2)\n",
    "        #print(\"Y shape after \", Y.shape)\n",
    "        Y_1 = self.relu(self.convAccess(Y))\n",
    "        #print(\"Y_1 shape \", Y_1.shape)\n",
    "        Y_2 = self.maxpool(Y_1).permute(0, 2, 1)\n",
    "        #print(\"Y_2 shape \", Y_2.shape)\n",
    "        Y_3, _ = self.lstmAccess(Y_2)\n",
    "        #print(\"Y_3 shape \", Y_3.shape)\n",
    "        Y_4 = Y_3[:, -1]\n",
    "        #print(\"Y_4 shape \", Y_4.shape)\n",
    "        X_1 = self.relu(self.conv(X))\n",
    "        # LSTM is expecting input of shape (batches, seq_len, conv_filters)\n",
    "        X_2 = self.maxpool(X_1).permute(0, 2, 1)\n",
    "        X_3, _ = self.lstm(X_2)\n",
    "        X_4 = X_3[:, -1]  # only need final output of LSTM\n",
    "        #print(\"X_4 shape \", X_4.shape)\n",
    "        XY_4 = torch.cat((X_4, Y_4))\n",
    "        # pass in 2 xs to this function\n",
    "        # concat the two xs at X_4, and then return 1 y\n",
    "        \n",
    "        \n",
    "        X_5 = self.relu(self.fc1(X_4))\n",
    "        X_6 = self.dropout(X_5)\n",
    "        X_7 = self.sigmoid(self.fc2(X_6))\n",
    "        y = self.sigmoid(self.fc_final(X_7)).squeeze()\n",
    "        return y\n",
    "    \n",
    "    def validation(self, data_loader):\n",
    "        # only run this within torch.no_grad() context!\n",
    "        losses = []\n",
    "        preds = []\n",
    "        labels = []\n",
    "        for seqs_onehot_batch, labels_batch, profiles in data_loader:\n",
    "            # push batch through model, get predictions, calculate loss\n",
    "            preds_batch = self(seqs_onehot_batch.squeeze().cuda(), profiles.cuda())\n",
    "            labels_batch = labels_batch.squeeze()\n",
    "            loss_batch = self.loss(preds_batch, labels_batch.cuda())\n",
    "            losses.append(loss_batch.item())\n",
    "\n",
    "            # storing labels + preds for auPRC calculation later\n",
    "            labels.extend(labels_batch.detach().numpy())  \n",
    "            preds.extend(preds_batch.cpu().detach().numpy())\n",
    "            \n",
    "        return np.array(losses), np.array(preds), np.array(labels)\n",
    "\n",
    "\n",
    "    def fit(self, train_gen, source_val_data_loader, target_val_data_loader,\n",
    "            optimizer, epochs=15):\n",
    "        \n",
    "        for epoch in range(epochs):\n",
    "            torch.cuda.empty_cache()  # clear memory to keep stuff from blocking up\n",
    "            \n",
    "            print(\"=== Epoch \" + str(epoch + 1) + \" ===\")\n",
    "            print(\"Training...\")\n",
    "            self.train()\n",
    "            #print (train_gen.profiles, \" is profiles in fit\")\n",
    "            # using a batch size of 1 here because the generator returns\n",
    "            # many examples in each batch\n",
    "            train_data_loader = DataLoader(train_gen,\n",
    "                               batch_size = 1, shuffle = True)\n",
    "\n",
    "            train_losses = []\n",
    "            train_preds = []\n",
    "            train_labels = []\n",
    "            for seqs_onehot_batch, labels_batch, profile_batch in train_data_loader:\n",
    "                # reset the optimizer; need to do each batch after weight update\n",
    "                optimizer.zero_grad()\n",
    "\n",
    "                # push batch through model, get predictions, and calculate loss\n",
    "                preds = self(seqs_onehot_batch.squeeze().cuda(), profile_batch.cuda())\n",
    "                labels_batch = labels_batch.squeeze()\n",
    "                loss_batch = self.loss(preds, labels_batch.cuda())\n",
    "                \n",
    "                # brackpropogate the loss and update model weights accordingly\n",
    "                loss_batch.backward()\n",
    "                optimizer.step()\n",
    "                \n",
    "                train_losses.append(loss_batch.item())\n",
    "                train_labels.extend(labels_batch)\n",
    "                train_preds.extend(preds.cpu().detach().numpy())\n",
    "\n",
    "            self.train_loss_by_epoch.append(np.mean(train_losses))\n",
    "            print_metrics(train_preds, train_labels)\n",
    "            \n",
    "            # load new set of negative examples for next epoch\n",
    "            train_gen.on_epoch_end()\n",
    "\n",
    "            \n",
    "            # Assess model performance on same-species validation set\n",
    "            print(\"Evaluating on source validation data...\")\n",
    "            \n",
    "            # Since we don't use gradients during model evaluation,\n",
    "            # the following two lines let the model predict for many examples\n",
    "            # more efficiently (without having to keep track of gradients)\n",
    "            self.eval()\n",
    "            with torch.no_grad():\n",
    "                source_val_losses, source_val_preds, source_val_labels = self.validation(source_val_data_loader)\n",
    "\n",
    "                print(\"Validation loss:\", np.mean(source_val_losses))\n",
    "                self.source_val_loss_by_epoch.append(np.mean(source_val_losses))\n",
    "\n",
    "                # calc auPRC over source validation set\n",
    "                source_val_auprc = print_metrics(source_val_preds, source_val_labels)\n",
    "                self.source_val_auprc_by_epoch.append(source_val_auprc)\n",
    "\n",
    "                # check if this is the best performance we've seen so far\n",
    "                # if yes, save the model weights -- we'll use the best model overall\n",
    "                # for later analyses\n",
    "                if source_val_auprc < self.best_auprc_so_far:\n",
    "                    self.best_auprc_so_far = source_val_auprc\n",
    "                    self.best_state_so_far = self.state_dict()\n",
    "                \n",
    "                \n",
    "                # now repeat for target species data \n",
    "                print(\"Evaluating on target validation data...\")\n",
    "                \n",
    "                target_val_losses, target_val_preds, target_val_labels = self.validation(target_val_data_loader)\n",
    "\n",
    "                print(\"Validation loss:\", np.mean(target_val_losses))\n",
    "                self.target_val_loss_by_epoch.append(np.mean(target_val_losses))\n",
    "\n",
    "                # calc auPRC over source validation set\n",
    "                target_val_auprc = print_metrics(target_val_preds, target_val_labels)\n",
    "                self.target_val_auprc_by_epoch.append(target_val_auprc)\n",
    "                "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Setup + Train"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2468\n"
     ]
    }
   ],
   "source": [
    "# setup generators / data loaders for training and validation\n",
    "tfType = \"CEBPA\"\n",
    "source = \"mouse\"\n",
    "target = \"human\"\n",
    "# we'll make the training data loader in the training loop,\n",
    "# since we need to update some of the examples used each epoch\n",
    "train_gen = TrainGenerator(source, tfType)\n",
    "\n",
    "source_val_gen = ValGenerator(source, tfType)\n",
    "# using a batch size of 1 here because the generator returns\n",
    "# many examples in each batch\n",
    "source_val_data_loader = DataLoader(source_val_gen, batch_size = 1, shuffle = False)\n",
    "\n",
    "target_val_gen = ValGenerator(target, tfType)\n",
    "target_val_data_loader = DataLoader(target_val_gen, batch_size = 1, shuffle = False)\n",
    "\n",
    "# consider issues of overfitting using auPRC in trainGen and ValGen -> issue arises from the fact that adding in accessibility\n",
    "# should improve same species prediction performance\n",
    "# check for overfitting -> consider reducing the number of parameters\n",
    "# last resort: binarize accessibility files"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "\n",
    "from __future__ import print_function\n",
    "import sys  "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "=== Epoch 1 ===\n",
      "Training...\n",
      "Loss:\t 0.4620185729008359\n",
      "auROC:\t 0.8612548547009501\n",
      "auPRC:\t 0.858659238609381\n",
      "Confusion Matrix (at t = 0.5):\n",
      " [[376427 117173]\n",
      " [106295 387305]]\n",
      "Evaluating on source validation data...\n",
      "Validation loss: 0.2743188961446285\n",
      "Loss:\t 0.274318883414389\n",
      "auROC:\t 0.9108266037438801\n",
      "auPRC:\t 0.22769938496729417\n",
      "Confusion Matrix (at t = 0.5):\n",
      " [[871248 116707]\n",
      " [  2796   9249]]\n",
      "Evaluating on target validation data...\n",
      "Validation loss: 0.4305892720520496\n",
      "Loss:\t 0.4305892514164263\n",
      "auROC:\t 0.9049039086988327\n",
      "auPRC:\t 0.08892357997183793\n",
      "Confusion Matrix (at t = 0.5):\n",
      " [[795749 197196]\n",
      " [  1006   6049]]\n",
      "=== Epoch 2 ===\n",
      "Training...\n",
      "Loss:\t 0.35866116750102695\n",
      "auROC:\t 0.9204518523609495\n",
      "auPRC:\t 0.9161829924048634\n",
      "Confusion Matrix (at t = 0.5):\n",
      " [[412651  80949]\n",
      " [ 72910 420690]]\n",
      "Evaluating on source validation data...\n",
      "Validation loss: 0.4232696333229542\n",
      "Loss:\t 0.4232696132154192\n",
      "auROC:\t 0.9269294662091987\n",
      "auPRC:\t 0.26818732769487375\n",
      "Confusion Matrix (at t = 0.5):\n",
      " [[794353 193602]\n",
      " [  1310  10735]]\n",
      "Evaluating on target validation data...\n",
      "Validation loss: 0.6211814179420471\n",
      "Loss:\t 0.6211813896650439\n",
      "auROC:\t 0.9218170803808966\n",
      "auPRC:\t 0.12838289187261434\n",
      "Confusion Matrix (at t = 0.5):\n",
      " [[704945 288000]\n",
      " [   432   6623]]\n",
      "=== Epoch 3 ===\n",
      "Training...\n",
      "Loss:\t 0.3275723112203171\n",
      "auROC:\t 0.9338207575544769\n",
      "auPRC:\t 0.92931679719272\n",
      "Confusion Matrix (at t = 0.5):\n",
      " [[419083  74517]\n",
      " [ 62609 430991]]\n",
      "Evaluating on source validation data...\n",
      "Validation loss: 0.26318813578784467\n",
      "Loss:\t 0.26318812431993893\n",
      "auROC:\t 0.9336477823075079\n",
      "auPRC:\t 0.2895650583190223\n",
      "Confusion Matrix (at t = 0.5):\n",
      " [[874383 113572]\n",
      " [  2066   9979]]\n",
      "Evaluating on target validation data...\n",
      "Validation loss: 0.4210749289393425\n",
      "Loss:\t 0.4210749084214405\n",
      "auROC:\t 0.9279710565695127\n",
      "auPRC:\t 0.14952386920030225\n",
      "Confusion Matrix (at t = 0.5):\n",
      " [[804272 188673]\n",
      " [   733   6322]]\n",
      "=== Epoch 4 ===\n",
      "Training...\n",
      "Loss:\t 0.30690356078085596\n",
      "auROC:\t 0.941827194620724\n",
      "auPRC:\t 0.9372587773885513\n",
      "Confusion Matrix (at t = 0.5):\n",
      " [[423363  70237]\n",
      " [ 56115 437485]]\n",
      "Evaluating on source validation data...\n",
      "Validation loss: 0.2902320208400488\n",
      "Loss:\t 0.2902320067843697\n",
      "auROC:\t 0.9371251163603083\n",
      "auPRC:\t 0.3052302020779335\n",
      "Confusion Matrix (at t = 0.5):\n",
      " [[856860 131095]\n",
      " [  1641  10404]]\n",
      "Evaluating on target validation data...\n",
      "Validation loss: 0.45725367280840873\n",
      "Loss:\t 0.4572536528701836\n",
      "auROC:\t 0.9315347727187668\n",
      "auPRC:\t 0.16441697948025083\n",
      "Confusion Matrix (at t = 0.5):\n",
      " [[780950 211995]\n",
      " [   544   6511]]\n",
      "=== Epoch 5 ===\n",
      "Training...\n",
      "Loss:\t 0.2924049346697351\n",
      "auROC:\t 0.9470367574544937\n",
      "auPRC:\t 0.9424490060435032\n",
      "Confusion Matrix (at t = 0.5):\n",
      " [[426812  66788]\n",
      " [ 51863 441737]]\n",
      "Evaluating on source validation data...\n"
     ]
    },
    {
     "ename": "KeyboardInterrupt",
     "evalue": "",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)",
      "\u001b[0;32m/tmp/ipykernel_11333/1187919220.py\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[1;32m      9\u001b[0m \u001b[0;31m# train!\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     10\u001b[0m \u001b[0mmodel\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mcuda\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 11\u001b[0;31m \u001b[0mmodel\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mfit\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtrain_gen\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0msource_val_data_loader\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtarget_val_data_loader\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0moptimizer\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mepochs\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;36m5\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     12\u001b[0m \u001b[0mmodel\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mload_state_dict\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mmodel\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mbest_state_so_far\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     13\u001b[0m \u001b[0mmodel\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mcpu\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/tmp/ipykernel_11333/1819983165.py\u001b[0m in \u001b[0;36mfit\u001b[0;34m(self, train_gen, source_val_data_loader, target_val_data_loader, optimizer, epochs)\u001b[0m\n\u001b[1;32m    155\u001b[0m             \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0meval\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    156\u001b[0m             \u001b[0;32mwith\u001b[0m \u001b[0mtorch\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mno_grad\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 157\u001b[0;31m                 \u001b[0msource_val_losses\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0msource_val_preds\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0msource_val_labels\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mvalidation\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0msource_val_data_loader\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    158\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    159\u001b[0m                 \u001b[0mprint\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m\"Validation loss:\"\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mnp\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmean\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0msource_val_losses\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/tmp/ipykernel_11333/1819983165.py\u001b[0m in \u001b[0;36mvalidation\u001b[0;34m(self, data_loader)\u001b[0m\n\u001b[1;32m     91\u001b[0m         \u001b[0mpreds\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     92\u001b[0m         \u001b[0mlabels\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 93\u001b[0;31m         \u001b[0;32mfor\u001b[0m \u001b[0mseqs_onehot_batch\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mlabels_batch\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mprofiles\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mdata_loader\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     94\u001b[0m             \u001b[0;31m# push batch through model, get predictions, calculate loss\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     95\u001b[0m             \u001b[0mpreds_batch\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mseqs_onehot_batch\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msqueeze\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mcuda\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mprofiles\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mcuda\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/miniconda3/envs/cs197_env1/lib/python3.7/site-packages/torch/utils/data/dataloader.py\u001b[0m in \u001b[0;36m__next__\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m    515\u001b[0m             \u001b[0;32mif\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_sampler_iter\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    516\u001b[0m                 \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_reset\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 517\u001b[0;31m             \u001b[0mdata\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_next_data\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    518\u001b[0m             \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_num_yielded\u001b[0m \u001b[0;34m+=\u001b[0m \u001b[0;36m1\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    519\u001b[0m             \u001b[0;32mif\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_dataset_kind\u001b[0m \u001b[0;34m==\u001b[0m \u001b[0m_DatasetKind\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mIterable\u001b[0m \u001b[0;32mand\u001b[0m\u001b[0;31m \u001b[0m\u001b[0;31m\\\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/miniconda3/envs/cs197_env1/lib/python3.7/site-packages/torch/utils/data/dataloader.py\u001b[0m in \u001b[0;36m_next_data\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m    555\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0m_next_data\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    556\u001b[0m         \u001b[0mindex\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_next_index\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m  \u001b[0;31m# may raise StopIteration\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 557\u001b[0;31m         \u001b[0mdata\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_dataset_fetcher\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mfetch\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mindex\u001b[0m\u001b[0;34m)\u001b[0m  \u001b[0;31m# may raise StopIteration\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    558\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_pin_memory\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    559\u001b[0m             \u001b[0mdata\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0m_utils\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mpin_memory\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mpin_memory\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mdata\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/miniconda3/envs/cs197_env1/lib/python3.7/site-packages/torch/utils/data/_utils/fetch.py\u001b[0m in \u001b[0;36mfetch\u001b[0;34m(self, possibly_batched_index)\u001b[0m\n\u001b[1;32m     42\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0mfetch\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mpossibly_batched_index\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     43\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mauto_collation\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 44\u001b[0;31m             \u001b[0mdata\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mdataset\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0midx\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;32mfor\u001b[0m \u001b[0midx\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mpossibly_batched_index\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     45\u001b[0m         \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     46\u001b[0m             \u001b[0mdata\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mdataset\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mpossibly_batched_index\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/miniconda3/envs/cs197_env1/lib/python3.7/site-packages/torch/utils/data/_utils/fetch.py\u001b[0m in \u001b[0;36m<listcomp>\u001b[0;34m(.0)\u001b[0m\n\u001b[1;32m     42\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0mfetch\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mpossibly_batched_index\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     43\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mauto_collation\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 44\u001b[0;31m             \u001b[0mdata\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mdataset\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0midx\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;32mfor\u001b[0m \u001b[0midx\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mpossibly_batched_index\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     45\u001b[0m         \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     46\u001b[0m             \u001b[0mdata\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mdataset\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mpossibly_batched_index\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/tmp/ipykernel_11333/4040659081.py\u001b[0m in \u001b[0;36m__getitem__\u001b[0;34m(self, batch_index)\u001b[0m\n\u001b[1;32m     96\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     97\u001b[0m         \u001b[0;31m# Second, convert the coordinates into one-hot encoded sequences\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 98\u001b[0;31m         \u001b[0monehot\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mconvert\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mcoords_batch\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     99\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    100\u001b[0m         \u001b[0;31m# array will be empty if coords are not found in the genome\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/tmp/ipykernel_11333/4040659081.py\u001b[0m in \u001b[0;36mconvert\u001b[0;34m(self, coords)\u001b[0m\n\u001b[1;32m     50\u001b[0m             \u001b[0mchrom\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mstart\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mstop\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mcoord\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     51\u001b[0m             \u001b[0mseq\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mconverter\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mchrom\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mstart\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0mstop\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mseq\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 52\u001b[0;31m             \u001b[0mseq_onehot\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mnp\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0marray\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mletter_dict\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mget\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mx\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m0\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;36m0\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;36m0\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;36m0\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;32mfor\u001b[0m \u001b[0mx\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mseq\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     53\u001b[0m             \u001b[0mseqs_onehot\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mappend\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mseq_onehot\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     54\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m: "
     ]
    }
   ],
   "source": [
    "\n",
    "#%%capture cap --no-stderr\n",
    "#with open ('ModelOutput.txt', 'w') as f:\n",
    "#    f.write(cap.stdout)\n",
    "#print(\"Test\")\n",
    "# initialize the model\n",
    "model = BasicModel()\n",
    "optimizer = torch.optim.Adam(model.parameters(), lr=0.001)\n",
    "\n",
    "# train!\n",
    "model.cuda()\n",
    "model.fit(train_gen, source_val_data_loader, target_val_data_loader, optimizer, epochs = 5)\n",
    "model.load_state_dict(model.best_state_so_far)\n",
    "model.cpu()\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "with open ('ModelOutput.txt', 'r') as f:\n",
    "    print(f.read());"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "\n",
    "\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "env1",
   "language": "python",
   "name": "env1"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.10"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}