diff --git a/notebooks/data-sources/TestGrid/metrics/unexpected_test_failures.ipynb b/notebooks/data-sources/TestGrid/metrics/unexpected_test_failures.ipynb new file mode 100644 index 00000000..05eeef0a --- /dev/null +++ b/notebooks/data-sources/TestGrid/metrics/unexpected_test_failures.ipynb @@ -0,0 +1,609 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "id": "986f8173-ec5d-4408-8b16-2870eba609ce", + "metadata": {}, + "source": [ + "# Unexpected Test Failures" + ] + }, + { + "cell_type": "markdown", + "id": "1b6b805a-cb06-48f8-834f-74297b75862a", + "metadata": {}, + "source": [ + "In this notebook we initially set out to predict infrastructure flakes from testgrid data. Usually an infrastructure flake can be categorized by several tests failing unexpectedly at the same time, which would mean that the problem wasn't the tests themselves, but rather the infrastructure they run on. The main challenge came from defining mathematically what it could mean for tests to fail unexpectedly. That became the main goal of this notebook and classifying why the tests failed unexpectedly, whether it be because of infrastructure or other reasons, will require further analysis. In this notebook, every column of testgrid data will be classified with a \"unexpectedness score\" that is really a probability from 0 to 100." + ] + }, + { + "cell_type": "markdown", + "id": "b77c4adb-9d60-455a-bbaf-7ddd004160d3", + "metadata": {}, + "source": [ + "## Load Data" + ] + }, + { + "cell_type": "code", + "execution_count": 533, + "id": "8ea6c93f-2136-4a36-9e72-9bfdfe02c5c8", + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "True" + ] + }, + "execution_count": 533, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "import json\n", + "import gzip\n", + "import os\n", + "import pandas as pd\n", + "import matplotlib.pyplot as plt\n", + "import seaborn as sns\n", + "import datetime\n", + "import numpy as np\n", + "from matplotlib import colors\n", + "\n", + "from ipynb.fs.defs.metric_template import CephCommunication\n", + "from dotenv import load_dotenv, find_dotenv\n", + "\n", + "import sys\n", + "\n", + "sys.path.append('../../../failure-type-classification')\n", + "\n", + "from ipynb.fs.defs.failure_type_functions import ( #noqa\n", + " normalize, #noqa\n", + " decode_run_length, #noqa\n", + ")\n", + "\n", + "load_dotenv(find_dotenv())" + ] + }, + { + "cell_type": "code", + "execution_count": 550, + "id": "34ab464c-5512-4096-aa81-5f579e30524b", + "metadata": {}, + "outputs": [], + "source": [ + "## Specify variables\n", + "METRIC_NAME = \"number_of_flakes\"\n", + "# Specify the path for input grid data,\n", + "INPUT_DATA_PATH = \"../../../../data/raw/testgrid_810.json.gz\"\n", + "\n", + "# Specify the path for output metric data\n", + "OUTPUT_DATA_PATH = f\"../../../../data/processed/metrics/{METRIC_NAME}\"\n", + "\n", + "# Specify whether or not we are running this as a notebook or part of an automation pipeline.\n", + "AUTOMATION = os.getenv(\"IN_AUTOMATION\")\n", + "\n", + "## CEPH Bucket variables\n", + "## Create a .env file on your local with the correct configs,\n", + "s3_endpoint_url = os.getenv(\"S3_ENDPOINT\")\n", + "s3_access_key = os.getenv(\"S3_ACCESS_KEY\")\n", + "s3_secret_key = os.getenv(\"S3_SECRET_KEY\")\n", + "s3_bucket = os.getenv(\"S3_BUCKET\")\n", + "s3_input_data_path = \"raw_data\"\n", + "metric_path = f\"ai4ci/testgrid/metrics/{METRIC_NAME}\"" + ] + }, + { + "cell_type": "code", + "execution_count": 553, + "id": "3ba80b38-21e9-4f7b-aee3-0095b4259e74", + "metadata": { + "tags": [] + }, + "outputs": [], + "source": [ + "## Import data\n", + "timestamp = datetime.datetime.today()\n", + "\n", + "if AUTOMATION:\n", + " filename = f\"testgrid_{timestamp.day}{timestamp.month}.json\"\n", + " cc = CephCommunication(s3_endpoint_url, s3_access_key, s3_secret_key, s3_bucket)\n", + " s3_object = cc.s3_resource.Object(s3_bucket, f\"{s3_input_data_path}/{filename}\")\n", + " file_content = s3_object.get()[\"Body\"].read().decode(\"utf-8\")\n", + " testgrid_data = json.loads(file_content)\n", + "\n", + "else:\n", + " with gzip.open(INPUT_DATA_PATH, \"rb\") as read_file:\n", + " testgrid_data = json.load(read_file)" + ] + }, + { + "cell_type": "code", + "execution_count": 555, + "id": "00ae3a7e-9298-4ff5-bd8f-60b1ee7950f8", + "metadata": {}, + "outputs": [], + "source": [ + "all_dashboards = list(testgrid_data.keys())\n", + "\n", + "all_jobs = {key: list(testgrid_data[key].keys()) for key in all_dashboards}" + ] + }, + { + "cell_type": "code", + "execution_count": 564, + "id": "1ef6920c-cb47-4e00-bf35-1217632e8d6e", + "metadata": {}, + "outputs": [], + "source": [ + "# for data exploration\n", + "\n", + "def get_random_grid():\n", + " \"\"\"Call this function and get a totally random grid.\"\"\"\n", + " first = np.random.choice(all_dashboards)\n", + " second = np.random.choice(all_jobs[first])\n", + " grid = testgrid_data[first][second]['grid']\n", + " x = np.array(list(pd.DataFrame(grid).statuses.apply(decode_run_length)))\n", + " x = pd.DataFrame(x).apply(lambda x: [normalize(y) for y in x])\n", + " return first, second, x" + ] + }, + { + "cell_type": "markdown", + "id": "085d9dc3-458a-431a-b142-1f8c6a0a2bef", + "metadata": {}, + "source": [ + "## Visualizing" + ] + }, + { + "cell_type": "markdown", + "id": "625f023d-208b-4a2b-bba4-0eb66d0b43f5", + "metadata": {}, + "source": [ + "Below we visualize some improbable failures. We define a metric (probability of failure) that is just the sample probability of failure, $\\overline{X}$. If the probability of failure is < 0.05 and it fails, we mark it in yellow." + ] + }, + { + "cell_type": "code", + "execution_count": 560, + "id": "178c7aff-a995-4c8a-ae01-179436875a92", + "metadata": {}, + "outputs": [], + "source": [ + "def naive_prob_failure(row):\n", + " \"\"\"Input a row and get the probability that test fails, given that it is run.\"\"\"\n", + " row = row.values\n", + " row = row[row != 0]\n", + " return 1 - (row.mean() + 1)/2" + ] + }, + { + "cell_type": "code", + "execution_count": 561, + "id": "d415c229-8a40-4409-bcba-84a743fa7bf1", + "metadata": {}, + "outputs": [], + "source": [ + "def get_grid(x):\n", + " \"\"\"Plot the grid with not run tests in white, improb\n", + " able failed tests in yellow, other failed tests in red, and passed in green.\"\"\"\n", + " plt.figure(figsize=(10, 5))\n", + " y = x[:50].copy()\n", + " for i, row in y.iterrows():\n", + " pf = naive_prob_failure(row)\n", + " if pf <= 0.05:\n", + " row[row == -1] = 0.5\n", + " y.iloc[i] = row\n", + " cmap = colors.ListedColormap(['red', 'white', 'yellow', 'green'])\n", + " boundaries = [-1.2, -0.2, 0.2, 0.7, 1.2]\n", + " norm = colors.BoundaryNorm(boundaries, cmap.N, clip=True)\n", + " sns.heatmap(y[:50], fmt=\"\", cmap=cmap, cbar=False, norm = norm)\n", + " plt.ylabel(\"Tests\")\n", + " plt.xlabel(\"Days\")\n", + " plt.show()" + ] + }, + { + "cell_type": "code", + "execution_count": 562, + "id": "3258601c-ae52-4fe6-a15f-08b82cb2895e", + "metadata": {}, + "outputs": [ + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAl4AAAFCCAYAAAAg3dP7AAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjQuMywgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/MnkTPAAAACXBIWXMAAAsTAAALEwEAmpwYAAAojUlEQVR4nO3de7QsZXnn8e9PLgZELoqiESIqEmO8YAaRaDIgGEOMS5yMSSZeAkaTGe+iidesHHBigsaF0TgmQxSvxDuocUQlRjRmKaiAiqJIUAyKovF+S0Se+aPqxGbbtc/uPl1VvXt/P2vVOt3V71vvU91Vfd5d/db7pKqQJElS/24wdgCSJElbhR0vSZKkgdjxkiRJGogdL0mSpIHY8ZIkSRqIHS9JkqSBjNLxSnJckk8nuTzJ08aIQZIkaWgZeh6vJLsAlwG/AlwFfAj4nar65KCBSJIkDWzXEdo8Ari8qq4ASPJa4Higu+OVOMurpMZ6fywmw8UhSV2qOr+Mxvip8VbAv048v6pdJ0mStNIcXC9JkjSQMTpeXwAOmnh+YLvuepL8QZIPJ/nw6YOFJkmS1J8xBtfvSjO4/liaDteHgAdX1SfWqeQYL0kNx3hJWnbrjPEafHB9VV2b5LHAO4FdgDPW7XRJkiStiMGveM2l44pXTl5cE9WxrUW2sYxtLyPfj+vrej/WvfLTIadsnitCtW36/m2mfVhWY763y/hdO8+5JO3AUt3VKEmStCXZ8ZIkSRrI4B2vJAcleU+STyb5RJInDB2DJEnSGMaYuf5a4MlVdWGSGwMfSXKuKYMkSdKqG/yKV1VdXVUXto+/DVyKM9dLkqQtYNQxXkkOBu4GnD9mHJIkSUMYreOVZC/gTcATq+pbY8UhSZI0lFE6Xkl2o+l0nVlVZ3WUMWWQJElaKYMPrk8S4KXApVV1Wle5qjodOL2t5Ox2kiRp0xvjite9gIcBxyS5uF3uN0IckiRJgxojV+P7WWcqfUmSpFXlzPWSJEkDseMlSZI0kDGnk9glyUVJ3jZWDJIkSUMa84rXE2hmrZckSdoSxsjVSJIDgV8Hng08aYwYpLHUydPXp2O9JGl1jHXF6y+BpwDXjdS+JEnS4AbveCW5P3BNVX1k6LYlSZLGNNYEqg9I8jngtTQTqb56bSFTBkmSpFUzxgSqTweeDpDkaOAPq+qhU8qZMkiSJK2UUQbXS1vZrIPou8r714g0G88lLYNRO15VdR5w3pgxSJIkDcWZ6yVJkgZix0uSJGkgo3S8kuyb5I1JPpXk0iS/OEYckiRJQxprjNcLgHdU1YOS7A7sOVIckiRJgxm845VkH+C/AicCVNV/AP+xyDa6UrJA/2lZFtm2qWU2t0V9fp3H1MmZbUMAM7at5THPd0tnnW07F8tmtchzaVHfw50x1Tr3WmZ6vKvwf8yY/38vUm3r/vzG+KnxNsBXgJcluSjJS5LcaIQ4JEmSBjVGx2tX4BeAv66quwHfBZ42QhySJEmDGqPjdRVwVVWd3z5/I01H7HpMGSRJklbNGCmDvpTkX5P8bFV9GjgW+OSUcqYMkiRJKyW13gC+vhpNDgNeAuwOXAE8vKq+vk6V6UF2DDCcZ1BiZ52u8l0W2fas21mkRcY063u7qM9ibIva70VaxvdwGY//efR93I753TJP213GPL8XeY4N8bnOatU/izHP/dm/zzuDHaXjNQc7XhvZziLZ8dp5drw2ZhmP/3nY8dqYzfaffRc7XjtnC3e8nLlekiRpIHa8JEmSBjJWyqCTknwiySVJXpPkp8aIQ5IkaUiDd7yS3Ap4PHB4Vd0J2AX4H0PHIUmSNLSxcjXuCuyR5Ic0eRq/OFIc6rLIQYzLOhh62Yw56H5Mq7Lfq3ycr/K+aRyb7bxf4Dkw+BWvqvoC8Dzg88DVwDer6l1DxyFJkjS0MX5q3A84niZn408DN0ry0Cnlfjxz/enOXS9Jkja/MX5qvA/w2ar6CkCSs4B7Aq+eLHS9meu75vGSJEnaRMa4q/HzwJFJ9kwSmpRBl44QhyRJ0qDGyNV4fpI3AhcC1wIX8eMrW73LyR1xzVi+y3qX5mZte9btrIpFfRYA1VFniPdwiGNqVpvp2FnkcbAKhvhumXX78xjzc13Gc2nMmJbxHJvnOJ+5jY7tDHGcw0h3NVbVNmDbGG1LkiSNxZnrJUmSBmLHS5IkaSC9dbySnJHkmiSXTKy7SZJzk3ym/Xe/vtqXJElaNn1e8Xo5cNyadU8D3l1Vtwfe3T6XJEnaEnobXF9V70ty8JrVxwNHt49fAZwHPHVH28op06fq77wjo6M8rHM3Ax1tdJTv0rWdedqedTurYlGfxXqGeA+HOKZmtamOnY7bbzbVPizQIr9batts99It8j0f4vyete15LCreeT6jzv8TT15Q2zNuZ5HmOc4XZaHbX+f2waHHeB1QVVe3j78EHDBw+5IkSaMZbXB9VRXrTHUxmTKIDw8YmCRJUk+G7nh9OcktAdp/r+kqWFWnV9XhVXU4hw8WnyRJUm+G7ni9FTihfXwC8JaB25ckSRpNml/8ethw8hqagfT7A1+mGWr2ZuD1wM8AVwK/VVVf28DmpgY5zwDDztQaHQMZ1xuovyidgygzve1VT5ky6mdx8vT1i0xX1GXWY3MeQ7yHizLmcbCM5vpe66rT8b0/5vfdmOd31/sh7YTOA7rPuxp/p+OlY/tqU5IkaZk5c70kSdJAhp65/i+SfCrJx5KcnWTfvtqXJElaNkPPXH8ucKequgtwGfD0HtuXJElaKoPOXF9V75p4+kHgQT013v1azwM455ltuHtjHduaYx+qYxbdnDLzplbaQm9cGHEAs1bQPN9rDhq/vgW+H4s6j8e8eWYZb2BZ6P+hI1pvP8Yc4/V7wDkjti9JkjSoUTpeSZ4JXAucOUb7kiRJYxi845XkROD+wENqnUnEJlMGnX766YPFJ0mS1JfexnhNk+Q44CnAUVX1vfXKVtXpwPYelwMVJEnSptfndBKvAT4A/GySq5I8AngRcGPg3CQXJ/mbvtqXJElaNkPPXP/SvtrTDnTdDHLykEFoVcyTvmYZLTJF1Jg6P4+Ou5ln3c5CU2YtKCaYI1XSjG1r5w1xjs3axtjHuTPXS5IkDcSOlyRJ0kAGTRk08dqTk1SS/ftqX5IkadkMnTKIJAcB9wU+32PbkiRJS2fQlEGt59NMKfGWvtrWT+ocZGgqIc1hsw0+77Lq+zHrPDyLfD/GjGlRbWvnDXGOzdrG2Mf5oGO8khwPfKGqPjpku5IkSctgsAlUk+wJPIPmZ0ZJkqQtZ8grXrcDbgN8NMnngAOBC5PcYlphUwZJkqRVM9gVr6r6OHDz7c/bztfhVfXVjvKmDJIkSStl6JRBkiRJW9bQKYMmXz+4r7YlSZKWkTPXS5IkDcSOlyRJ0kAGTxmU5HFJPpXkE0me21f7kiRJy2bQlEFJ7g0cD9y1qn4eeF6P7UuSJC2VVPU3U0ObMuhtVXWn9vnrgdOr6h9m3JTTSfQlmb6+x+NCkqQV1/Gf6/BjvA4FfjnJ+Unem+TuA7cvSZI0msEmUJ1o7ybAkcDdgdcnuW31edlNkiRpSQx9xesq4KxqXABcB+w/raApgyRJ0qoZ+orXm4F7A+9JciiwO2DKIEmStCX01vFqUwYdDeyf5CpgG3AGcEY7xcR/ACf4M6MkSdoqer2rcYE2RZCbknc1SpK0aEtzV6MkSdKWZcdLkiRpIIOmDEpyWJIPJrm4vWPxiL7alyRJWjaDpgwCngucUlWHAX/SPpckSdoSeut4VdX7gK+tXQ3s3T7eB/hiX+1LkiQtm6Hn8Xoi8M4kz6Pp9N1z4PYlSZJGM/Tg+kcBJ1XVQcBJwEsHbl+SJGk0Q3e8TgDOah+/AegcXG/KIEmStGqG/qnxi8BRwHnAMcBnugqaMkiSJK2a3maun0wZBHyZJmXQp4EX0HT4fgA8uqo+soHN2fHqizPXS5K0aJ0z12/qlEE5Zfp+1bbufZq1Tlf5Lotse9btzKO2dbWxsCZmtqjPYmxDHFOz2kzv4aocB4uy2b5buoz5uS7juTRmTMt4js1znC+j2lamDJIkSRqbHS9JkqSB9Jky6KAk70nyySSfSPKEdv1Nkpyb5DPtv/v1FYMkSdIy6fOK17XAk6vqjsCRwGOS3BF4GvDuqro98O72uSRJ0srrbTqJqroauLp9/O0klwK3Ao6nudsR4BU0U0s8ta84tFjLOBhfkqTNYpAxXkkOBu4GnA8c0HbKAL4EHDBEDJIkSWPrveOVZC/gTcATq+pbk69VM5fFppjPQpIkaWf12vFKshtNp+vMqtqeKujLSW7Zvn5L4JqOuqYMkiRJK6W3MV5JQpME+9KqOm3ipbfS5Gw8tf33LdPqmzJIkiStmj5zNd4LeBjw8SQXt+ueQdPhen2SRwBXAr/VYwxaMAfRS5I0vz7vanw/3bmKju2rXUmSpGXlzPWSJEkDseMlSZI0kDFSBv1Fkk8l+ViSs5Ps21cMkiRJy2SMlEHnAneqqrsAlwFP7zEGSZKkpdFbx6uqrq6qC9vH3wYuBW5VVe+qqmvbYh8EDuwrBkmSpGUyRsqgSb8HnDNEDJIkSWMbLWVQkmfS/Bx5Zt8xSJIkLYMxUgaR5ETg/sBD2nyN0+qaMkiSJK2UwVMGJTkOeApwVFV9r6u+KYMkSdKqmemKV5L9ktxlg8W3pww6JsnF7XI/4EXAjYFz23V/M1vIkiRJm9MOr3glOQ94QFv2I8A1Sf65qp60Xr11Uga9fY44JUmSNr2NXPHapx0U/xvAK6vqHsB9+g1LkiRp9Wyk47VrklsCvwW8red4JEmSVtZGOl6nAO8ELq+qDyW5LfCZHVXqShk08fqTk1SS/ecLXZIkaXPZyF2NV7fpfQCoqiuSnLZehdb2lEEXJrkx8JEk51bVJ5McBNwX+Px8YUuSJG0+G7ni9VcbXHc9XSmD2pefTzOlhNNESJKkLaPzileSXwTuCdwsyeQdjHsDu8zSyGTKoCTHA1+oqo82U31JkiRtDetd8dod2Iumc3bjieVbwIM22sBkyiCanx+fAfzJBuo5c70kSVopnVe8quq9wHuTvLyqrgRIcgNgr8mci+tZmzIoyZ2B2wDbr3YdCFyY5Iiq+tKa9p25XpIkrZSNjPH68yR7J7kRcAnwySR/tKNK01IGVdXHq+rmVXVwVR0MXAX8wtpOlyRJ0iraSMfrju0VrgcC59BcsXrYBup1pQySJEnakjYyncRu7U+GDwReVFU/TLLDn/7WSRk0WebgjQQpSZK0CjZyxev/Ap8DbgS8L8mtaQbYS5IkaQY7vOJVVS8EXjix6sok9+4vJEmSpNW0wyteSQ5I8tIk57TP7wicsIF6nSmDkjwuyafa9c/dqT2QJEnaJDYyxuvlwMuAZ7bPLwNeR3PH4nqmpgwCDgCOB+5aVf+e5OZzRS5JkrTJdF7xSrK9U7Z/Vb0euA6gqq4FfrSjDa+TMuhRwKlV9e/ta9fs1B5IkiRtEuv91HhB++93k9yUdhLTJEcC35ylkcmUQcChwC8nOT/Je5PcfeaoJUmSNqH1Ol7bp4J4EvBW4HZJ/hl4JfC4jTYwmTKonQ9sV+AmwJHAHwGvz5SkjaYMkiRJq2a9MV6TybHPBt5O0xn7d+A+wMd2tPG1KYPa1VcBZ1VVARckuQ7YH/jKZF1TBkmSpFWz3hWvXWiSZN+YZg6vXdt1e7br1jUtZVDrzcC92zKH0iTj/uocsUuSJG0q613xurqqnrUT296eMujjSS5u1z0DOAM4I8klwH8AJ7RXvyRJklbaeh2vddP97MgOUgY9dGe2LUmStBmt91PjsYNFIUmStAV0dryq6ms7s+GumeuTHJbkg0kubu9aPGJn2pEkSdosNjJz/by6Zq5/LnBKVZ2T5H7t86N7jEOSJGkp9Nbxqqqrgavbx99Osn3m+gL2bovtA3yxrxgkSZKWSZ9XvP7Tmpnrnwi8M8nzaH7qvOcQMUiSJI1tvcH1CzFl5vpHASdV1UHASew42bYkSdJK6LXj1TFz/QnA9sdvAKYOrjdlkCRJWjW9/dS4zsz1XwSOAs4DjgE+M62+KYMkSdKq6XOMV9fM9b8PvCDJrsAPgD/oMQZJkqSl0eddjevNXP9f+mpXkiRpWfU+uF6SJEkNO16SJEkD6a3jleSnklyQ5KNtyqBT2vW3SXJ+ksuTvC7J7n3FIEmStEz6vOL178AxVXVX4DDguCRHAs8Bnl9VhwBfBx7RYwySJElLo7eOVzW+0z7drV2KZgqJN7brXwE8sK8YJEmSlknfE6ju0k4lcQ1wLvAvwDeq6tq2yFU0+RslSZJWXq8dr6r6UVUdBhxIM0P9HfpsT5IkaZkNcldjVX0DeA/wi8C+7eSp0HTIvjCtjimDJEnSqukzZdDNgB9W1TeS7AH8Cs3A+vcADwJeS5O38S3T6psySJIkrZo+UwbdEnhFkl1orqy9vqreluSTwGuT/ClwEU0+R0mSpJXXZ8qgjwF3m7L+CprxXpIkSVuKM9dLkiQNxI6XJEnSQMZIGXRmkk8nuSTJGUl26ysGSZKkZTJGyqAzaebzujOwB/DIHmOQJElaGn0Ori/gJ1IGVdXbt5dJcgHNXF6SJEkrb9CUQVV1/sRruwEPA97RZwySJEnLYtCUQUnuNPHyi4H3VdU/9RmDJEnSshg6ZdBxAEm2ATcDntRVx5RBkiRp1QyeMijJI4FfBY6tquu66psySJIkrZoxUgZdC1wJfCAJwFlV9awe45AkSVoKY6QM6rOzJ0mStLScuV6SJGkgdrwkSZIGMnjKoInXX5jkO131JUmSVk2f4622pwz6TjtZ6vuTnFNVH0xyOLBfj21LkiQtnd6ueFXjJ1IGtXc5/gXwlL7aliRJWkZjpAx6LPDWqrq6z7YlSZKWTa9TO1TVj4DDkuwLnJ3kvwK/CRzdZ7uSJEnLaOiUQfcGDgEuT/I5YM8kl0+rY8ogSZK0agZPGVRVt5go852qOmRafVMGSZKkVTN4yqAe25MkSVpqg6cMWlNmr77a18Zcf3a1Hatti9mOJElbkTPXS5IkDcSOlyRJ0kAGTxmUxrOTXJbk0iSP7ysGSZKkZTJ4yiDg54CDgDtU1XVJbt5jDJIkSUujz8H1BfxEyiDgUcCDq+q6ttw1fcUgSZK0TMZIGXQ74LfbyVHPSXL7PmOQJElaFr12vKrqR1V1GHAgcESSOwE3BH5QVYcDfwuc0WcMkiRJy2LolEHHAVcBZ7UvnQ3cZVodUwZJkqRVM3jKIODNNDkbPwscBVw2rb4pgyRJ0qpJMwa+hw0ndwFeAUymDHpWkn2BM4GfoRl8/7+q6qM72Jwdr2WRTF/f03EkSdIm1PGfZY8drwXbFEFuCXa8JEnakc6OlzPXS5IkDcSOlyRJ0kDGSBl0bJILk1yc5P1JDukrBkmSpGXS5+D6ADeaTBkEPAF4JXB8VV2a5NHAEVV14g425wCiZeEYL0mSdqRzjNcYKYMK2Ltdvw/wxXnbyCnT96u2dXcCZq3TVb7LItuedTuDOLlj/QJjmvWzqJO7t5WO19ars7DtdHRGF3lMzWrUY2dGizonV8WY3y0LPccWdF7MY4hzadbvlkX+0bqw75auP7Dp/rw725ix/Hrvx8z719H2rPswz7bWO9b6TJJNkl2AjwCHAP+nqs5P8kjg7Um+D3wLOLLPGCRJkpbFGCmDTgLuV1UHAi8DTuszBkmSpGUxdMqgXwPu2ibLBngdcM9pdUwZJEmSVs0YKYP2SXJoVV3Wrrt0Wn1TBkmSpFXT5xivWwKvaMd5bU8Z9LYkvw+8Kcl1wNeB3+sxBm0B6w2UXOTgypnb7lrfUWdRMUnzmufYnHnQ8cbD2ZRmHnzeSxQ7Z4jB553bmb3pTrPe+LFerIv8fu7zrsaPAXebsv5s4Oy+2pUkSVpWzlwvSZI0EDtekiRJA+m945VklyQXJXlb+/w2Sc5PcnmS1yXZve8YJEmSlsEQV7yewPXvXHwO8PyqOoRmcP0jBohBkiRpdH3PXH8g8OvAs4EntfkbjwEe3BZ5BU0Smr/uMw5tYV23yJwyaBTX492LWlpznC8ez4LNdRyMHWvfV7z+EngKcF37/KbAN6rq2vb5VcCteo5BkiRpKfTW8Upyf+CaqvrInPWduV6SJK2UPn9qvBfwgCT3A34K2Bt4AbBvkl3bq14HAl+YVtmZ6yVJ0qrp7YpXVT29qg6sqoOB/wH8Y1U9hCZn44PaYicAb+krBkmSpGXS6+D6Dk8FXpvkT4GLgJeOEIO2inSsP3nIIKRNwvNlJXWlyGFb/22MPZB9mrFjHaTjVVXnAee1j68AjhiiXUmSpGXizPWSJEkDseMlSZI0kDFSBp2Z5NNJLklyRpLd+o5BkiRpGYyRMuhM4A7AnYE9gEcOEIMkSdLoBk0ZBFBVb594/QKaubykfixhyiBpWS3jHWjaeV2f6yInyNxMx87YsQ6dMug/tT8xPgx4R88xSJIkLYUxUwa9GHhfVf1TR31TBkmSpJUyaMqgJK+uqocm2QbcDPifXZVNGSRJklbN0CmDHprkkcCvAr9TVT/xE6QkSdKqGiNl0N8AVwIfSAJwVlU9a4Q4pFFUR5qOOOB/afgZSerLGCmDxujsSZIkjc6Z6yVJkgYy+Mz1E+tfmOQ7fbcvSZK0LMaYuZ4khwP7DdC2JEnS0hh85vokuwB/ATwY+G99ti+RjvUnDxnE9TlAexNYwuNmCHXy9PVjz/St5eexs3FjzFz/WOCtVXV1z21LkiQtlUFnrk/y08BvAn/VV7uSJEnLqs8rXttnrv8c8FrgGOATwCHA5e36PZNcPq2yKYMkSdKq6W2MV1U9HXg6QJKjgT+sqvtPlknynao6pKO+KYMkSdJKcR4vSZKkgQw+c/2a9XsN0b62sK5rpd5ZqHVs1Tuxtup+a+d57GycV7wkSZIGYsdLkiRpIIOnDErj2UkuS3Jpksf3HYMkSdIyGGKM1/aUQXu3z08EDgLuUFXXJbn5ADFIkiSNrtcrXhMpg14ysfpRwLOq6jqAqrqmzxgkSZKWxRgpg24H/HY7Oeo5SW7fcwySJElLYdCUQa0bAj+oqsOBvwXO6CsGSZKkZTJoyqAkrwauAs5qy5wN3GVaZVMGSZKkVTN0yqCHJjkVuDfwWeAo4LKO+qYMkiRJK2WQmevXOBU4M8lJwHeAR44QgyRJ0uAGTxlUVd+gudNR6l861p88ZBDXVx1tm3JD0ma1mb7Xxo7VmeslSZIGYsdLkiRpIGOkDDo2yYVJLk7y/iSH9B2DJEnSMhjiitf2lEHb/TXwkKo6DPg74I8HiEGSJGl0vQ6un0gZ9GzgSe3q4sd5G/cBvthnDNriuiYiOWXQKK5nGQeb6vpq2/T1GfG4kZbZpvpeG/n/hb7vavxLmpRBN55Y90jg7Um+D3wLOLLnGCRJkpbCGCmDTgLuV1UHAi8DTusrBkmSpGUydMqg/wfctarOb8u8DrjntMqmDJIkSatm0JRBwAOBLyU5tKouA36F6w+8n6xvyiBJkrRSBk0ZVFXXJvl94E1JrgO+DvzekDFIkiSNZYyUQWcDZw/RriTNw7sXh+edpNoqnLlekiRpIHa8JEmSBmLHS5IkaSB2vCRJkoZSVZtqAf6gz/Kr0sYyxuR+L0/5VWljGWNyv5en/Kq0sYwxud+z1ble/Z2pPMYCfLjP8qvSxjLG5H4vT/lVaWMZY3K/l6f8qrSxjDG537PVmVz8qVGSJGkgdrwkSZIGshk7XrMmbpwn0eMqtLGMMQ3RxjLGNEQbyxjTEG0sY0xDtLGMMQ3RxjLGNEQbyxjTEG0sY0zz1vlPaX+vlCRJUs824xUvSZKkTcmOlyRJ0kDseEmSJA1k6TteSe6Q5KlJXtguT03yczsof2ySvdasP26D7b1yB6/fI8ne7eM9kpyS5O+TPCfJPlPK757kd5Pcp33+4CQvSvKYJLttJCb1K8nNZyx/075ikfqyVY9z93vD5VdivzeDpe54JXkq8FogwAXtEuA1SZ42pfzjgbcAjwMuSXL8xMt/NqX8W9csfw/8xvbnHWGdAXyvffwCYB/gOe26l00p/zLg14EnJHkV8JvA+cDdgZes+wYMbOwTNck+SU5N8qkkX0vyb0kubdftO6X83kn+PMmrkjx4zWsv7mjjJmuWmwIXJNkvyU2mlD81yf7t48OTXAGcn+TKJEdNKX94kvckeXWSg5Kcm+SbST6U5G4dMe2a5H8meUeSj7XLOUn+17TOeZJd2vL/O8m91rz2x9Pa6Gj3snVee+zEfh+S5H1JvpHk/CR3nlL+tknOSPKnSfZK8rdJLknyhiQH7+w+L2q/19vnIfa7rbNVj3P3e2vt9+Dfa1mC83tDdmb21b4X4DJgtynrdwc+M2X9x4G92scHAx8GntA+v2hK+QuBVwNHA0e1/17dPj6qI6ZLJ+uvee3iKeU/1v67K/BlYJf2eba/NqXOPsCpwKeArwH/Blzartu3o87ewJ8DrwIevOa1F08pf5M1y02BzwH7ATeZUv5UYP/28eHAFcDlwJXrvFeHA+9p3+ODgHOBbwIfAu42pfw7gacCt5hYd4t23bumlH9TG9cDgbe2z2847bOZqHMd8Nk1yw/bf6+YdkxNPH4PcPf28aFMmb2Y5o+DXwN+B/hX4EHt+mOBD3TE9Brgr4EjgQPb5ch23eumlH8J8HfAE4GPAKd1HZMT678NfKtdvt0uP9q+fkr5T0w8/n/Af2sfHw3885Ty7wMeBTwNuAR4cvuZPwL4x53d53n2e9Z9HmK/t/hx7n5vrf3u9XuNJT2/N7LMVWmohabjcesp628NfHq9N7V9vhfwDuA0pneKbgCcRNMhOKxd9xMH6Zo6bwAe3j5+GXD4xAH7oSnlL6HpKO7XHhA3adf/FBOduDV1ZjpR29dnOln7PlHb12Y6Wad9puu9tvYzBZ4J/DNNJ7LrC+rJ7TFx54l1n12n3UuBXdvHH+x6TybWXTTx+PNdr61Zf9k67f/Ea0x02Gk69KcDZwE3XKeNFwKvBA7Y4H5/euLxh7ran3e/Z93nefZ71n0eYr/XtrGR11boOHe/t9Z+9/q9tqzn90aWuSoNtQDH0VxVOaf9EE5vD67LgeOmlP9H2g7Umg/wlcCP1mnnQJoO1YvWvrlTyu4DvBz4F5qfDH9Ic/XnvcBdp5Q/qX39SuDxwLuBv6W5OrdtRwfHRl+b9WTt+0Sd56AF3gU8Zc2JdABNh/MfOmK6wZp1JwKfAK7cwOd9GnBj1uls0/xs/S7gGOBkmp+XjwJOAV41pfwHgPvS/KR8JfDAdv1RdHdQP9iWv8HEuhsAvw2cP6X8p6as29Z+3j9xJXiizH9pz5HHt9tfb7+f3R7ntwWeQfNX6K2BhwNvm1L+IzSd8COAr/LjP0gOYfoX2kz7PO9+z7LPO7nfd9/Ifm/x49z93lr73fv3GsOd32u/125Px/m9kWWuSkMu7Zt5JPDf2+VI2p/rOg68W3S8dq8NtPXrwJ9tMK69gbu2H/wBOyj708BPt4/3BR4EHLFO+ZlO1Pb1mU/WPk/Uts5MJyvNVcHn0Fzp/DrNz6yXtuum/fz5XOA+U9Yf13Wirin3gPbL4Us7KHc08DrgIpoO89uBP2D6z+B3pblieQ5wh/Z9+kb7OdyzY/sHt9u/hubn9cvax68DbjOl/KuZ/ofHI4EfbuB8ejzwT8AXd1D2RJo/Lr5Kc7X2kzRjJfeZUvZY4NPt5/VLNFdcP9Pux/Hr7PNX2v3dXnbqPu/Mfs+yz235hy9ovx/Ysf1VOM4Pm3Kcf709zqd+167Ifk87vzfrft97J/f7GyzB9xqzn98n0tP32kaXuSq59LusOVG/tuZE3a+jztwn6wK+oHbtKD/PyXoH4D60Y/Um92Od8sdOKf9r6+zHf9YB9gDuNGcbXeV/bpby7Wv3oPmr6qbAvYA/BO63Tvkj+PHPvXcEnrRe+Sl1fhn4kxna+Hmaq6Trlb/HmvLr7sNEvZu2y6vnOFdeOUPZWwL/NkcbU/+wWKf821jzR9AOyv9y+97ed4Plf6n9vDdUfqKNP56hjVlj2mH59vjYp328J/Cs9r16DtP/05ssv0db/u+7yk/U2XuijecC/7CDNvbeyZg6y7flHg8cNMNnNVP5aXWY+F4bMabdgROAX2nP74cALwYew/TO3Q2B36X9fwx4MM2vUI8Bdu/Y/mT5h9H8uvToadvvqPMQ4P+sE9P2fZgs37kPG11MGbTJJHl4Vb1s0XWS7AHcrqoumbWNRcWU5q7Ux9B0Mg+juTHiLe1rF1bVL6wp/zjgsRstP2cb85R/NE2neaMxbaMZC7crzXjDI4DzaL6w3llVz95B+XvQjLubWn5BbSy6/LS7ho+h+dmAqnrAlH1YWyc0f7VPrbOgNtatM2cbF1TVEe3jR9IcX2+muTr891V16jrlf78tf3ZX+Y46j56hjXliWnf7bblP0AzHuDbJ6cB3aa4gHNuu/40dlP8e8Mau8gtqY6Hl2zrfbMv9C83g8TdU1VfXluso/5q2/Fe6ynfUef0MbfQV05k03wd70NxUdSOa4/ZYmnSFJ3SU35PmD/S9aMZ4HQtQVSfuoPy625+nzqz7sGHz9thcxlnYwRi0RdTpu3xXHWa/K3Wm8kO0sRMx7ULzZfAtfvwX+B5MHx81U/kh2pij/Dx3FF80S50525ipzqwxrT0OaO7wvVn7+EbseGDzDssP0cacMc16R/hM5YdoY86YLqL5Oey+wEtpfl5/B82VlBvvbPkh2pgzppnu6O+7/FBtbGTZFS2dJB/reolmrNdO1+m7/Jx1blBV3wGoqs8lORp4Y5Jbt3V2tvwQbcwT07VV9SPge0n+paq+1db/fpLrFlB+iDZmLX848ASam0D+qKouTvL9qnpvR/zQjKecpc48bcxaZ9aYAG6QZD+a/8hS7ZWDqvpukmsXUH6INuaJafJq+keTHF5VH05yKM1NSjtbfog25ompquo6mjGy70ozh9X2u72fB9xsJ8sP0cY8Md0gye40nfE9aW5M+xrNT4rT5urru/xQbezYvD02l/4Wmp71YTR3W0wuB9MxeHDWOn2Xn7ONme5KnbX8EG3MGdP5wJ7t48k7gPZh+h2pM5Ufoo15Ympf3/AdxfPWWbY2aObLu4J26hbglu36vZh+lWWm8kO0MWdM+zDbHeEzlR+ijTljumidY2HPnS0/RBtzxjTTHf19lx+qjY0sc1Vy6XehuZT7Sx2v/d0i6vRdfs42ZrorddbyQ7QxZ0w37Fi/PxPTfcxbfog25olpTbkN31E8b51lbWOi7p503NG5iPJDtLGR8sxwR/g85YdoY5bywKEzfkYzlR+ijXliauvNekd/r+WHamNHi4PrJUmSBrLUuRolSZJWiR0vSZKkgXhXo6SVkeRHNANfdwOupbmp4fnV3JElSaOz4yVplXy/qg4DSHJzmskh96bJ+SZJo/OnRkkrqaquoUlp9dg0Dk7yT0kubJd7AiR5ZZIHbq+X5Mwkxyf5+SQXJLk4yceS3H6kXZG0QryrUdLKSPKdqtprzbpvAD9LkxD3uqr6QduJek1VHZ7kKOCkqnpgkn2Ai4HbA88HPlhVZ7aTKO5SVd8fcn8krR5/apS0VewGvCjJYcCPgEMBquq9SV6c5GbAfwfeVE0uvg8Az0xyIHBWVX1mrMAlrQ5/apS0spLclqaTdQ3NLNRfppn48nBg94mirwQeCjwcOAOgqv4OeADwfeDtSY4ZLnJJq8orXpJWUnsF62+AF1VVtT8jXlVV1yU5gSap93YvBy4AvlRVn2zr3xa4oqpemORngLvQpISSpLnZ8ZK0SvZIcjE/nk7iVcBp7WsvBt6U5HeBdwDf3V6pqr6c5FLgzRPb+i3gYUl+CHwJ+LPeo5e08hxcL2nLS7Inzfxfv1BV3xw7HkmryzFekra0JPcBLgX+yk6XpL55xUuSJGkgXvGSJEkaiB0vSZKkgdjxkiRJGogdL0mSpIHY8ZIkSRqIHS9JkqSB/H99s7ABAxX92QAAAABJRU5ErkJggg==\n", + "text/plain": [ + "
" + ] + }, + "metadata": { + "needs_background": "light" + }, + "output_type": "display_data" + } + ], + "source": [ + "dashboard, job, x = get_random_grid()\n", + "get_grid(x)" + ] + }, + { + "cell_type": "markdown", + "id": "1669cb1c-ae87-43c1-aa68-e0c2f2566fca", + "metadata": {}, + "source": [ + "## Unexpected Failure Classification" + ] + }, + { + "cell_type": "markdown", + "id": "fa1efee6-fb4c-4686-9837-ebd676e06ceb", + "metadata": {}, + "source": [ + "First we will walk through the process with visualizations to motivate our methods before defining an overall function to calculate an \"unexpectedness score\"." + ] + }, + { + "cell_type": "code", + "execution_count": 580, + "id": "3a71e3f1-3a1f-4799-8b3f-5489dad8dcd2", + "metadata": {}, + "outputs": [], + "source": [ + "def preprocess(x):\n", + " \"\"\"Preprocess the data to make our methods work.\"\"\"\n", + " # unsure why we have NA values; safest bet to fix is to assume test didn't run\n", + " x = x.fillna(0)\n", + " # if we have all the same values, there is no real info (for our methods)\n", + " x = x[~(x.apply(np.std, axis = 1) == 0)]\n", + " return x\n", + "\n", + "\n", + "x = preprocess(x)" + ] + }, + { + "cell_type": "markdown", + "id": "f5ec74bb-0efd-435a-8b45-1a00445d7750", + "metadata": {}, + "source": [ + "Now we define a column score. We earlier defined the row score as the sample probability of a test failing. Rows correspond to tests and columns refer to time. For the column score, we take all the tests that failed at that time, and take the sums of the entropies of the failed scores. Entropy is just the negative of the log of the row score. It just makes probabilities work in a more linear fashion. So if a 2 tests fail that had a $\\frac25$ probability of failing and a test fails that had a $\\frac13$ probability of failing our column score will be $-2\\log\\frac25 - \\log\\frac13$." + ] + }, + { + "cell_type": "code", + "execution_count": 581, + "id": "968ce931-73a7-4f0b-873d-33e0e41a4781", + "metadata": {}, + "outputs": [], + "source": [ + "def column_scores(x):\n", + " # defined as the sum of the log of probability to fail of the failed tests\n", + " row_scores = x.apply(naive_prob_failure, axis = 1).values\n", + " return x.apply(lambda x: column_score(x, row_scores))\n", + "\n", + "\n", + "def column_score(column, row_scores):\n", + " return np.sum(-np.log(row_scores[column == -1]))" + ] + }, + { + "cell_type": "markdown", + "id": "27bb49d0-01f7-464e-bef6-44c28e72efb9", + "metadata": {}, + "source": [ + "Now let's visualize what our column scores look like next to the grid itself." + ] + }, + { + "cell_type": "code", + "execution_count": 572, + "id": "46df01ac-793a-43bb-a4d2-e46136720cbb", + "metadata": {}, + "outputs": [ + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAl4AAAFCCAYAAAAg3dP7AAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjQuMywgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/MnkTPAAAACXBIWXMAAAsTAAALEwEAmpwYAAAojUlEQVR4nO3de7QsZXnn8e9PLgZELoqiESIqEmO8YAaRaDIgGEOMS5yMSSZeAkaTGe+iidesHHBigsaF0TgmQxSvxDuocUQlRjRmKaiAiqJIUAyKovF+S0Se+aPqxGbbtc/uPl1VvXt/P2vVOt3V71vvU91Vfd5d/db7pKqQJElS/24wdgCSJElbhR0vSZKkgdjxkiRJGogdL0mSpIHY8ZIkSRqIHS9JkqSBjNLxSnJckk8nuTzJ08aIQZIkaWgZeh6vJLsAlwG/AlwFfAj4nar65KCBSJIkDWzXEdo8Ari8qq4ASPJa4Higu+OVOMurpMZ6fywmw8UhSV2qOr+Mxvip8VbAv048v6pdJ0mStNIcXC9JkjSQMTpeXwAOmnh+YLvuepL8QZIPJ/nw6YOFJkmS1J8xBtfvSjO4/liaDteHgAdX1SfWqeQYL0kNx3hJWnbrjPEafHB9VV2b5LHAO4FdgDPW7XRJkiStiMGveM2l44pXTl5cE9WxrUW2sYxtLyPfj+vrej/WvfLTIadsnitCtW36/m2mfVhWY763y/hdO8+5JO3AUt3VKEmStCXZ8ZIkSRrI4B2vJAcleU+STyb5RJInDB2DJEnSGMaYuf5a4MlVdWGSGwMfSXKuKYMkSdKqG/yKV1VdXVUXto+/DVyKM9dLkqQtYNQxXkkOBu4GnD9mHJIkSUMYreOVZC/gTcATq+pbY8UhSZI0lFE6Xkl2o+l0nVlVZ3WUMWWQJElaKYMPrk8S4KXApVV1Wle5qjodOL2t5Ox2kiRp0xvjite9gIcBxyS5uF3uN0IckiRJgxojV+P7WWcqfUmSpFXlzPWSJEkDseMlSZI0kDGnk9glyUVJ3jZWDJIkSUMa84rXE2hmrZckSdoSxsjVSJIDgV8Hng08aYwYpLHUydPXp2O9JGl1jHXF6y+BpwDXjdS+JEnS4AbveCW5P3BNVX1k6LYlSZLGNNYEqg9I8jngtTQTqb56bSFTBkmSpFUzxgSqTweeDpDkaOAPq+qhU8qZMkiSJK2UUQbXS1vZrIPou8r714g0G88lLYNRO15VdR5w3pgxSJIkDcWZ6yVJkgZix0uSJGkgo3S8kuyb5I1JPpXk0iS/OEYckiRJQxprjNcLgHdU1YOS7A7sOVIckiRJgxm845VkH+C/AicCVNV/AP+xyDa6UrJA/2lZFtm2qWU2t0V9fp3H1MmZbUMAM7at5THPd0tnnW07F8tmtchzaVHfw50x1Tr3WmZ6vKvwf8yY/38vUm3r/vzG+KnxNsBXgJcluSjJS5LcaIQ4JEmSBjVGx2tX4BeAv66quwHfBZ42QhySJEmDGqPjdRVwVVWd3z5/I01H7HpMGSRJklbNGCmDvpTkX5P8bFV9GjgW+OSUcqYMkiRJKyW13gC+vhpNDgNeAuwOXAE8vKq+vk6V6UF2DDCcZ1BiZ52u8l0W2fas21mkRcY063u7qM9ibIva70VaxvdwGY//efR93I753TJP213GPL8XeY4N8bnOatU/izHP/dm/zzuDHaXjNQc7XhvZziLZ8dp5drw2ZhmP/3nY8dqYzfaffRc7XjtnC3e8nLlekiRpIHa8JEmSBjJWyqCTknwiySVJXpPkp8aIQ5IkaUiDd7yS3Ap4PHB4Vd0J2AX4H0PHIUmSNLSxcjXuCuyR5Ic0eRq/OFIc6rLIQYzLOhh62Yw56H5Mq7Lfq3ycr/K+aRyb7bxf4Dkw+BWvqvoC8Dzg88DVwDer6l1DxyFJkjS0MX5q3A84niZn408DN0ry0Cnlfjxz/enOXS9Jkja/MX5qvA/w2ar6CkCSs4B7Aq+eLHS9meu75vGSJEnaRMa4q/HzwJFJ9kwSmpRBl44QhyRJ0qDGyNV4fpI3AhcC1wIX8eMrW73LyR1xzVi+y3qX5mZte9btrIpFfRYA1VFniPdwiGNqVpvp2FnkcbAKhvhumXX78xjzc13Gc2nMmJbxHJvnOJ+5jY7tDHGcw0h3NVbVNmDbGG1LkiSNxZnrJUmSBmLHS5IkaSC9dbySnJHkmiSXTKy7SZJzk3ym/Xe/vtqXJElaNn1e8Xo5cNyadU8D3l1Vtwfe3T6XJEnaEnobXF9V70ty8JrVxwNHt49fAZwHPHVH28op06fq77wjo6M8rHM3Ax1tdJTv0rWdedqedTurYlGfxXqGeA+HOKZmtamOnY7bbzbVPizQIr9batts99It8j0f4vyete15LCreeT6jzv8TT15Q2zNuZ5HmOc4XZaHbX+f2waHHeB1QVVe3j78EHDBw+5IkSaMZbXB9VRXrTHUxmTKIDw8YmCRJUk+G7nh9OcktAdp/r+kqWFWnV9XhVXU4hw8WnyRJUm+G7ni9FTihfXwC8JaB25ckSRpNml/8ethw8hqagfT7A1+mGWr2ZuD1wM8AVwK/VVVf28DmpgY5zwDDztQaHQMZ1xuovyidgygzve1VT5ky6mdx8vT1i0xX1GXWY3MeQ7yHizLmcbCM5vpe66rT8b0/5vfdmOd31/sh7YTOA7rPuxp/p+OlY/tqU5IkaZk5c70kSdJAhp65/i+SfCrJx5KcnWTfvtqXJElaNkPPXH8ucKequgtwGfD0HtuXJElaKoPOXF9V75p4+kHgQT013v1azwM455ltuHtjHduaYx+qYxbdnDLzplbaQm9cGHEAs1bQPN9rDhq/vgW+H4s6j8e8eWYZb2BZ6P+hI1pvP8Yc4/V7wDkjti9JkjSoUTpeSZ4JXAucOUb7kiRJYxi845XkROD+wENqnUnEJlMGnX766YPFJ0mS1JfexnhNk+Q44CnAUVX1vfXKVtXpwPYelwMVJEnSptfndBKvAT4A/GySq5I8AngRcGPg3CQXJ/mbvtqXJElaNkPPXP/SvtrTDnTdDHLykEFoVcyTvmYZLTJF1Jg6P4+Ou5ln3c5CU2YtKCaYI1XSjG1r5w1xjs3axtjHuTPXS5IkDcSOlyRJ0kAGTRk08dqTk1SS/ftqX5IkadkMnTKIJAcB9wU+32PbkiRJS2fQlEGt59NMKfGWvtrWT+ocZGgqIc1hsw0+77Lq+zHrPDyLfD/GjGlRbWvnDXGOzdrG2Mf5oGO8khwPfKGqPjpku5IkSctgsAlUk+wJPIPmZ0ZJkqQtZ8grXrcDbgN8NMnngAOBC5PcYlphUwZJkqRVM9gVr6r6OHDz7c/bztfhVfXVjvKmDJIkSStl6JRBkiRJW9bQKYMmXz+4r7YlSZKWkTPXS5IkDcSOlyRJ0kAGTxmU5HFJPpXkE0me21f7kiRJy2bQlEFJ7g0cD9y1qn4eeF6P7UuSJC2VVPU3U0ObMuhtVXWn9vnrgdOr6h9m3JTTSfQlmb6+x+NCkqQV1/Gf6/BjvA4FfjnJ+Unem+TuA7cvSZI0msEmUJ1o7ybAkcDdgdcnuW31edlNkiRpSQx9xesq4KxqXABcB+w/raApgyRJ0qoZ+orXm4F7A+9JciiwO2DKIEmStCX01vFqUwYdDeyf5CpgG3AGcEY7xcR/ACf4M6MkSdoqer2rcYE2RZCbknc1SpK0aEtzV6MkSdKWZcdLkiRpIIOmDEpyWJIPJrm4vWPxiL7alyRJWjaDpgwCngucUlWHAX/SPpckSdoSeut4VdX7gK+tXQ3s3T7eB/hiX+1LkiQtm6Hn8Xoi8M4kz6Pp9N1z4PYlSZJGM/Tg+kcBJ1XVQcBJwEsHbl+SJGk0Q3e8TgDOah+/AegcXG/KIEmStGqG/qnxi8BRwHnAMcBnugqaMkiSJK2a3maun0wZBHyZJmXQp4EX0HT4fgA8uqo+soHN2fHqizPXS5K0aJ0z12/qlEE5Zfp+1bbufZq1Tlf5Lotse9btzKO2dbWxsCZmtqjPYmxDHFOz2kzv4aocB4uy2b5buoz5uS7juTRmTMt4js1znC+j2lamDJIkSRqbHS9JkqSB9Jky6KAk70nyySSfSPKEdv1Nkpyb5DPtv/v1FYMkSdIy6fOK17XAk6vqjsCRwGOS3BF4GvDuqro98O72uSRJ0srrbTqJqroauLp9/O0klwK3Ao6nudsR4BU0U0s8ta84tFjLOBhfkqTNYpAxXkkOBu4GnA8c0HbKAL4EHDBEDJIkSWPrveOVZC/gTcATq+pbk69VM5fFppjPQpIkaWf12vFKshtNp+vMqtqeKujLSW7Zvn5L4JqOuqYMkiRJK6W3MV5JQpME+9KqOm3ipbfS5Gw8tf33LdPqmzJIkiStmj5zNd4LeBjw8SQXt+ueQdPhen2SRwBXAr/VYwxaMAfRS5I0vz7vanw/3bmKju2rXUmSpGXlzPWSJEkDseMlSZI0kDFSBv1Fkk8l+ViSs5Ps21cMkiRJy2SMlEHnAneqqrsAlwFP7zEGSZKkpdFbx6uqrq6qC9vH3wYuBW5VVe+qqmvbYh8EDuwrBkmSpGUyRsqgSb8HnDNEDJIkSWMbLWVQkmfS/Bx5Zt8xSJIkLYMxUgaR5ETg/sBD2nyN0+qaMkiSJK2UwVMGJTkOeApwVFV9r6u+KYMkSdKqmemKV5L9ktxlg8W3pww6JsnF7XI/4EXAjYFz23V/M1vIkiRJm9MOr3glOQ94QFv2I8A1Sf65qp60Xr11Uga9fY44JUmSNr2NXPHapx0U/xvAK6vqHsB9+g1LkiRp9Wyk47VrklsCvwW8red4JEmSVtZGOl6nAO8ELq+qDyW5LfCZHVXqShk08fqTk1SS/ecLXZIkaXPZyF2NV7fpfQCoqiuSnLZehdb2lEEXJrkx8JEk51bVJ5McBNwX+Px8YUuSJG0+G7ni9VcbXHc9XSmD2pefTzOlhNNESJKkLaPzileSXwTuCdwsyeQdjHsDu8zSyGTKoCTHA1+oqo82U31JkiRtDetd8dod2Iumc3bjieVbwIM22sBkyiCanx+fAfzJBuo5c70kSVopnVe8quq9wHuTvLyqrgRIcgNgr8mci+tZmzIoyZ2B2wDbr3YdCFyY5Iiq+tKa9p25XpIkrZSNjPH68yR7J7kRcAnwySR/tKNK01IGVdXHq+rmVXVwVR0MXAX8wtpOlyRJ0iraSMfrju0VrgcC59BcsXrYBup1pQySJEnakjYyncRu7U+GDwReVFU/TLLDn/7WSRk0WebgjQQpSZK0CjZyxev/Ap8DbgS8L8mtaQbYS5IkaQY7vOJVVS8EXjix6sok9+4vJEmSpNW0wyteSQ5I8tIk57TP7wicsIF6nSmDkjwuyafa9c/dqT2QJEnaJDYyxuvlwMuAZ7bPLwNeR3PH4nqmpgwCDgCOB+5aVf+e5OZzRS5JkrTJdF7xSrK9U7Z/Vb0euA6gqq4FfrSjDa+TMuhRwKlV9e/ta9fs1B5IkiRtEuv91HhB++93k9yUdhLTJEcC35ylkcmUQcChwC8nOT/Je5PcfeaoJUmSNqH1Ol7bp4J4EvBW4HZJ/hl4JfC4jTYwmTKonQ9sV+AmwJHAHwGvz5SkjaYMkiRJq2a9MV6TybHPBt5O0xn7d+A+wMd2tPG1KYPa1VcBZ1VVARckuQ7YH/jKZF1TBkmSpFWz3hWvXWiSZN+YZg6vXdt1e7br1jUtZVDrzcC92zKH0iTj/uocsUuSJG0q613xurqqnrUT296eMujjSS5u1z0DOAM4I8klwH8AJ7RXvyRJklbaeh2vddP97MgOUgY9dGe2LUmStBmt91PjsYNFIUmStAV0dryq6ms7s+GumeuTHJbkg0kubu9aPGJn2pEkSdosNjJz/by6Zq5/LnBKVZ2T5H7t86N7jEOSJGkp9Nbxqqqrgavbx99Osn3m+gL2bovtA3yxrxgkSZKWSZ9XvP7Tmpnrnwi8M8nzaH7qvOcQMUiSJI1tvcH1CzFl5vpHASdV1UHASew42bYkSdJK6LXj1TFz/QnA9sdvAKYOrjdlkCRJWjW9/dS4zsz1XwSOAs4DjgE+M62+KYMkSdKq6XOMV9fM9b8PvCDJrsAPgD/oMQZJkqSl0eddjevNXP9f+mpXkiRpWfU+uF6SJEkNO16SJEkD6a3jleSnklyQ5KNtyqBT2vW3SXJ+ksuTvC7J7n3FIEmStEz6vOL178AxVXVX4DDguCRHAs8Bnl9VhwBfBx7RYwySJElLo7eOVzW+0z7drV2KZgqJN7brXwE8sK8YJEmSlknfE6ju0k4lcQ1wLvAvwDeq6tq2yFU0+RslSZJWXq8dr6r6UVUdBhxIM0P9HfpsT5IkaZkNcldjVX0DeA/wi8C+7eSp0HTIvjCtjimDJEnSqukzZdDNgB9W1TeS7AH8Cs3A+vcADwJeS5O38S3T6psySJIkrZo+UwbdEnhFkl1orqy9vqreluSTwGuT/ClwEU0+R0mSpJXXZ8qgjwF3m7L+CprxXpIkSVuKM9dLkiQNxI6XJEnSQMZIGXRmkk8nuSTJGUl26ysGSZKkZTJGyqAzaebzujOwB/DIHmOQJElaGn0Ori/gJ1IGVdXbt5dJcgHNXF6SJEkrb9CUQVV1/sRruwEPA97RZwySJEnLYtCUQUnuNPHyi4H3VdU/9RmDJEnSshg6ZdBxAEm2ATcDntRVx5RBkiRp1QyeMijJI4FfBY6tquu66psySJIkrZoxUgZdC1wJfCAJwFlV9awe45AkSVoKY6QM6rOzJ0mStLScuV6SJGkgdrwkSZIGMnjKoInXX5jkO131JUmSVk2f4622pwz6TjtZ6vuTnFNVH0xyOLBfj21LkiQtnd6ueFXjJ1IGtXc5/gXwlL7aliRJWkZjpAx6LPDWqrq6z7YlSZKWTa9TO1TVj4DDkuwLnJ3kvwK/CRzdZ7uSJEnLaOiUQfcGDgEuT/I5YM8kl0+rY8ogSZK0agZPGVRVt5go852qOmRafVMGSZKkVTN4yqAe25MkSVpqg6cMWlNmr77a18Zcf3a1Hatti9mOJElbkTPXS5IkDcSOlyRJ0kAGTxmUxrOTXJbk0iSP7ysGSZKkZTJ4yiDg54CDgDtU1XVJbt5jDJIkSUujz8H1BfxEyiDgUcCDq+q6ttw1fcUgSZK0TMZIGXQ74LfbyVHPSXL7PmOQJElaFr12vKrqR1V1GHAgcESSOwE3BH5QVYcDfwuc0WcMkiRJy2LolEHHAVcBZ7UvnQ3cZVodUwZJkqRVM3jKIODNNDkbPwscBVw2rb4pgyRJ0qpJMwa+hw0ndwFeAUymDHpWkn2BM4GfoRl8/7+q6qM72Jwdr2WRTF/f03EkSdIm1PGfZY8drwXbFEFuCXa8JEnakc6OlzPXS5IkDcSOlyRJ0kDGSBl0bJILk1yc5P1JDukrBkmSpGXS5+D6ADeaTBkEPAF4JXB8VV2a5NHAEVV14g425wCiZeEYL0mSdqRzjNcYKYMK2Ltdvw/wxXnbyCnT96u2dXcCZq3TVb7LItuedTuDOLlj/QJjmvWzqJO7t5WO19ars7DtdHRGF3lMzWrUY2dGizonV8WY3y0LPccWdF7MY4hzadbvlkX+0bqw75auP7Dp/rw725ix/Hrvx8z719H2rPswz7bWO9b6TJJNkl2AjwCHAP+nqs5P8kjg7Um+D3wLOLLPGCRJkpbFGCmDTgLuV1UHAi8DTuszBkmSpGUxdMqgXwPu2ibLBngdcM9pdUwZJEmSVs0YKYP2SXJoVV3Wrrt0Wn1TBkmSpFXT5xivWwKvaMd5bU8Z9LYkvw+8Kcl1wNeB3+sxBm0B6w2UXOTgypnb7lrfUWdRMUnzmufYnHnQ8cbD2ZRmHnzeSxQ7Z4jB553bmb3pTrPe+LFerIv8fu7zrsaPAXebsv5s4Oy+2pUkSVpWzlwvSZI0EDtekiRJA+m945VklyQXJXlb+/w2Sc5PcnmS1yXZve8YJEmSlsEQV7yewPXvXHwO8PyqOoRmcP0jBohBkiRpdH3PXH8g8OvAs4EntfkbjwEe3BZ5BU0Smr/uMw5tYV23yJwyaBTX492LWlpznC8ez4LNdRyMHWvfV7z+EngKcF37/KbAN6rq2vb5VcCteo5BkiRpKfTW8Upyf+CaqvrInPWduV6SJK2UPn9qvBfwgCT3A34K2Bt4AbBvkl3bq14HAl+YVtmZ6yVJ0qrp7YpXVT29qg6sqoOB/wH8Y1U9hCZn44PaYicAb+krBkmSpGXS6+D6Dk8FXpvkT4GLgJeOEIO2inSsP3nIIKRNwvNlJXWlyGFb/22MPZB9mrFjHaTjVVXnAee1j68AjhiiXUmSpGXizPWSJEkDseMlSZI0kDFSBp2Z5NNJLklyRpLd+o5BkiRpGYyRMuhM4A7AnYE9gEcOEIMkSdLoBk0ZBFBVb594/QKaubykfixhyiBpWS3jHWjaeV2f6yInyNxMx87YsQ6dMug/tT8xPgx4R88xSJIkLYUxUwa9GHhfVf1TR31TBkmSpJUyaMqgJK+uqocm2QbcDPifXZVNGSRJklbN0CmDHprkkcCvAr9TVT/xE6QkSdKqGiNl0N8AVwIfSAJwVlU9a4Q4pFFUR5qOOOB/afgZSerLGCmDxujsSZIkjc6Z6yVJkgYy+Mz1E+tfmOQ7fbcvSZK0LMaYuZ4khwP7DdC2JEnS0hh85vokuwB/ATwY+G99ti+RjvUnDxnE9TlAexNYwuNmCHXy9PVjz/St5eexs3FjzFz/WOCtVXV1z21LkiQtlUFnrk/y08BvAn/VV7uSJEnLqs8rXttnrv8c8FrgGOATwCHA5e36PZNcPq2yKYMkSdKq6W2MV1U9HXg6QJKjgT+sqvtPlknynao6pKO+KYMkSdJKcR4vSZKkgQw+c/2a9XsN0b62sK5rpd5ZqHVs1Tuxtup+a+d57GycV7wkSZIGYsdLkiRpIIOnDErj2UkuS3Jpksf3HYMkSdIyGGKM1/aUQXu3z08EDgLuUFXXJbn5ADFIkiSNrtcrXhMpg14ysfpRwLOq6jqAqrqmzxgkSZKWxRgpg24H/HY7Oeo5SW7fcwySJElLYdCUQa0bAj+oqsOBvwXO6CsGSZKkZTJoyqAkrwauAs5qy5wN3GVaZVMGSZKkVTN0yqCHJjkVuDfwWeAo4LKO+qYMkiRJK2WQmevXOBU4M8lJwHeAR44QgyRJ0uAGTxlUVd+gudNR6l861p88ZBDXVx1tm3JD0ma1mb7Xxo7VmeslSZIGYsdLkiRpIGOkDDo2yYVJLk7y/iSH9B2DJEnSMhjiitf2lEHb/TXwkKo6DPg74I8HiEGSJGl0vQ6un0gZ9GzgSe3q4sd5G/cBvthnDNriuiYiOWXQKK5nGQeb6vpq2/T1GfG4kZbZpvpeG/n/hb7vavxLmpRBN55Y90jg7Um+D3wLOLLnGCRJkpbCGCmDTgLuV1UHAi8DTusrBkmSpGUydMqg/wfctarOb8u8DrjntMqmDJIkSatm0JRBwAOBLyU5tKouA36F6w+8n6xvyiBJkrRSBk0ZVFXXJvl94E1JrgO+DvzekDFIkiSNZYyUQWcDZw/RriTNw7sXh+edpNoqnLlekiRpIHa8JEmSBmLHS5IkaSB2vCRJkoZSVZtqAf6gz/Kr0sYyxuR+L0/5VWljGWNyv5en/Kq0sYwxud+z1ble/Z2pPMYCfLjP8qvSxjLG5H4vT/lVaWMZY3K/l6f8qrSxjDG537PVmVz8qVGSJGkgdrwkSZIGshk7XrMmbpwn0eMqtLGMMQ3RxjLGNEQbyxjTEG0sY0xDtLGMMQ3RxjLGNEQbyxjTEG0sY0zz1vlPaX+vlCRJUs824xUvSZKkTcmOlyRJ0kDseEmSJA1k6TteSe6Q5KlJXtguT03yczsof2ySvdasP26D7b1yB6/fI8ne7eM9kpyS5O+TPCfJPlPK757kd5Pcp33+4CQvSvKYJLttJCb1K8nNZyx/075ikfqyVY9z93vD5VdivzeDpe54JXkq8FogwAXtEuA1SZ42pfzjgbcAjwMuSXL8xMt/NqX8W9csfw/8xvbnHWGdAXyvffwCYB/gOe26l00p/zLg14EnJHkV8JvA+cDdgZes+wYMbOwTNck+SU5N8qkkX0vyb0kubdftO6X83kn+PMmrkjx4zWsv7mjjJmuWmwIXJNkvyU2mlD81yf7t48OTXAGcn+TKJEdNKX94kvckeXWSg5Kcm+SbST6U5G4dMe2a5H8meUeSj7XLOUn+17TOeZJd2vL/O8m91rz2x9Pa6Gj3snVee+zEfh+S5H1JvpHk/CR3nlL+tknOSPKnSfZK8rdJLknyhiQH7+w+L2q/19vnIfa7rbNVj3P3e2vt9+Dfa1mC83tDdmb21b4X4DJgtynrdwc+M2X9x4G92scHAx8GntA+v2hK+QuBVwNHA0e1/17dPj6qI6ZLJ+uvee3iKeU/1v67K/BlYJf2eba/NqXOPsCpwKeArwH/Blzartu3o87ewJ8DrwIevOa1F08pf5M1y02BzwH7ATeZUv5UYP/28eHAFcDlwJXrvFeHA+9p3+ODgHOBbwIfAu42pfw7gacCt5hYd4t23bumlH9TG9cDgbe2z2847bOZqHMd8Nk1yw/bf6+YdkxNPH4PcPf28aFMmb2Y5o+DXwN+B/hX4EHt+mOBD3TE9Brgr4EjgQPb5ch23eumlH8J8HfAE4GPAKd1HZMT678NfKtdvt0uP9q+fkr5T0w8/n/Af2sfHw3885Ty7wMeBTwNuAR4cvuZPwL4x53d53n2e9Z9HmK/t/hx7n5vrf3u9XuNJT2/N7LMVWmohabjcesp628NfHq9N7V9vhfwDuA0pneKbgCcRNMhOKxd9xMH6Zo6bwAe3j5+GXD4xAH7oSnlL6HpKO7XHhA3adf/FBOduDV1ZjpR29dnOln7PlHb12Y6Wad9puu9tvYzBZ4J/DNNJ7LrC+rJ7TFx54l1n12n3UuBXdvHH+x6TybWXTTx+PNdr61Zf9k67f/Ea0x02Gk69KcDZwE3XKeNFwKvBA7Y4H5/euLxh7ran3e/Z93nefZ71n0eYr/XtrGR11boOHe/t9Z+9/q9tqzn90aWuSoNtQDH0VxVOaf9EE5vD67LgeOmlP9H2g7Umg/wlcCP1mnnQJoO1YvWvrlTyu4DvBz4F5qfDH9Ic/XnvcBdp5Q/qX39SuDxwLuBv6W5OrdtRwfHRl+b9WTt+0Sd56AF3gU8Zc2JdABNh/MfOmK6wZp1JwKfAK7cwOd9GnBj1uls0/xs/S7gGOBkmp+XjwJOAV41pfwHgPvS/KR8JfDAdv1RdHdQP9iWv8HEuhsAvw2cP6X8p6as29Z+3j9xJXiizH9pz5HHt9tfb7+f3R7ntwWeQfNX6K2BhwNvm1L+IzSd8COAr/LjP0gOYfoX2kz7PO9+z7LPO7nfd9/Ifm/x49z93lr73fv3GsOd32u/125Px/m9kWWuSkMu7Zt5JPDf2+VI2p/rOg68W3S8dq8NtPXrwJ9tMK69gbu2H/wBOyj708BPt4/3BR4EHLFO+ZlO1Pb1mU/WPk/Uts5MJyvNVcHn0Fzp/DrNz6yXtuum/fz5XOA+U9Yf13Wirin3gPbL4Us7KHc08DrgIpoO89uBP2D6z+B3pblieQ5wh/Z9+kb7OdyzY/sHt9u/hubn9cvax68DbjOl/KuZ/ofHI4EfbuB8ejzwT8AXd1D2RJo/Lr5Kc7X2kzRjJfeZUvZY4NPt5/VLNFdcP9Pux/Hr7PNX2v3dXnbqPu/Mfs+yz235hy9ovx/Ysf1VOM4Pm3Kcf709zqd+167Ifk87vzfrft97J/f7GyzB9xqzn98n0tP32kaXuSq59LusOVG/tuZE3a+jztwn6wK+oHbtKD/PyXoH4D60Y/Um92Od8sdOKf9r6+zHf9YB9gDuNGcbXeV/bpby7Wv3oPmr6qbAvYA/BO63Tvkj+PHPvXcEnrRe+Sl1fhn4kxna+Hmaq6Trlb/HmvLr7sNEvZu2y6vnOFdeOUPZWwL/NkcbU/+wWKf821jzR9AOyv9y+97ed4Plf6n9vDdUfqKNP56hjVlj2mH59vjYp328J/Cs9r16DtP/05ssv0db/u+7yk/U2XuijecC/7CDNvbeyZg6y7flHg8cNMNnNVP5aXWY+F4bMabdgROAX2nP74cALwYew/TO3Q2B36X9fwx4MM2vUI8Bdu/Y/mT5h9H8uvToadvvqPMQ4P+sE9P2fZgs37kPG11MGbTJJHl4Vb1s0XWS7AHcrqoumbWNRcWU5q7Ux9B0Mg+juTHiLe1rF1bVL6wp/zjgsRstP2cb85R/NE2neaMxbaMZC7crzXjDI4DzaL6w3llVz95B+XvQjLubWn5BbSy6/LS7ho+h+dmAqnrAlH1YWyc0f7VPrbOgNtatM2cbF1TVEe3jR9IcX2+muTr891V16jrlf78tf3ZX+Y46j56hjXliWnf7bblP0AzHuDbJ6cB3aa4gHNuu/40dlP8e8Mau8gtqY6Hl2zrfbMv9C83g8TdU1VfXluso/5q2/Fe6ynfUef0MbfQV05k03wd70NxUdSOa4/ZYmnSFJ3SU35PmD/S9aMZ4HQtQVSfuoPy625+nzqz7sGHz9thcxlnYwRi0RdTpu3xXHWa/K3Wm8kO0sRMx7ULzZfAtfvwX+B5MHx81U/kh2pij/Dx3FF80S50525ipzqwxrT0OaO7wvVn7+EbseGDzDssP0cacMc16R/hM5YdoY86YLqL5Oey+wEtpfl5/B82VlBvvbPkh2pgzppnu6O+7/FBtbGTZFS2dJB/reolmrNdO1+m7/Jx1blBV3wGoqs8lORp4Y5Jbt3V2tvwQbcwT07VV9SPge0n+paq+1db/fpLrFlB+iDZmLX848ASam0D+qKouTvL9qnpvR/zQjKecpc48bcxaZ9aYAG6QZD+a/8hS7ZWDqvpukmsXUH6INuaJafJq+keTHF5VH05yKM1NSjtbfog25ompquo6mjGy70ozh9X2u72fB9xsJ8sP0cY8Md0gye40nfE9aW5M+xrNT4rT5urru/xQbezYvD02l/4Wmp71YTR3W0wuB9MxeHDWOn2Xn7ONme5KnbX8EG3MGdP5wJ7t48k7gPZh+h2pM5Ufoo15Ympf3/AdxfPWWbY2aObLu4J26hbglu36vZh+lWWm8kO0MWdM+zDbHeEzlR+ijTljumidY2HPnS0/RBtzxjTTHf19lx+qjY0sc1Vy6XehuZT7Sx2v/d0i6vRdfs42ZrorddbyQ7QxZ0w37Fi/PxPTfcxbfog25olpTbkN31E8b51lbWOi7p503NG5iPJDtLGR8sxwR/g85YdoY5bywKEzfkYzlR+ijXliauvNekd/r+WHamNHi4PrJUmSBrLUuRolSZJWiR0vSZKkgXhXo6SVkeRHNANfdwOupbmp4fnV3JElSaOz4yVplXy/qg4DSHJzmskh96bJ+SZJo/OnRkkrqaquoUlp9dg0Dk7yT0kubJd7AiR5ZZIHbq+X5Mwkxyf5+SQXJLk4yceS3H6kXZG0QryrUdLKSPKdqtprzbpvAD9LkxD3uqr6QduJek1VHZ7kKOCkqnpgkn2Ai4HbA88HPlhVZ7aTKO5SVd8fcn8krR5/apS0VewGvCjJYcCPgEMBquq9SV6c5GbAfwfeVE0uvg8Az0xyIHBWVX1mrMAlrQ5/apS0spLclqaTdQ3NLNRfppn48nBg94mirwQeCjwcOAOgqv4OeADwfeDtSY4ZLnJJq8orXpJWUnsF62+AF1VVtT8jXlVV1yU5gSap93YvBy4AvlRVn2zr3xa4oqpemORngLvQpISSpLnZ8ZK0SvZIcjE/nk7iVcBp7WsvBt6U5HeBdwDf3V6pqr6c5FLgzRPb+i3gYUl+CHwJ+LPeo5e08hxcL2nLS7Inzfxfv1BV3xw7HkmryzFekra0JPcBLgX+yk6XpL55xUuSJGkgXvGSJEkaiB0vSZKkgdjxkiRJGogdL0mSpIHY8ZIkSRqIHS9JkqSB/H99s7ABAxX92QAAAABJRU5ErkJggg==\n", + "text/plain": [ + "
" + ] + }, + "metadata": { + "needs_background": "light" + }, + "output_type": "display_data" + } + ], + "source": [ + "get_grid(x)" + ] + }, + { + "cell_type": "code", + "execution_count": 596, + "id": "e6a9cb65-06a6-4e46-89f5-c2a39df5f2ba", + "metadata": {}, + "outputs": [ + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAmQAAAE9CAYAAACleH4eAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjQuMywgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/MnkTPAAAACXBIWXMAAAsTAAALEwEAmpwYAAAVGUlEQVR4nO3df7BtZXkf8O8TLhrUVKHcUMKPXpJczWCnor11SM20RDOJmjSQaQexiTKOnZvpYBTHTkfNH9o/6KQzURubSEqUCo2iVEhlWsaGoBPNtKIXZVAgREYhQPlxrYo/YlXg6R9nXT3gvfecq2edd5+zP5+ZPWetd629z7PP2mvf733Xu9aq7g4AAOP8yOgCAACWnUAGADCYQAYAMJhABgAwmEAGADCYQAYAMNiO0QX8MI4//vjetWvX6DIAANZ04403frG7dx5s2ZYOZLt27cq+fftGlwEAsKaquutQyxyyBAAYTCADABhMIAMAGGy2QFZVp1TVR6rq1qq6papeM7W/uaruraqbpseLVz3nDVV1R1XdXlW/NFdtAACLZM5B/Q8neV13f6qqfizJjVV13bTsbd39u6tXrqrTk5yX5JlJfiLJn1XV07v7kRlrBAAYbrYesu6+r7s/NU1/LcltSU46zFPOTvK+7v5Wd38hyR1JnjtXfQAAi2JTxpBV1a4kz05yw9T0qqq6uaourapjp7aTkty96mn35PABDgBgW5g9kFXVU5JcleTC7v5qkouT/FSSM5Lcl+QtR/h6e6tqX1Xt279//0aXCwCw6WYNZFV1dFbC2Hu6++ok6e4HuvuR7n40yR/le4cl701yyqqnnzy1PUZ3X9Lde7p7z86dB73YLQDAljLnWZaV5F1Jbuvut65qP3HVar+W5LPT9DVJzquqJ1bVaUl2J/nEXPUBACyKOc+yfF6SlyX5TFXdNLW9MclLq+qMJJ3kziS/mSTdfUtVXZnk1qycoXmBMywBgGUwWyDr7r9IUgdZdO1hnnNRkovmqgkAtqR63D+n3WPqYDau1A8AMJhABgAwmEAGADCYQAYAMJhABgAwmEAGADCYQAYAMJhABgAwmEAGADCYQAYAMJhABgAwmEAGADCYQAYAMJhABgAwmEAGADCYQAYAMJhABgAwmEAGADCYQAYAMJhABgAwmEAGADCYQAYAMJhABgAwmEAGADCYQAYAMJhABgAwmEAGADCYQAYAMJhABgAwmEAGADCYQAYAMJhABgAwmEAGADCYQAYAMJhABgAwmEAGADCYQAYAMJhABgAwmEAGADCYQAYAMJhABgAwmEAGADCYQAYAMJhABgAw2GyBrKpOqaqPVNWtVXVLVb1maj+uqq6rqs9NP4+d2quq3l5Vd1TVzVX1nLlqAwBYJHP2kD2c5HXdfXqSM5NcUFWnJ3l9kuu7e3eS66f5JHlRkt3TY2+Si2esDQBgYcwWyLr7vu7+1DT9tSS3JTkpydlJLptWuyzJOdP02Uku7xUfT/K0qjpxrvoAABbFpowhq6pdSZ6d5IYkJ3T3fdOi+5OcME2flOTuVU+7Z2oDANjWZg9kVfWUJFclubC7v7p6WXd3kj7C19tbVfuqat/+/fs3sFIAgDFmDWRVdXRWwth7uvvqqfmBA4cip58PTu33Jjll1dNPntoeo7sv6e493b1n586d8xUPALBJ5jzLspK8K8lt3f3WVYuuSXL+NH1+kg+uan/5dLblmUkeWnVoEwBg29ox42s/L8nLknymqm6a2t6Y5HeSXFlVr0xyV5Jzp2XXJnlxkjuS/E2SV8xYGwDAwpgtkHX3XySpQyx+wUHW7yQXzFUPAMCicqV+AIDBBDIAgMEEMgCAwQQyAIDBBDIAgMEEMgCAwQQyAIDBBDIAgMEEMgCAwQQyAIDBBDIAgMEEMgCAwQQyAIDBBDIAgMEEMgCAwQQyAIDBBDIAgMEEMgCAwQQyAIDBBDIAgMEEMgCAwQQyAIDBBDIAgMEEMgCAwQQyAIDBdowuAABYMFWPne8eU8cS0UMGADCYQAYAMJhABgAwmEAGADCYQAYAMJhABgAwmEAGADCYQAYAMJhABgAwmEAGADCYWycBAFvH42/rlGyLWzsJZLAe2/QLAIDF4JAlAMBgAhkAwGACGQDAYAIZAMBgAhkAwGCzBbKqurSqHqyqz65qe3NV3VtVN02PF69a9oaquqOqbq+qX5qrLgCARTNnD9m7k7zwIO1v6+4zpse1SVJVpyc5L8kzp+e8o6qOmrE2AICFMVsg6+6PJvnSOlc/O8n7uvtb3f2FJHckee5ctQFwCFWPfQCbYsQYsldV1c3TIc1jp7aTkty9ap17pjYAgG1vswPZxUl+KskZSe5L8pYjfYGq2ltV+6pq3/79+ze4PACAzbepgay7H+juR7r70SR/lO8dlrw3ySmrVj15ajvYa1zS3Xu6e8/OnTvnLRgAYBNsaiCrqhNXzf5akgNnYF6T5LyqemJVnZZkd5JPbGZtAACjzHZz8aq6IslZSY6vqnuSvCnJWVV1RpJOcmeS30yS7r6lqq5McmuSh5Nc0N2PzFUbAMAimS2QdfdLD9L8rsOsf1GSi+aqBwBgUblSPwDAYAIZAMBgAhkAwGACGQDAYAIZAMBg6w5kVXVMVT1jzmIAAJbRugJZVf3TJDcl+dA0f0ZVXTNjXQAAS2O9PWRvzsptjr6SJN19U5LTZqkIAGDJrDeQfae7H3pcW290MQAAy2i9V+q/par+RZKjqmp3klcn+V/zlQUAsDzW20P2W0memeRbSd6b5KEkF85UEwDAUlmzh6yqjkryP7r755P89vwlAQAslzV7yLr7kSSPVtVTN6EeAICls94xZF9P8pmqui7JNw40dverZ6kKAGCJrDeQXT09AADYYOsKZN19WVU9IcnTp6bbu/s785UFALA81hXIquqsJJcluTNJJTmlqs7v7o/OVhkAwJJY7yHLtyT5xe6+PUmq6ulJrkjyD+YqDABgWaz3OmRHHwhjSdLdf5Xk6HlKAgBYLuvtIdtXVe9M8sfT/K8n2TdPSQAAy2W9gexfJbkgK7dMSpKPJXnHLBUBACyZ9QayHUl+r7vfmnz36v1PnK0qAIAlst4xZNcnOWbV/DFJ/mzjywEAWD7rDWQ/2t1fPzAzTT9pnpIAAJbLegPZN6rqOQdmqmpPkm/OUxIAwHJZ7xiyC5P816r6P9P8iUleMktFAABL5rA9ZFX1D6vq73T3J5P8TJL3J/lOkg8l+cIm1AcAsO2tdcjyPyX59jT9s0nemOQPknw5ySUz1gUAsDTWOmR5VHd/aZp+SZJLuvuqJFdV1U2zVgYAsCTW6iE7qqoOhLYXJPnwqmXrHX8GAMBhrBWqrkjy51X1xaycVfmxJKmqn07y0My1AQAshcMGsu6+qKquz8pZlX/a3T0t+pEkvzV3cQAAy2DNw47d/fGDtP3VPOUAACyf9V4YFgCAmQhkAACDCWQAAIMJZAAAgwlkAACDCWQAAIMJZAAAgwlkAACDCWQAAIMJZAAAgwlkAACDzRbIqurSqnqwqj67qu24qrquqj43/Tx2aq+qentV3VFVN1fVc+aqCwBg0czZQ/buJC98XNvrk1zf3buTXD/NJ8mLkuyeHnuTXDxjXQAAC2W2QNbdH03ypcc1n53ksmn6siTnrGq/vFd8PMnTqurEuWoDAFgkmz2G7ITuvm+avj/JCdP0SUnuXrXePVMbAMC2N2xQf3d3kj7S51XV3qraV1X79u/fP0NlAACba7MD2QMHDkVOPx+c2u9Ncsqq9U6e2r5Pd1/S3Xu6e8/OnTtnLRYAYDNsdiC7Jsn50/T5ST64qv3l09mWZyZ5aNWhTQCAbW3HXC9cVVckOSvJ8VV1T5I3JfmdJFdW1SuT3JXk3Gn1a5O8OMkdSf4mySvmqgsAYNHMFsi6+6WHWPSCg6zbSS6YqxYAgEXmSv0AAIMJZAAAgwlkAACDCWQAAIMJZAAAgwlkAACDCWQAAIMJZAAAgwlkAACDCWQAAIMJZAAAgwlkAACDCWQAAIMJZAAAgwlkAACDCWQAAIPtGF0AAEus6vvbuje/DhhMDxkAwGACGQDAYAIZAMBgAhkAwGAG9TOvxw/YNVgXAL6PHjIAgMEEMgCAwQQyAIDBBDIAgMEEMgCAwQQyAIDBXPYCgB+M+1DChtFDBgAwmEAGADCYQAYAMJgxZADrZcwUMBM9ZAAAgwlkAACDCWQAAIMJZAAAgwlkAACDCWQAAIMJZAAAgwlkAACDCWQAAIO5Uj9sJa4UD7AtDQlkVXVnkq8leSTJw929p6qOS/L+JLuS3Jnk3O7+8oj6AAA208hDlj/f3Wd0955p/vVJru/u3Umun+YBALa9RRpDdnaSy6bpy5KcM64UAIDNMyqQdZI/raobq2rv1HZCd983Td+f5IQxpQEAbK5Rg/p/rrvvraofT3JdVf3l6oXd3VV10JHKU4DbmySnnnrq/JUCAMxsSA9Zd987/XwwyZ8keW6SB6rqxCSZfj54iOde0t17unvPzp07N6tkAIDZbHogq6onV9WPHZhO8otJPpvkmiTnT6udn+SDm10bAMAIIw5ZnpDkT2rleko7kry3uz9UVZ9McmVVvTLJXUnOHVAbAMCm2/RA1t2fT/Ksg7T/3yQv2Ox6AABGW6TLXgAALCWBDABgMIEMAGAwgQwAYDCBDABgMIEMAGAwgQwAYDCBDABgMIEMAGAwgQwAYDCBDABgsBE3FweA+VU9dr57TB2wDnrIAAAGE8gAAAYTyAAABhPIAAAGE8gAAAYTyAAABhPIAAAGcx0yALaGx19XLHFtMbYNPWQAAIPpIQPm54rpAIelhwwAYDCBDABgMIEMAGAwgQwAYDCBDABgMIEMAGAwgQwAYDCBDABgMIEMAGAwgQwAYDCBDABgMIEMAGAwgQwAYLAdowvYEqoeO989z+tu5Gsvwu87UnP9ned+beCxDra/Lfr3DwymhwwAYDA9ZDCHRemRW5Q6YDtZ1t6+ZX3fm0QPGQDAYHrIAH5Yeg5gHhvRy79FjhQIZMByE6aABSCQsTGW9R+1jXjfy/q3A+C7BDIAYLks4H+EBTLYLAv4BbCQjmS8h78pc9si44/WZTu9l21o4c6yrKoXVtXtVXVHVb1+dD1HrOqxj63g8TVvlbq3O9tkY22nz/mRvpft8r5hG1uoQFZVRyX5gyQvSnJ6kpdW1eljqzqE7fTlDkfK5/8H52/HRvFZ2lYW7ZDlc5Pc0d2fT5Kqel+Ss5PcOrSqReaQzcbb7t36h/rMbPf3faS24t9jO91+bK5D1+v9/K/1O3+Y1z6S97IVLvMwx/tea/1taKF6yJKclOTuVfP3TG0AANvWovWQramq9ibZO81+vapu38Rff3ySLx6yW/hg7UfahbwRXc5H8hpH8l42oo6NqO2Hc/hteCR1bMTnYM7XmOv3Helrb/y6K9twzjrm2oZHWseRWITPzPrr2Ljv0u30Gj/s6x7p79uI/XCzv1NGfKY31t891IJFC2T3Jjll1fzJU9t3dfclSS7ZzKIOqKp93b1nxO9mY9iGW59tuPXZhlufbbjxFu2Q5SeT7K6q06rqCUnOS3LN4JoAAGa1UD1k3f1wVb0qyf9MclSSS7v7lsFlAQDMaqECWZJ097VJrh1dxyEMOVTKhrINtz7bcOuzDbc+23CDVS/ZaaUAAItm0caQAQAsHYFsHbb87ZyWUFWdUlUfqapbq+qWqnrN1H5cVV1XVZ+bfh47ulYOr6qOqqpPV9V/n+ZPq6obpv3x/dMJQCyoqnpaVX2gqv6yqm6rqp+1H24tVfXa6Xv0s1V1RVX9qP1w4wlka9hSt3NitYeTvK67T09yZpILpu32+iTXd/fuJNdP8yy21yS5bdX8v0/ytu7+6SRfTvLKIVWxXr+X5EPd/TNJnpWVbWk/3CKq6qQkr06yp7v/XlZOuDsv9sMNJ5Ct7bu3c+rubyc5cDsnFlh339fdn5qmv5aVfwROysq2u2xa7bIk5wwpkHWpqpOT/HKSd07zleT5ST4wrWIbLrCqemqSf5zkXUnS3d/u7q/EfrjV7EhyTFXtSPKkJPfFfrjhBLK1uZ3TFldVu5I8O8kNSU7o7vumRfcnOWFUXazLf0jyb5I8Os3/7SRf6e6Hp3n742I7Lcn+JP95Ouz8zqp6cuyHW0Z335vkd5P8dVaC2ENJboz9cMMJZGxrVfWUJFclubC7v7p6Wa+cYuw04wVVVb+S5MHuvnF0LfzAdiR5TpKLu/vZSb6Rxx2etB8utml839lZCdc/keTJSV44tKhtSiBb25q3c2IxVdXRWQlj7+nuq6fmB6rqxGn5iUkeHFUfa3pekl+tqjuzMlTg+VkZj/S06dBJYn9cdPckuae7b5jmP5CVgGY/3Dp+IckXunt/d38nydVZ2TfthxtMIFub2zltQdNYo3clua2737pq0TVJzp+mz0/ywc2ujfXp7jd098ndvSsr+92Hu/vXk3wkyT+fVrMNF1h335/k7qp6xtT0giS3xn64lfx1kjOr6knT9+qBbWg/3GAuDLsOVfXirIxlOXA7p4vGVsRaqurnknwsyWfyvfFHb8zKOLIrk5ya5K4k53b3l4YUybpV1VlJ/nV3/0pV/WRWesyOS/LpJL/R3d8aWB6HUVVnZOWkjCck+XySV2SlM8B+uEVU1b9N8pKsnL3+6ST/MitjxuyHG0ggAwAYzCFLAIDBBDIAgMEEMgCAwQQyAIDBBDIAgMF2rL0KwNZXVY9k5TIoR2fl9P3Ls3Jz5EcP+0SATSCQAcvim919RpJU1Y8neW+Sv5XkTSOLAkgcsgSWUHc/mGRvklfVil1V9bGq+tT0+EdJUlWXV9U5B55XVe+pqrOr6plV9Ymquqmqbq6q3YPeCrBNuDAssBSq6uvd/ZTHtX0lyTOSfC3Jo939/6ZwdUV376mqf5Lktd19TlU9NclNSXYneVuSj3f3e6Zbqh3V3d/czPcDbC8OWQKsjCv7/ek2P48keXqSdPefV9U7qmpnkn+W5Krufriq/neS366qk5Nc3d2fG1U4sD04ZAkspememI8keTDJa5M8kORZSfZk5b6LB1ye5Deycg/GS5Oku9+b5FeTfDPJtVX1/M2rHNiO9JABS2fq8frDJL/f3T0djrynux+tqvOTHLVq9Xcn+USS+7v71un5P5nk89399qo6NcnfT/LhTX0TwLYikAHL4piquinfu+zFf0ny1mnZO5JcVVUvT/KhJN848KTufqCqbkvy31a91rlJXlZV30lyf5J/N3v1wLZmUD/AYVTVk7Jy/bLndPdDo+sBtidjyAAOoap+IcltSf6jMAbMSQ8ZAMBgesgAAAYTyAAABhPIAAAGE8gAAAYTyAAABhPIAAAG+/9Ca/zob4vDvwAAAABJRU5ErkJggg==\n", + "text/plain": [ + "
" + ] + }, + "metadata": { + "needs_background": "light" + }, + "output_type": "display_data" + } + ], + "source": [ + "col_scores = column_scores(x)\n", + "\n", + "plt.figure(figsize=(10, 5))\n", + "plt.bar(x = x.columns, height = col_scores, color = 'r')\n", + "plt.ylabel(\"Score\")\n", + "plt.xlabel(\"Days\")\n", + "plt.show()" + ] + }, + { + "cell_type": "markdown", + "id": "84add652-4199-4483-884a-3ad3fa767df3", + "metadata": {}, + "source": [ + "Interestingly, the one flake we think we see at around day 13 has a score that is not as large as serveral of the other days. It is important to keep in mind that we are only seeing the first 50 tests that had a failure, and there are hundreds more that extend theoretically very far below this graph." + ] + }, + { + "cell_type": "markdown", + "id": "ea26bbe7-c547-4d1a-93c2-a4f6a24ded5d", + "metadata": {}, + "source": [ + "Now that we have these scores, we need to decide what a high score would be. We do this by performing a monte carlo simulation to get an approximate idea of the distribution. With every test there is a porbability associated with it, and a score. To generate a random column we simply add $-\\log p$ to the score with a probability of $p$. We do this for every test and return the resulting total score." + ] + }, + { + "cell_type": "code", + "execution_count": 598, + "id": "15aa17f3-79f5-42fc-8481-121df72cbada", + "metadata": {}, + "outputs": [], + "source": [ + "def random_trial(row_scores):\n", + " row_scores = row_scores[row_scores != 0]\n", + " scores_scores = -np.log(row_scores)\n", + " tot = 0\n", + " for a, b in zip(row_scores, scores_scores):\n", + " tot += b * np.random.binomial(1, p = a)\n", + " return tot" + ] + }, + { + "cell_type": "code", + "execution_count": 600, + "id": "d88192e6-2de7-4995-aa94-5f9b3faa1f2f", + "metadata": {}, + "outputs": [], + "source": [ + "# this is a large number of simulations and can be adjusted later\n", + "n_sims = 10000\n", + "\n", + "row_scores = x.apply(naive_prob_failure, axis = 1).values\n", + "out = [random_trial(row_scores) for i in range(n_sims)]" + ] + }, + { + "cell_type": "code", + "execution_count": 601, + "id": "109a3992-19db-437f-9fe4-67721993a655", + "metadata": {}, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/opt/app-root/lib64/python3.8/site-packages/seaborn/distributions.py:2619: FutureWarning: `distplot` is a deprecated function and will be removed in a future version. Please adapt your code to use either `displot` (a figure-level function with similar flexibility) or `histplot` (an axes-level function for histograms).\n", + " warnings.warn(msg, FutureWarning)\n" + ] + }, + { + "data": { + "text/plain": [ + "" + ] + }, + "execution_count": 601, + "metadata": {}, + "output_type": "execute_result" + }, + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYgAAAD4CAYAAAD2FnFTAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjQuMywgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/MnkTPAAAACXBIWXMAAAsTAAALEwEAmpwYAAArpElEQVR4nO3dd3xcV5n/8c8zo96rJVnFktx7iWOnEJIQnEIgARLSWGB3WbIsyTaWXcKWLLDsLvBbAsuLLEsomxBwEpIAcRJDKqlO5BY3Wa6yLKv3Xmfm+f0x40RRZFu2NLpTnvfrNS/N3Htn5muPpEf3nHvOEVXFGGOMGc/ldABjjDGhyQqEMcaYCVmBMMYYMyErEMYYYyZkBcIYY8yEYpwOMF1ycnK0tLTU6RjGGBNWduzY0aaquRPti5gCUVpayvbt252OYYwxYUVEjp9qnzUxGWOMmZAVCGOMMROyAmGMMWZCViCMMcZMyAqEMcaYCVmBMMYYMyErEMYYYyZkBcIYY8yErEAYY4yZUMSMpDaRa2NF7YTbb1tfMsNJjIkudgZhjDFmQlYgjDHGTMgKhDHGmAlZgTAhy+dT6rsGOdExQN+wx+k4xkQd66Q2jjhdx/Oo18fGilp+/Go1dZ2Db++blRrPpQtyWVmcgUtkpqIaE7WsQJiQ0jUwwh0bd/L6kXbWzsnk85fO5WBTL+19w+ys7eLRHXXsruvi5rV2BZMxwWYFwoSMYY+XW+57k+rWfv7fjSv4xNpi4J2zjYvm5bCtpoOndjfyw5eP8tHVs8lOiXcysjERzfogTEhQVR7fUceh5l7u+/R5bxeHsVwirC/L5k/eV0rXwAh/ev82+q1vwpigsTMIExK21XSyr6GHa5bl09A1dMo+CoDynBRuXVfCLyuO8y+/3cc9N6+auaDGRBErEMZxA8MenqlsoiwnmffNy5nUcxYXpHHZwln8+q16YmNcrCzKAGx0tTHTyZqYjOOerWpm2OPlIytmI2dxddLlC2dRkpXEE7vq7TJYY4LACoRxVOfACNtrOlhXlkV+esJZPdftEj6+ppBRj/LMvqYgJTQmelmBMI569XAbgvD++bnn9PxZqQlcPC+HHbWdHG/vn+Z0xkQ3KxDGMX3DHrbXdLCqOIOMpLhzfp3LF+WSlhDD7/Y1oarTmNCY6GYFwjhm67F2PD7lkgWT65g+lfgYN1csyqO2Y4Bn9zdPUzpjjBUI4wifKttqOpmXm8Ks1LPre5jImjmZ5KTE8+3fH8Dj9U1DQmOMFQjjiEPNvXQPjnJ+Wda0vJ7bJVy1NI+jrf08vrNuWl7TmGhnBcI4YuuxDlLiY1hSkDZtr7mkII3VJRl897nDDI54p+11jYlWViDMjGvrG+ZQcy9rSjJxu6ZvVlYRYe2cLJp6hvibR3axsaL2tCOyjTGnZwXCzLjNexvxKawqzpj21y7LSWZhXiovH2phYMQGzxkzFUEtECJytYgcFJEjInLXBPvjReSRwP4KESkdt79ERPpE5EvBzGlm1qZdDeSlxZ/1wLjJumppPsOjPl4+2BqU1zcmWgStQIiIG7gXuAZYAtwqIkvGHfZZoFNV5wHfBb41bv89wO+CldHMvBMdA2w/3vn23EnBkJ+ewOqSDN6obqdrYCRo72NMpAvmGcQ64IiqVqvqCPAwcP24Y64HHgjcfwy4QgKT8YjIR4FjQGUQM5oZ9uSeBgBWBLFAAHxwcR4AL1S1BPV9jIlkwSwQhcCJMY/rAtsmPEZVPUA3kC0iKcCXga+d7g1E5HYR2S4i21tbrTkhHGza1cDqkgyyks995PRkZCTFcUF5NjtrOznY1BvU9zImUoVqJ/VXge+qat/pDlLV+1R1raquzc09t7l8zMw51NzLgaZerl85e0be77IFucTHuvjXTfvw+WwKDmPOVjDXg6gHxi4LVhTYNtExdSISA6QD7cB64EYR+TaQAfhEZEhVfxDEvCZITl5q+uz+JgQY9viIi3EH/X2T4mO4ZlkBv3mrnl9WHOdTF5YG/T2NiSTBLBDbgPkiUoa/ENwC3DbumE3AZ4A3gBuBF9U/29olJw8Qka8CfVYcwpuqsqeum7mzUkhNiJ2x9107J5O2vmH+83cHuHBuNvNmpc7YexsT7oLWxBToU7gTeAaoAn6lqpUi8nURuS5w2E/x9zkcAb4IvOdSWBMZ6joH6egfYWVR+oy+r4jw7RtXkBTn5vYHd9AzNDqj729MOAvqkqOquhnYPG7b3WPuDwGfOMNrfDUo4cyM2lXXhdslLCmY2QIBUJCeyL23reGTP6ngzo1v8ZNPryUuxjXhKGtbstSYd4RqJ7WJID5V9tZ1szAvlcS44Pc9TGR9eTb/8bHlvHKolS89uts6rY2ZhKCeQRgDUN3aT9+wh5VBmFrjbNx0fjHt/SN86/cHyEyKZUFe6lmtgW1MtLECYYJud10XcTEuFuU730H8+UvL6egf5sevHuOKxbO4YlGe05GMCVlWIExQDXu8VDZ0s7QgjVi38y2aIsI/fmgxHf2jPL6zjuS4GC4oz3Y6ljEhyfmfWBPRXjrYytCoz/HmpbFEhG/dsJxF+ak8ubuBwy020tqYiViBMEG1aVcDSXFu5uamOB3lXWLcLm45v4SclHh+vbOeoVFbYMiY8axAmKDpHRrl+apmlhemT+vCQNMlLsbFjecV0TM4yua9jU7HMSbkWIEwQbN5byPDHh+rQ6h5abzirCQunpfDjuOdNPcMOR3HmJBiBcIEzeM76inPSaY4K8npKKd12YJc4mJcPLe/2ekoxoQUKxAmKGrbB9ha08EN5xWF/FiDpPgYLpmfw/7GHvbUdTkdx5iQYZe5mqB4fGcdIvCx1YW85PDSnxNNqTHexXNzePVwG3c/UclNa4vfs9+m4DDRyM4gzLTz+ZRfv1XHRXOzmZ2R6HScSYmPdbOmJJO9dd302oR+xgBWIEwQbKvp4ETHIDeeV+R0lLNyYXk2XlW21nQ4HcWYkGAFwky7x3bUkRzn5qql+U5HOSs5qfHMn5XC9ppOfGqT+RljBcJMq4ERD5v3NvKh5QUkxYVfF9d5czLpHhylurXf6SjGOM4KhJlWz1Q20T/i5YYwa146aXFBGgmxLt6q7XQ6ijGOswJhptXjO+opzkpkXWmW01HOSazbxfLCdCobehjx+JyOY4yjrECYadPQNcjrR9v4+OoiXCE4tcZkrSrOZMTro7Kh2+koxjgq/BqJTUiYaGzBSwdbUIUb1oRn89JJc7KTSE+MZW99N6tLMp2OY4xjrECYaaGq7DrRxZysJF470gZHnE507lwiLC9M542j7QyOeB1bJtUYp1kTk5kWTT1DtPQOh9S6D1OxvDAdryr7G3ucjmKMY6xAmGmxp64bl8CywnSno0yLosxEMpNi2Vvf5XQUYxxjBcJMmaqyu66LebNSSImPjFZLEWF5YQZHWvoYGPY4HccYR1iBMFNW2zFA18AoK4synI4yrVYUpeNTqGywZiYTnaxAmCnbXddNjEtYUpDmdJRpVZCeQHZyHHusmclEKSsQZkq8PmVvfTeLCtKIj42sq31EhBVF6VS39tPaO+x0HGNmnBUIMyXVrX30D3tYWRQZndPjLS/KQIHf77M1q030sQJhpmR3XTfxMS4W5KU6HSUo8tMSmJUaz5N7rECY6GMFwpwzr0+pauxhcUEase7I/VZaXpTOtpoOmnuGnI5izIyK3J9qE3THO/oZHPWyOMI6p8dbUZiBKmzea2cRJrpYgTDnrKqhB7dLWDArxekoQZWbGs/igjSesmYmE2WsQJhzoqpUNfUyNzc54q5emsiHVxSw43gnDV2DTkcxZsZYgTDnpKV3mI7+kYhvXjrpIytmA/C0nUWYKGIFwpyTQ829ACzKj44CUZKdxIqidJ7a0+B0FGNmjBUIc04ONveSlxZPemKs01FmzLXLC9hd101t+4DTUYyZEVYgzFnrH/ZwvG0gYsc+nMq1KwoAeGJXvcNJjJkZViDMWdtytB2vatQViKLMJC6el83D207g9anTcYwJuqAWCBG5WkQOisgREblrgv3xIvJIYH+FiJQGtq8TkV2B224R+Vgwc5qz8/KhFuLcLuZkJzkdZcZ9cv0c6rsGeeVQq9NRjAm6oBUIEXED9wLXAEuAW0VkybjDPgt0quo84LvAtwLb9wFrVXUVcDXwIxGJjIUGIsCWI+2U5SQT44q+E9ANS/LITY3nF28edzqKMUEXzJ/wdcARVa1W1RHgYeD6ccdcDzwQuP8YcIWIiKoOqOrJVVoSADufDxEtPUNUt/VTnpvsdBRHxLpd3HJ+MS8ebKGu0zqrTWQL5l/lhcCJMY/rgPWnOkZVPSLSDWQDbSKyHvgZMAf41JiC8TYRuR24HaCkpGTa/wHmvbbWdABQlhNdBWJjRe3b9xNj3aDwyLYT/N2VCx1MZUxwhWwbgapWqOpS4HzgKyKSMMEx96nqWlVdm5ubO/Mho1BFdQfJcW4K0hOdjuKYjKQ4Fuan8vC2E4x6fU7HMSZoglkg6oHiMY+LAtsmPCbQx5AOtI89QFWrgD5gWdCSmkmrONbOeaVZuF3idBRHrSvLorV3mGcrm52OYkzQBLNAbAPmi0iZiMQBtwCbxh2zCfhM4P6NwIuqqoHnxACIyBxgEVATxKxmEjr6RzjU3Mf6siynozhuQV4qRZmJPPBGjdNRjAmaoBWIQJ/BncAzQBXwK1WtFJGvi8h1gcN+CmSLyBHgi8DJS2HfB+wWkV3Ab4AvqGpbsLKaydl6zN//cEG5FQiXCJ+5sJStxzqobOh2Oo4xQRHUPghV3ayqC1R1rqr+e2Db3aq6KXB/SFU/oarzVHWdqlYHtj+oqktVdZWqrlHV3wYzp5mcimPtJMS6WF6Y4XSUkHDT+cUkxbn5v9drnI5iTFCEbCe1CT0V1R2sKckkLsa+bQDSE2O5YU0Rm3Y10No77HQcY6ad/aSbSekeHKWqqYf1ZdlORwkZGytqyUmJZ8Tr465f73nXpbDGRAIrEGZSttd0oOq/ese8Izc1ngV5KWyt7sDjs0teTWSxAmHOaGNFLfe/XoPbJRxq7rW/lMe5aG4OvcMe9tZZZ7WJLFYgzKQca++nODORWLd9y4w3b1YKOSlxVASu8jImUthPuzmj4VEvDV2DlEbZ9BqT5RJhXWkWtR0DHGjqcTqOMdNmUgVCRH4tIteKiBWUKHS8YwCfRt/8S2djTUkmMS6x5jcTUSb7C/9/gNuAwyLyTRGxGcqiSE1bPy6BOVlWIE4lKT6GZYXp/GZnPYMjXqfjGDMtJlUgVPV5Vf0ksAb/lBfPi8gWEfkTEYmeRYmj1LG2fgozEm38wxmsnZNJ77CHZ/c3OR3FmGkx6Z94EckG/hj4M+At4L/xF4zngpLMhITBES91nYPWvDQJpTnJFGYk8vhOW7PaRIbJ9kH8BngVSAI+oqrXqeojqvqXQEowAxpnvXWiE6+qFYhJcInw0dWzee1wKy09Q07HMWbKJrtg0I9VdfPYDSISr6rDqro2CLlMiKio7kCAOdlWICYj1u3Cp/Cvmyq5ZP47a5Tctt4WtDLhZ7JNTN+YYNsb0xnEhKaKY+0UZCSQEOt2OkpYmJWaQGFGInvrbdCcCX+nPYMQkXz8y4Imishq4OQqMWn4m5tMBBv2eHmrtou1czKdjhJWls1O45n9zXQNjJCRFOd0HGPO2ZmamK7C3zFdBNwzZnsv8I9BymRCxJ66boY9PspyrJvpbCwtTOeZ/c1UNvRw8bwcp+MYc85OWyBU9QHgARG5QVUfn6FMJkRUVPtXfy3NtpPFs5GTEk9+WgKVDd1WIExYO1MT0x+p6i+AUhH54vj9qnrPBE8zEaLiWAeL8lNJip/stQzmpKWz03jxQAu9Q6OkJthQIROeztRJffLSlRQgdYKbiVCjXh87jnfa+tPnaOnsdBTY32hzM5nwdaYmph8Fvn5tZuKYULGvvpuBES/ryrLpHhx1Ok7YyUuLJzs5jsoGW2TJhK/JDpT7toikiUisiLwgIq0i8kfBDmecc3Lqalsg6NyICMsK06lu7WNgxON0HGPOyWTHQVypqj3Ah/HPxTQP+PtghTLO23qsg7m5yeSmxjsdJWwtnZ2GT6GqsdfpKMack8kWiJNNUdcCj6qqjQKKYF6fsu1YB+usaWRKCjMSyUiMpbLBflxMeJpsgXhKRA4A5wEviEguYJPNRKiqxh56hz1cUG7NS1MhIiyencaRFmtmMuFpstN93wVcBKxV1VGgH7g+mMGMc6z/Yfoszk/D41NePdzmdBRjztrZXOC+CP94iLHP+fk05zEhoKK6nZKsJArSE52OEvbKcpJJiHXx3P5mrlqa73QcY87KpAqEiDwIzAV2ASeXy1KsQEQcn0/ZWtPBhsV5TkeJCG6XsDAvlRcPtOD1KW6XnPlJxoSIyZ5BrAWWqKoGM4xx1saKWpp6hugaGEUVW195miwuSGN3XTc7azs5v9Sa7Uz4mGwn9T7Azo+jwLG2fsC/OpqZHgvyUol1C8/tb3Y6ijFnZbIFIgfYLyLPiMimk7dgBjPOONbWT3piLJlJNn/QdEmIdXNBeTbP7W/GTsJNOJlsE9NXgxnChAZVpaatn3mzUhCxtvLpdOWSPP7liUqOtvr/f40JB5O9zPVl/COoYwP3twE7g5jLOKCld5i+YQ/l1rw07T64xN/pb81MJpxMdi6mzwGPAT8KbCoEfhukTMYh1a19AMzNtb9wp1tBeiLLCtN4bn+T01GMmbTJ9kHcAVwM9ACo6mFgVrBCGWccbe0nMymWzGRbJjMYNizO560TXbT2DjsdxZhJmWyBGFbVkZMPAoPlrLctgnh9yrG2fjt7CKINS/JQhRcPWDOTCQ+TLRAvi8g/AokisgF4FHgyeLHMTKtq7GFw1Eu5FYigWVyQSmFGovVDmLAx2QJxF9AK7AX+HNgM/HOwQpmZt+Wof66g8lzroA4WEWHDkjxePdxmk/eZsDDZq5h8+Dulv6CqN6rqjyczqlpErhaRgyJyRETummB/vIg8EthfISKlge0bRGSHiOwNfP3A2f2zzNnacrSd3NR40mz95KDasCSPYY+PVw61Oh3FmDM6bYEQv6+KSBtwEDgYWE3u7jO9sIi4gXuBa4AlwK0ismTcYZ8FOlV1HvBd4FuB7W3AR1R1OfAZ4MGz+UeZszPq9b29QJAJrvVlWeSkxLFpd4PTUYw5ozOdQfwt/quXzlfVLFXNAtYDF4vI357hueuAI6paHejgfpj3ThF+PfBA4P5jwBUiIqr6lqqe/AmqxN/3YUubBcmeui4GRryU51j/Q7DFuF1cu7yAF6pa6B2ytb5NaDvTSOpPARtU9e3J7FW1OrAe9bP4/+o/lULgxJjHdfiLy4THqKpHRLqBbPxnECfdAOxUVbs2MEi2HGlHBBsgF0RjJz5MjHUz7PHxbGUzN5xX5GAqY07vTGcQsWOLw0mq2goEvbFaRJbib3b681Psv11EtovI9tZWa9M9V1uOtrOkII2k+LNZHsScq+KsJDKTYnnCmplMiDtTgRg5x30A9UDxmMdFgW0THhMYW5EOtAceFwG/AT6tqkcnegNVvU9V16rq2tzc3DPEMRMZHPGyo7aTC8tt/emZIiKsKMrg9SNttPXZibEJXWcqECtFpGeCWy+w/AzP3QbMF5EyEYkDbgHGzwC7CX8nNMCNwIuqqiKSATwN3KWqr5/Vv8iclTer2xnx+Hj/AiuwM2llcQZen7J5b6PTUYw5pdMWCFV1q2raBLdUVT1tE5OqeoA7gWeAKuBXqlopIl8XkesCh/0UyBaRI8AX8Y+3IPC8ecDdIrIrcLOpPYLg5UOtJMS6bP3pGZaflsCi/FSe2GXNTCZ0BbXRWVU34x9UN3bb3WPuDwGfmOB53wC+Ecxsxu+VQ61cUJ5NQqzb6ShR57pVs/n27w9yomOA4qwkp+MY8x6THUltIlBt+wDVbf1cas1Ljrhu5WwAHt9Z53ASYyZmBSKKvXzYf+WX9T84oygziUvm5/CrbSfw+mzuSxN6rEBEsVcOtVKUmWjjHxz0yfUlNHQP8dLBFqejGPMeViCi1IjHx5YjbVy6INeWF3XQFYvzyE2N56GttWc+2JgZZgUiSu043kn/iNf6HxwW63Zx09oiXjzQQkPXoNNxjHkXKxBR6uVDrcS4hIvm5TgdJerdcn4JCjyy7cQZjzVmJlmBiFIvH2rlvDmZpNj0Go4rzkrikvm5PLLtBB6vz+k4xrzNCkQUauoeoqqxh0sXWvOSkzZW1L59K85MpKlniK89ud/pWMa8zf58jEIvBNZEHh71vWuWUeOcRflppCXE8GZ1u9NRjHmbFYgo9EJVC5lJscxKtSU2QoXbJawry+L5qha+//xhcsZ9NretL3EomYlm1sQUZQZHvLx+pI1FBWl2eWuIOb80C7cIFcfsLMKEBisQUea1I20Me3wszk9zOooZJzUhlqWFaeyo7WTY43U6jjFWIKLN8/ubSY2PoTTHJocLRReWZzM06mPXiS6noxhjBSKa+HzKCwdaeP/CXGJc9tGHopKsJGanJ1BR3YGqzc9knGW/JaLInvpu2vqG+eBiW1ojVIkIF5Rn09QzRE37gNNxTJSzAhFFnt/fjNslXL7QCkQoW1GUQWKsmzfsklfjMCsQUeT5qmbOm5NJRlKc01HMacTFuFg7J5P9Dd10D446HcdEMSsQUWBjRS33/uEIB5p6yU6Os8FxYWB9eTaqsK2mw+koJopZgYgSB5p6Af+IXRP6spLjWJCXyrZjHXh8Nj+TcYYViChxoLGHnJQ4cm30dNi4oDyb3mEPlQ09TkcxUcoKRBQYHvVS3dZvZw9hZn5eClnJcbx51DqrjTOsQESBwy19eH3KooJUp6OYs+AKXPJ6vGOAyoZup+OYKGQFIgpUNfaQGOtmTpatPR1uzivJJNYtPPjGcaejmChkBSLCebw+DjT1sjA/FbfLJucLN4lxblYVZ/DbXfV0D9glr2ZmWYGIcNuPdzI46mVxgfU/hKsLAvMzPbrDliQ1M8sKRIR7trKZGJewYFaK01HMOSpIT+T80kwefPM4Pp/Nz2RmjhWICKaqPFfVxNzcFOJj3U7HMVPw6QtLOd4+wMuHW52OYqKIFYgIdrC5lxMdg9a8FAGuWprPrNR4fr6lxukoJopYgYhgz1X61562y1vDX1yMi1vXlfDSoVaOt/c7HcdECSsQEezZ/c2sLskgLSHW6ShmGty2vgS3CA9ssUtezcywAhGhGrsH2VvfzYYleU5HMdMkLy2Bj6yczcPbaunsH3E6jokCViAi1PP7/c1LV1qBiCh/cdlcBka83G99EWYGWIGIUM/ub6YsJ5m5uXZ5ayRZkJfKhiV53L+lhr5hj9NxTISzAhGBeoZGebO6nSuX5CFio6cjzRcum0v34CgP2boeJsisQESglw62MupV63+IUKtLMrlobjY/frWaoVGv03FMBLMCEUE2VtSysaKWn7xaTXKcmwNNvbZ6XIS64/J5tPQO8/jOOqejmAgW43QAM708Ph8Hm3pZVpiOy5qXIsb4Qq+qrCrO4N4Xj3DjeUXEx9hIeTP9gnoGISJXi8hBETkiIndNsD9eRB4J7K8QkdLA9mwR+YOI9InID4KZMdIca+1n2ONjiY2ejmgiwpeuXEhD9xC/fNPOEk1wBO0MQkTcwL3ABqAO2CYim1R1/5jDPgt0quo8EbkF+BZwMzAE/AuwLHAzk7S/sYdYtzDPJueLeLUdA5TnJvOdZw8iwttnEbetL3E4mYkUwTyDWAccUdVqVR0BHgauH3fM9cADgfuPAVeIiKhqv6q+hr9QmElSVaoae5g/K5VYt3UvRYMrl+TTP+Jliy1LaoIgmL9FCoGxE9jXBbZNeIyqeoBuIHuybyAit4vIdhHZ3tpqs1zWdw3SM+SxyfmiSElWEosL0nj1cCsDIzYuwkyvsP4zU1XvU9W1qro2NzfX6TiOq2rsQYBF+TY5XzTZsCSP4VEfrxyyP5LM9ApmgagHisc8Lgpsm/AYEYkB0gE7Vz5HlQ09zMlOJjneLk6LJvlpCawqzmDL0Xa6BmyOJjN9glkgtgHzRaRMROKAW4BN447ZBHwmcP9G4EVVtSWzzsGh5l5aeodZXmjNS9Ho5KDI5wJzcBkzHYJWIAJ9CncCzwBVwK9UtVJEvi4i1wUO+ymQLSJHgC8Cb18KKyI1wD3AH4tInYgsCVbWSPD0nkYEWFqY7nQU44CMpDgumpvDrhNd7KvvdjqOiRBBbYtQ1c3A5nHb7h5zfwj4xCmeWxrMbJFm895G5mQn29oPUeyyhblsP97Bf2yu4pd/tt7m4TJTFtad1MbvUHMvh1v6WF5kZw/RLCHWzQcWzWLL0XZeOmgd1mbqrEBEgKf3NCICy2Zb/0O0W1eWRWl2Ev/5uyo8Xp/TcUyYswIRAZ7e28i60ixSrXkp6sW4XHz56kUcau7jsR02kZ+ZGisQYe5Qcy9HWvr48IoCp6OYEHH1snzOm5PJPc8dot8WFTJTYAUizD0VaF66alm+01FMiHho6wnOn5NJS+8wf/XQWzbluzlnViDCmKry1O4G1pdlMSs1wek4JoSUZCezbHYarx5uo3do1Ok4JkxZgQhjb53oorqtn4+vLnI6iglBVy3Nx+tTnq+ywXPm3FiBCGOP76gjIdbFNcutecm8V3ZKPBeUZ7G9ppNdJ7qcjmPCkBWIMPXAlhoe31nHovw0ntzdaO3MZkJXLM4jJSGGf/7tXrw+m8XGnB0rEGFqf2MPQ6M+VpdkOB3FhLCEWDfXLi9gX30Pv3jzuNNxTJixAhGmKqrbyUqOY26urRxnTm95YTqXzM/hv545SEuPrcFlJs8KRBg60NRDTfsA68uycNl8O+YMRISvX7+MYa+Pf3u6yuk4JoxYgQhDv3jzODEu4bySTKejmDBRlpPMnZfP48ndDTy5u8HpOCZMWIEIMx39Izy+o54VRRkk2cJA5ix84bK5rC7J4J9+s5f6rkGn45gwYAUizNy/pYbBUS+XzM9xOooJMzFuF9+7eRU+hb/4xQ6GRr1ORzIhzgpEGOkb9nD/68e4ckkeeWk2ctqcvTnZyXzv5lXsre/my4/vwWeXvprTsDaKMHL/68foGfLwhcvnsb+hx+k4JoyMHyezYXEeT+xqID0xlq9dt9QWFzITsgIRJlp7h/nhS0e5ckkeq4ozrECYKbl0QS6Do15+/sZxPD7l69ctJcZtDQrm3axAhInvPX+IYY+Pu65Z5HQUEwFEhKuX5rOsMJ0fvnSU+s5B7rlpJdkp8U5HMyHE/mQIAztrO3loay1/dMEcym1gnJkmIsKXr17Ef3xsOW8cbeeq773Cc/ttYj/zDisQIW5o1MvfP7qb/LQE/u7KBU7HMRHotvUlbPrLi8lNTeBzP9/OPzy2m+4BmyLcWBNTyPvm7w5wtLWfB/50nS0paqbd2M7rW88v5oUDLTy6vY6n9zZx7fJ8VhZlICLctr7EwZTGKXYGEcKe2FXP/Vtq+NOLy7h0Qa7TcUyEi3G7uGppPndcPo/MpFh+tb2O/3u9hra+YaejGYdYgQhRO4538OXH97CuLIuvfMg6ps3MmZ2RyOcvnct1K2dzonOA779wmB+8eBiP1+d0NDPDRDUyBsqsXbtWt2/f7nSMaXGwqZebfvQGWclx3LquhBSbUsM4pGdolKf3NLK3vpsVRel85xMrmZ+X6nQsM41EZIeqrp1on51BhJCNFbX81zMH+dj/vI5PlRvWFFlxMI5KS4jl1nUl/M8n11DXOci133+N/335qC0+FCWsQISQEx0D/OS1auLcLm6/pJys5DinIxkDwIeWF/Ds376fDyyaxTd/d4Ab/3cLR1v7nI5lgswKRIjYVtPBz14/RlJcDJ97f7kNWDIhZWNFLc9WNnPJ/BxuXlvMgcZervruK/zk1WqbzymCWYEIAa8faePTP91KakIsn7uknMwkO3MwoUlEWFmcwV9/cD7zZqXwjaeruOW+N6ls6HY6mgkCKxAO+8OBFv7k/m3MyU7ic5eUkZ5oYx1M6EtLiOVTF8zhO59YycHmXq79/mvcuXGnNTtFGCsQDtq8t5HbH9zOwrxUHvrcBTYQzoQVEeGG84p45R8u5y8/MI8XD7Sw4Z6X+Ytf7OCNo+1EyhWS0cwuc3XIHb/cyea9jRRnJfHHF5WSEOt2OpIxU9I37OG1w21sq+lgcNRLXlo8F5Rns6o4g/gYt43GDlGnu8zVrqGcYT6f8u+bq3h6byNLCtK4+fxiYm2aZRMBUuJjuHpZPlcsnsWeui7eONrOE7sa+P2+JlYVZ7CiKJ1lhelOxzRnwc4gZlD3wChfemw3z+1v5sLybK5dUYDLFmoxEUpVqe0YYOuxDvbWd+PxKcsL07l1XQnXrZptY3xCxOnOIKxAzJA/HGjhn3+7j5beIf7xQ4uJc7tsFS8TNQZHvLhd8NDWExxs7iUpzs21ywu4YnEeF8/Ltv43B1kTk4N2HO/kBy8e5g8HW5mbm8yjn7+IVcUZ71kC0phIlhjn72P79IVzONE5yLaaDjbtbuDRHXXEuIQ1JZksLkhlXl4qc3OSyUqJIyMxjvTEWBJi7Y8pp1iBmGYjHh/7G3t49VArT+9t5EBTLxlJsXz56kV89n1lxMVYf4OJXiJCSVYSJVlJXL9qNrUdAxxq6uNYWx+767oY9rx3QkARSIx1I0BcjItYt4u4GBepCbGsL8tidkYCpdnJzJuVQklWki2dOo2CWiBE5GrgvwE38BNV/ea4/fHAz4HzgHbgZlWtCez7CvBZwAv8lao+E8ys52Jo1EtVYw/7GnqorO/mlcOtNPcMvz1PTXFmIv92/VI+vqaIZGtvNeZdYlwuynNSKM/xr5KoqvQMeWjrG2ZgxMvgiJfBUS8jHh+jXh8jXh+jHv/XEY+Pjv5hHt5Wy9DoO0XF7RJyUuJYOyeLubNSmD8rhXmzUijLSbYrBc9B0PogRMQNHAI2AHXANuBWVd0/5pgvACtU9fMicgvwMVW9WUSWAA8B64DZwPPAAlX1nur9pqMPQlUZ9Soen49RjzLi9dE9OErXwAgd/SOc6BzkeHs/x9r6Od4+QF3nACdnGchMiiU7OZ7ZGQnMzkikPDfFOuGMmQFDo15ae4dp6R2mtXco8HWYjv4RTv52E6AkO4mynGSykv1NVxmJcWQkxZIY5ybGJbhdQozLFfjqfywCLpHAzf9CYx/LuK8uefdz/PdP7n/nGACX673PEcY81wVu8edwvf2VaW9uc6oPYh1wRFWrAyEeBq4H9o855nrgq4H7jwE/EP+//nrgYVUdBo6JyJHA670x3SH31HVx04/ewONVPJOYUyY1IYaynGRWFmfw0dWFLJ2dxrLCdGanJ/DQ1hPTHc8YcwYJsW6Ks5Iozkp61/ZRr4+2vuG3i0dirJvjHf0caemje2CU3mGPQ4mn5mRRcbkEd6C4XLOsgO/ctHLa3yuYBaIQGPsbsw5Yf6pjVNUjIt1AdmD7m+OeWzj+DUTkduD2wMM+ETk4PdFPb9+pd+UAbTOR4RxZvqmxfFNj+abmlPmqgHtuPufXnXOqHWHdBqKq9wH3OZ3jJBHZfqpTtVBg+abG8k2N5ZsaJ/IFs7u/Hige87gosG3CY0QkBkjH31k9mecaY4wJomAWiG3AfBEpE5E44BZg07hjNgGfCdy/EXhR/b3mm4BbRCReRMqA+cDWIGY1xhgzTtCamAJ9CncCz+C/zPVnqlopIl8HtqvqJuCnwIOBTugO/EWEwHG/wt+h7QHuON0VTCEkZJq7TsHyTY3lmxrLNzUzni9iptowxhgzvWzIoTHGmAlZgTDGGDMhKxDTQESuFpGDInJERO5yOg+AiPxMRFpEZN+YbVki8pyIHA58zXQoW7GI/EFE9otIpYj8dYjlSxCRrSKyO5Dva4HtZSJSEficHwlcfOEYEXGLyFsi8lSo5RORGhHZKyK7RGR7YFtIfL6BLBki8piIHBCRKhG5MFTyicjCwP/byVuPiPyNE/msQExRYEqRe4FrgCXArYGpQpx2P3D1uG13AS+o6nzghcBjJ3iAv1PVJcAFwB2B/7NQyTcMfEBVVwKrgKtF5ALgW8B3VXUe0Il/rjAn/TX+MVInhVq+y1V11Zhr90Pl8wX/HHG/V9VFwEr8/48hkU9VDwb+31bhn6duAPiNI/lU1W5TuAEXAs+MefwV4CtO5wpkKQX2jXl8ECgI3C8ADjqdMZDlCfxzdoVcPiAJ2Il/FoA2IGaiz92BXEX4f0l8AHgK/3RDoZSvBsgZty0kPl/8462OEbhIJ9Tyjct0JfC6U/nsDGLqJppS5D3TgoSIPFVtDNxvAvKcDAMgIqXAaqCCEMoXaL7ZBbQAzwFHgS5VPTmBj9Of8/eAfwBOTmWaTWjlU+BZEdkRmBIHQufzLQNagf8LNNH9RESSQyjfWLfgn7gUHMhnBSJKqf/PEEevcRaRFOBx4G9UtWfsPqfzqapX/af4RfgnilzkVJbxROTDQIuq7nA6y2m8T1XX4G96vUNE3j92p8OfbwywBvihqq4G+hnXXOP09x9AoA/pOuDR8ftmKp8ViKkLp2lBmkWkACDwtcWpICISi784/FJVfx1q+U5S1S7gD/ibbDICU8KAs5/zxcB1IlIDPIy/mem/CZ18qGp94GsL/vbzdYTO51sH1KlqReDxY/gLRqjkO+kaYKeqNgcez3g+KxBTN5kpRULF2KlNPoO/7X/GiYjgH0Vfpar3jNkVKvlyRSQjcD8Rf/9IFf5CcaPT+VT1K6papKql+L/fXlTVT4ZKPhFJFpHUk/fxt6PvI0Q+X1VtAk6IyMLApivwz9oQEvnGuJV3mpfAiXxOd8JEwg34EP7FkY4C/+R0nkCmh4BGYBT/X0yfxd9O/QJwGP8iTFkOZXsf/tPjPcCuwO1DIZRvBfBWIN8+4O7A9nL8c4IdwX/aHx8Cn/NlwFOhlC+QY3fgVnnyZyJUPt9AllXA9sBn/FsgM8TyJeOfuDR9zLYZz2dTbRhjjJmQNTEZY4yZkBUIY4wxE7ICYYwxZkJWIIwxxkzICoQxxpgJWYEwxhgzISsQxhhjJvT/AabpkmNr5g01AAAAAElFTkSuQmCC\n", + "text/plain": [ + "
" + ] + }, + "metadata": { + "needs_background": "light" + }, + "output_type": "display_data" + } + ], + "source": [ + "sns.distplot(out)" + ] + }, + { + "cell_type": "markdown", + "id": "52fda8ba-d6cd-4447-9b3b-3b74fd8d0a08", + "metadata": {}, + "source": [ + "We can see our distribution is fairly smooth, which happens when we have enough tests that have any probability of failing. We will also output the true distribution of scores to compare." + ] + }, + { + "cell_type": "code", + "execution_count": 615, + "id": "de14103d-3f09-47a4-9eb5-d096064432cb", + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "" + ] + }, + "execution_count": 615, + "metadata": {}, + "output_type": "execute_result" + }, + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYIAAAD4CAYAAADhNOGaAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjQuMywgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/MnkTPAAAACXBIWXMAAAsTAAALEwEAmpwYAAATwUlEQVR4nO3df5BdZ33f8fcH2YI2MT+1TW1JIEFEqCYJ2N0o6YQhLsGp5CRSCDRIE1rcMdGkRUBK0qkYqOoqmWlxhmSSjtKJAm4cEhCuG5LNRFShYA8tww+tHdsgu4JFsSMpTrwQArQZYgTf/nGPwmV9d/dqd8+92j3v18ydPT+ee+/38bHvx+d5zj03VYUkqbueNO4CJEnjZRBIUscZBJLUcQaBJHWcQSBJHXfFuAu4VBs2bKgtW7aMuwxJWlXuueeez1XVxKB9qy4ItmzZwvT09LjLkKRVJckj8+1zaEiSOs4gkKSOMwgkqeMMAknqOINAkjrOIJCkjms1CJLsTHI6yUySgwP2PzvJXUn+OMkDSW5ssx5J0hO1FgRJ1gFHgF3AdmBfku1zmr0VuKOqrgX2Ar/WVj2SpMHaPCPYAcxU1Zmqehw4BuyZ06aApzbLTwP+rMV6JEkDtBkEG4Gzfevnmm39bgFeneQccBx4/aAXSrI/yXSS6dnZ2SUXtOWaa0iy5MeWa65Z8ntL0uVq3LeY2Af8ZlW9Pck/At6V5Dur6uv9jarqKHAUYHJycsk/qfbIo49S11+/5GJz991Lfq4kXa7aPCM4D2zuW9/UbOt3M3AHQFV9FHgKsKHFmiRJc7QZBCeBbUm2JllPbzJ4ak6bPwV+ECDJP6AXBEsf+5EkXbLWgqCqLgAHgBPAQ/SuDjqV5HCS3U2znwV+Ksn9wHuAm6pqyUM/kqRL1+ocQVUdpzcJ3L/tUN/yg8D3t1mDJGlhfrNYkjrOIJCkjjMIJKnjDAJJ6jiDQJI6ziCQpI4zCCSp4wwCSeo4g0CSOs4gkKSOMwgkqeMMAknqOINAkjrOIJCkjjMIJKnjDAJJ6rhWgyDJziSnk8wkOThg/y8nua95fDrJX7VZjyTpiVr7hbIk64AjwA3AOeBkkqnmV8kAqKp/3df+9cC1bdUjSRqszTOCHcBMVZ2pqseBY8CeBdrvo/e7xZKkEWozCDYCZ/vWzzXbniDJc4CtwIdarEeSNMDlMlm8F7izqr42aGeS/Ummk0zPzs6OuDRJWtvaDILzwOa+9U3NtkH2ssCwUFUdrarJqpqcmJhYwRIlSW0GwUlgW5KtSdbT+7CfmtsoyQuAZwAfbbEWSdI8WguCqroAHABOAA8Bd1TVqSSHk+zua7oXOFZV1VYtkqT5tXb5KEBVHQeOz9l2aM76LW3WIEla2OUyWSxJGhODQJI6ziCQpI4zCCSp4wwCSeo4g0CSOs4gkKSOMwgkqeMMAknqOINAkjrOIJCkjjMIJKnjDAJJ6jiDQJI6ziCQpI4zCCSp4wwCSeq4VoMgyc4kp5PMJDk4T5ufSPJgklNJ3t1mPZKkJ2rtpyqTrAOOADcA54CTSaaq6sG+NtuANwPfX1VfSPL32qpHkjRYm2cEO4CZqjpTVY8Dx4A9c9r8FHCkqr4AUFWPtViPJGmANoNgI3C2b/1cs63f84HnJ/lIko8l2TnohZLsTzKdZHp2dralciWpm8Y9WXwFsA24HtgH/EaSp89tVFVHq2qyqiYnJiZGW6EkrXFtBsF5YHPf+qZmW79zwFRVfbWq/gT4NL1gkCSNSJtBcBLYlmRrkvXAXmBqTpvfo3c2QJIN9IaKzrRYkyRpjtaCoKouAAeAE8BDwB1VdSrJ4SS7m2YngM8neRC4C/g3VfX5tmqSJD1Ra5ePAlTVceD4nG2H+pYLeFPzkCSNwbgniyVJY2YQSFLHGQSS1HEGgSR1nEEgSR1nEEhSxxkEktRxBoEkdZxBIEkdZxBIUscZBJLUcQaBJHWcQSBJHWcQSFLHGQSS1HEGgSR1nEEgSR03VBAk+dEklxwaSXYmOZ1kJsnBAftvSjKb5L7m8dpLfQ9J0vIM++H+KuAzSW5N8oJhnpBkHXAE2AVsB/Yl2T6g6Xur6kXN4x1D1iNJWiFDBUFVvRq4Fvgs8JtJPppkf5KrFnjaDmCmqs5U1ePAMWDPsiuWJK2ooYd7qupLwJ30PtCvBl4O3Jvk9fM8ZSNwtm/9XLNtrlckeSDJnUk2D3qhJnSmk0zPzs4OW7IkaQjDzhHsSfI+4G7gSmBHVe0CXgj87DLe/w+ALVX13cAHgNsHNaqqo1U1WVWTExMTy3g7SdJcVwzZ7seBX66qD/dvrKq/TnLzPM85D/T/H/6mZlv/8z/ft/oO4NYh65EkrZBhh4b+fG4IJHkbQFV9cJ7nnAS2JdmaZD2wF5ia8xpX963uBh4ash5J0goZNghuGLBt10JPqKoLwAHgBL0P+Duq6lSSw0l2N83ekORUkvuBNwA3DVmPJGmFLDg0lORfAv8KeF6SB/p2XQV8ZLEXr6rjwPE52w71Lb8ZePOlFCxJWlmLzRG8G3g/8B+B/i+Efbmq/rK1qiRJI7NYEFRVPZzkdXN3JHmmYSBJq98wZwQ/AtwDFJC+fQU8t6W6JEkjsmAQVNWPNH+3jqYcSdKoLTZZfN1C+6vq3pUtR5I0aosNDb19gX0FvHQFa5EkjcFiQ0P/eFSFSJLGY7GhoZdW1YeS/Pig/VX1u+2UJUkalcWGhn4A+BDwowP2FWAQSNIqt9jQ0L9v/v6L0ZQjSRq1YW9D/awkv5rk3iT3JPmVJM9quzhJUvuGvencMWAWeAXwymb5vW0VJUkanWF/j+Dqqvr5vvVfSPKqNgqSJI3WsGcEf5Rkb5InNY+foHd7aUnSKrfY5aNf5hv3GPoZ4LebXU8C/i/wc20WJ0lq32JXDV01qkIkSeMx7BwBSZ4BbAOecnHb3J+vlCStPsNePvpa4MP05gX+Q/P3liGetzPJ6SQzSQ4u0O4VSSrJ5HBlS5JWyrCTxW8Evgd4pLn/0LXAXy30hCTrgCP0ftt4O7AvyfYB7a5qXv/jw5ctSVopwwbBV6rqKwBJnlxV/wf4jkWeswOYqaozVfU4ve8i7BnQ7ueBtwFfGbIWSdIKGjYIziV5OvB7wAeS/D7wyCLP2Qic7X+NZtvfan7vYHNV/eFCL5Rkf5LpJNOzs7NDlixJGsZQk8VV9fJm8ZYkdwFPA/7Hct44yZOAXwJuGuL9jwJHASYnJ2s57ytJ+maXctXQdcCL6X2v4CPNcM9CzgOb+9Y3Ndsuugr4TuDuJAB/H5hKsruqpoetS5K0PMNeNXQIuB14FrAB+K9J3rrI004C25JsTbIe2AtMXdxZVV+sqg1VtaWqtgAfAwwBSRqxYc8IfhJ4Yd+E8X8C7gN+Yb4nVNWFJAfoXWq6Dritqk4lOQxMV9XUfM+VJI3OsEHwZ/S+SHbxyp4n883DPANV1XHg+Jxth+Zpe/2QtUiSVtBi9xr6z/TmBL4InErygWb9BuAT7ZcnSWrbYmcEF8fr7wHe17f97laqkSSN3GI3nbv94nIz4fv8ZvV0VX21zcIkSaMx1BxBkuvpXTX0ML1bUm9O8hpvOidJq9+wk8VvB36oqk4DJHk+8B7gH7ZVmCRpNIa9xcSVF0MAoKo+DVzZTkmSpFEa9ozgniTv4Bu/UPaTfGMiWZK0ig0bBD8NvA54Q7P+v4Bfa6UiSdJILRoEze8K3F9VL6B3kzhJ0hqy6BxBVX0NOJ3k2SOoR5I0YsMODT2D3jeLPwH8v4sbq2p3K1VJkkZm2CD4d61WIUkam8XuNfQUehPF3w58EnhnVV0YRWGSpNFYbI7gdmCSXgjsovfFMknSGrLY0ND2qvougCTvxDuOStKas9gZwd/eWM4hIUlamxY7I3hhki81ywH+TrMeoKrqqa1WJ0lq3YJnBFW1rqqe2jyuqqor+pYXDYEkO5OcTjKT5OCA/T+d5JNJ7kvyv5NsX05nJEmXbtibzl2y5hvJR+hNMm8H9g34oH93VX1XVb0IuBW/uSxJI9daEAA7gJmqOlNVjwPHgD39DarqS32r30LvZzAlSSM07BfKlmIjcLZv/RzwvXMbJXkd8CZgPfDSQS+UZD+wH+DZz/ZOF5K0kto8IxhKVR2pqucB/xZ46zxtjlbVZFVNTkxMjLZASVrj2gyC88DmvvVNzbb5HAN+rMV6JEkDtBkEJ4FtSbY2P3y/F5jqb5BkW9/qDwOfabEeSdIArc0RVNWFJAeAE8A64LaqOpXkMDBdVVPAgSQvo/fFtS8Ar2mrHknSYG1OFlNVx4Hjc7Yd6lt+Y5vvL0la3NgniyVJ42UQSFLHGQSS1HEGgSR1nEEgSR1nEEhSxxkEktRxBoEkdZxBIEkdZxBIUscZBJLUcQaBJHWcQSBJHWcQSFLHGQSS1HEGgSR1nEEgSR3XahAk2ZnkdJKZJAcH7H9TkgeTPJDkg0me02Y9kqQnai0IkqwDjgC7gO3AviTb5zT7Y2Cyqr4buBO4ta16JEmDtXlGsAOYqaozVfU4cAzY09+gqu6qqr9uVj8GbGqxHknSAG0GwUbgbN/6uWbbfG4G3j9oR5L9SaaTTM/Ozq5giZKky2KyOMmrgUngFwftr6qjVTVZVZMTExOjLU6S1rgrWnzt88DmvvVNzbZvkuRlwFuAH6iqv2mxHknSAG2eEZwEtiXZmmQ9sBeY6m+Q5Frg14HdVfVYi7VIkubRWhBU1QXgAHACeAi4o6pOJTmcZHfT7BeBbwX+W5L7kkzN83KSpJa0OTREVR0Hjs/Zdqhv+WVtvr8kaXGXxWSxJGl8DAJJ6jiDQJI6ziCQpI4zCCSp4wwCSeo4g0CSOs4gkKSOMwgkqeMMAknqOINAkjrOIJCkjjMIJKnjDAJJ6jiDQJI6ziCQpI4zCCSp41oNgiQ7k5xOMpPk4ID9L0lyb5ILSV7ZZi2SpMFaC4Ik64AjwC5gO7AvyfY5zf4UuAl4d1t1SJIW1uZvFu8AZqrqDECSY8Ae4MGLDarq4Wbf11usQ5K0gDaHhjYCZ/vWzzXbLlmS/Ummk0zPzs6uSHGSpJ5VMVlcVUerarKqJicmJsZdjiStKW0GwXlgc9/6pmabJOky0mYQnAS2JdmaZD2wF5hq8f0kSUvQWhBU1QXgAHACeAi4o6pOJTmcZDdAku9Jcg74p8CvJznVVj2SpMHavGqIqjoOHJ+z7VDf8kl6Q0aSpDFZFZPFkqT2GASS1HEGgSR1nEEgSR1nEEhSxxkEl+BKIMmSHluuuWbc5UvSQK1ePrrWfBWo669f0nNz990rWYokrRjPCCSp4wwCSeo4g0CSOs4gkKSOMwhGxCuOJF2uvGpoRLziSNLlyjMCSeo4g0Dz2nLNNUsezkrC3123zuGwEVnOsfI4yaEhzeuRRx9d8nAW9Ia0HA4bjeUcK4+TPCOQpI5rNQiS7ExyOslMkoMD9j85yXub/R9PsqXNerpoOUMGklbWcodb2xqKa21oKMk64AhwA3AOOJlkqqoe7Gt2M/CFqvr2JHuBtwGvaqumLlrukIGklbMSw61taPOMYAcwU1Vnqupx4BiwZ06bPcDtzfKdwA/G/xWVpJFKVbXzwskrgZ1V9dpm/Z8B31tVB/rafKppc65Z/2zT5nNzXms/sL9Z/Q7g9BLL2gB8btFWa4/97o4u9hns9zCeU1UTg3asiquGquoocHS5r5NkuqomV6CkVcV+d0cX+wz2e7mv0+bQ0Hlgc9/6pmbbwDZJrgCeBny+xZokSXO0GQQngW1JtiZZD+wFpua0mQJe0yy/EvhQtTVWJUkaqLWhoaq6kOQAcAJYB9xWVaeSHAamq2oKeCfwriQzwF/SC4s2LXt4aZWy393RxT6D/V6W1iaLJUmrg98slqSOMwgkqeM6EwSL3e5iLUnycJJPJrkvyXSz7ZlJPpDkM83fZ4y7zuVIcluSx5rvolzcNrCP6fnV5tg/kOS68VW+PPP0+5Yk55vjfV+SG/v2vbnp9+kk/2Q8VS9fks1J7kryYJJTSd7YbF+zx3yBPq/88a6qNf+gN1n9WeC5wHrgfmD7uOtqsb8PAxvmbLsVONgsHwTeNu46l9nHlwDXAZ9arI/AjcD7gQDfB3x83PWvcL9vAX5uQNvtzb/rTwa2Nv8NrBt3H5bY76uB65rlq4BPN/1bs8d8gT6v+PHuyhnBMLe7WOv6b+dxO/Bj4ytl+arqw/SuNOs3Xx/3AL9VPR8Dnp7k6pEUusLm6fd89gDHqupvqupPgBl6/y2sOlX1aFXd2yx/GXgI2MgaPuYL9Hk+Sz7eXQmCjcDZvvVzLPwPdLUr4I+S3NPcngPg26rq0Wb5z4FvG09prZqvj104/geaIZDb+ob91mS/07tL8bXAx+nIMZ/TZ1jh492VIOiaF1fVdcAu4HVJXtK/s3rnkWv6uuEu9LHPfwGeB7wIeBR4+1iraVGSbwX+O/AzVfWl/n1r9ZgP6POKH++uBMEwt7tYM6rqfPP3MeB99E4P/+LiqXHz97HxVdia+fq4po9/Vf1FVX2tqr4O/AbfGA5YU/1OciW9D8TfqarfbTav6WM+qM9tHO+uBMEwt7tYE5J8S5KrLi4DPwR8im++ncdrgN8fT4Wtmq+PU8A/b64k+T7gi33DCavenLHvl9M73tDr9970fgBqK7AN+MSo61sJSULvTgQPVdUv9e1as8d8vj63crzHPTM+whn4G+nNun8WeMu462mxn8+ld+XA/cCpi30FngV8EPgM8D+BZ4671mX28z30Tou/Sm8s9Ob5+kjvypEjzbH/JDA57vpXuN/vavr1QPNhcHVf+7c0/T4N7Bp3/cvo94vpDfs8ANzXPG5cy8d8gT6v+PH2FhOS1HFdGRqSJM3DIJCkjjMIJKnjDAJJ6jiDQJI6ziCQpI4zCCSp4/4//xFtnwXUw4gAAAAASUVORK5CYII=\n", + "text/plain": [ + "
" + ] + }, + "metadata": { + "needs_background": "light" + }, + "output_type": "display_data" + } + ], + "source": [ + "sns.histplot(column_scores(x), stat = 'probability', color = 'r', bins = 20)" + ] + }, + { + "cell_type": "markdown", + "id": "5ce9840c-c457-4510-a932-0f2843157ca2", + "metadata": {}, + "source": [ + "We see these look nothing alike. Randomly running simulations does not accurately imitate what the testgrid data looks like, but rather exposes where the unexpectedness is too high to have been a result of a couple random failures." + ] + }, + { + "cell_type": "markdown", + "id": "e9cf953f-e2fb-4954-8e0b-7d7985b326d0", + "metadata": {}, + "source": [ + "We classify a couple levels of significance: 10%, 5%, and 1%. We see if the true scores are in the top x% of the simulated distribution." + ] + }, + { + "cell_type": "code", + "execution_count": 620, + "id": "ceaab32f-20c7-4072-97af-a5b48c6c27fe", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "0.05 significance: 36.11580390447193\n", + "0.01 significance: 39.80852213489304\n", + "0.001 significance: 47.53129776754298\n" + ] + } + ], + "source": [ + "out = sorted(out)\n", + "\n", + "n = len(out)\n", + "\n", + "idx1 = int(0.90*n)\n", + "idx2 = int(0.95*n)\n", + "idx3 = int(0.99*n)\n", + "\n", + "print(f'0.05 significance: {out[idx1]}')\n", + "print(f'0.01 significance: {out[idx2]}')\n", + "print(f'0.001 significance: {out[idx3]}')" + ] + }, + { + "cell_type": "markdown", + "id": "2d5bd073-891f-40e2-b0e6-79010dd94e6c", + "metadata": {}, + "source": [ + "We visualize these cutoffs below." + ] + }, + { + "cell_type": "code", + "execution_count": 621, + "id": "6540ff59-806a-4c40-8bbf-3e507e401be1", + "metadata": {}, + "outputs": [ + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAmQAAAE9CAYAAACleH4eAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjQuMywgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/MnkTPAAAACXBIWXMAAAsTAAALEwEAmpwYAAAV4UlEQVR4nO3dfawlZ30f8O8PryG8tLzUG9exTddJFiJTFUO3iJSodSBKgLzYUStjmoCFqBxVJryIqgLyB6SSKyoFaGgCrQMU04LBxU6xWovGMSQQtbyswQJsx8ECE9v1y1LAGEIB27/+cWfx9WZ371185zzn3vP5SFd35pk55/7OmTNnv/vMMzPV3QEAYJyHjS4AAGDVCWQAAIMJZAAAgwlkAACDCWQAAIMJZAAAg+0aXcBDccIJJ/SePXtGlwEAsKFrrrnmq929+3DLtnUg27NnT/bv3z+6DACADVXVV460zCFLAIDBBDIAgMEEMgCAwWYLZFV1alV9tKqur6rrquoVU/sbquq2qrp2+nn+use8tqpuqqobq+oX5qoNAGCZzDmo/94kr+7uz1TV30hyTVVdNS17S3f/zvqVq+r0JOcmeUqSH0vyx1X1pO6+b8YaAQCGm62HrLtv7+7PTNP3JLkhyclHechZSd7f3d/t7i8nuSnJM+aqDwBgWSxkDFlV7UnytCSfnJpeVlWfq6p3VdXjp7aTk9yy7mG35ugBDgBgR5g9kFXVY5JcluSV3f3NJG9P8hNJzkhye5I3HePznV9V+6tq/4EDB7a6XACAhZs1kFXV8VkLY+/t7suTpLvv7O77uvv+JH+QBw5L3pbk1HUPP2Vqe5Duvqi793X3vt27D3uxWwCAbWXOsywryTuT3NDdb17XftK61X41yRem6SuSnFtVj6iq05LsTfKpueoDAFgWc55l+awkL0ry+aq6dmp7XZIXVtUZSTrJzUl+I0m6+7qqujTJ9Vk7Q/MCZ1gCAKtgtkDW3X+WpA6z6MqjPObCJBfOVRMAbBd1yL+g3WPqYDFcqR8AYDCBDABgMIEMAGAwgQwAYDCBDABgMIEMAGAwgQwAYDCBDABgMIEMAGAwgQwAYDCBDABgMIEMAGAwgQwAYDCBDABgMIEMAGAwgQwAYDCBDABgMIEMAGAwgQwAYDCBDABgMIEMAGAwgQwAYDCBDABgMIEMAGAwgQwAYDCBDABgMIEMAGAwgQwAYDCBDABgMIEMAGAwgQwAYDCBDABgMIEMAGAwgQwAYDCBDABgMIEMAGAwgQwAYDCBDABgMIEMAGAwgQwAYDCBDABgMIEMAGAwgQwAYDCBDABgsNkCWVWdWlUfrarrq+q6qnrF1P6Eqrqqqr44/X781F5V9daquqmqPldVT5+rNgCAZTJnD9m9SV7d3acneWaSC6rq9CSvSXJ1d+9NcvU0nyTPS7J3+jk/ydtnrA0AYGnMFsi6+/bu/sw0fU+SG5KcnOSsJBdPq12c5Oxp+qwk7+k1n0jyuKo6aa76AACWxULGkFXVniRPS/LJJCd29+3TojuSnDhNn5zklnUPu3VqAwDY0WYPZFX1mCSXJXlld39z/bLu7iR9jM93flXtr6r9Bw4c2MJKAQDGmDWQVdXxWQtj7+3uy6fmOw8eipx+3zW135bk1HUPP2Vqe5Duvqi793X3vt27d89XPADAgsx5lmUleWeSG7r7zesWXZHkvGn6vCQfWtf+4ulsy2cmuXvdoU0AgB1r14zP/awkL0ry+aq6dmp7XZI3Jrm0ql6a5CtJzpmWXZnk+UluSvJXSV4yY20AAEtjtkDW3X+WpI6w+DmHWb+TXDBXPQAAy8qV+gEABhPIAAAGE8gAAAYTyAAABhPIAAAGE8gAAAYTyAAABhPIAAAGE8gAAAYTyAAABhPIAAAGE8gAAAYTyAAABhPIAAAGE8gAAAYTyAAABhPIAAAGE8gAAAYTyAAABhPIAAAGE8gAAAYTyAAABhPIAAAGE8gAAAYTyAAABts1ugAAYLyqB893j6ljVekhAwAYTCADABhMIAMAGEwgAwAYTCADABhMIAMAGEwgAwAYTCADABhMIAMAGEwgAwAYzK2TAIClduhtnZKdd2sngQw2wT3eAJiTQ5YAAIMJZAAAgwlkAACDCWQAAIMJZAAAg80WyKrqXVV1V1V9YV3bG6rqtqq6dvp5/rplr62qm6rqxqr6hbnqAgBYNnP2kL07yXMP0/6W7j5j+rkySarq9CTnJnnK9Ji3VdVxM9YGALA0Zgtk3f2xJF/b5OpnJXl/d3+3u7+c5KYkz5irNgAerOrBP8BijRhD9rKq+tx0SPPxU9vJSW5Zt86tUxsAwI636ED29iQ/keSMJLcnedOxPkFVnV9V+6tq/4EDB7a4PACAxVtoIOvuO7v7vu6+P8kf5IHDkrclOXXdqqdMbYd7jou6e19379u9e/e8BQMALMBCA1lVnbRu9leTHDwD84ok51bVI6rqtCR7k3xqkbUBAIwy283Fq+qSJGcmOaGqbk3y+iRnVtUZSTrJzUl+I0m6+7qqujTJ9UnuTXJBd983V20AAMtktkDW3S88TPM7j7L+hUkunKseAIBl5Ur9AACDCWQAAIMJZAAAgwlkAACDCWQAAINtOpBV1SOr6slzFgMAsIo2Fciq6peTXJvkw9P8GVV1xYx1AQCsjM32kL0ha7c5+kaSdPe1SU6bpSIAgBWz2UD2/e6++5C23upiAABW0Wav1H9dVf2zJMdV1d4kL0/yv+YrCwBgdWy2h+w3kzwlyXeTvC/J3UleOVNNAAArZcMesqo6Lsn/6O6fTfJb85cEALBaNuwh6+77ktxfVY9dQD0AACtns2PIvpXk81V1VZJvH2zs7pfPUhUAwArZbCC7fPoBAGCLbSqQdffFVfXwJE+amm7s7u/PVxYAwOrYVCCrqjOTXJzk5iSV5NSqOq+7PzZbZQAAK2KzhyzflOTnu/vGJKmqJyW5JMnfn6swAIBVsdnrkB1/MIwlSXf/RZLj5ykJAGC1bLaHbH9VvSPJf5nmfy3J/nlKAgBYLZsNZP8iyQVZu2VSknw8ydtmqQgAYMVsNpDtSvK73f3m5AdX73/EbFUBAKyQzY4huzrJI9fNPzLJH299OQAAq2ezgexHuvtbB2em6UfNUxIAwGrZbCD7dlU9/eBMVe1L8p15SgIAWC2bHUP2yiT/tar+zzR/UpIXzFIRAMCKOWoPWVX9g6r629396SQ/leQDSb6f5MNJvryA+gAAdryNDln+xyTfm6Z/Osnrkvx+kq8nuWjGugAAVsZGhyyP6+6vTdMvSHJRd1+W5LKqunbWygAAVsRGPWTHVdXB0PacJB9Zt2yz488AADiKjULVJUn+tKq+mrWzKj+eJFX1k0nunrk2AICVcNRA1t0XVtXVWTur8o+6u6dFD0vym3MXBwCwCjY87NjdnzhM21/MUw4AwOrZ7IVhAQCYiUAGADCYQAYAMJhABgAwmEAGADCYQAYAMJhABgAwmEAGADCYQAYAMJhABgAwmEAGADDYbIGsqt5VVXdV1RfWtT2hqq6qqi9Ovx8/tVdVvbWqbqqqz1XV0+eqCwBg2czZQ/buJM89pO01Sa7u7r1Jrp7mk+R5SfZOP+cnefuMdQEALJXZAll3fyzJ1w5pPivJxdP0xUnOXtf+nl7ziSSPq6qT5qoNAGCZLHoM2Yndffs0fUeSE6fpk5Pcsm69W6c2AIAdb9ig/u7uJH2sj6uq86tqf1XtP3DgwAyVAQAs1qID2Z0HD0VOv++a2m9Lcuq69U6Z2v6a7r6ou/d1977du3fPWiwAwCIsOpBdkeS8afq8JB9a1/7i6WzLZya5e92hTQCAHW3XXE9cVZckOTPJCVV1a5LXJ3ljkkur6qVJvpLknGn1K5M8P8lNSf4qyUvmqgsAYNnMFsi6+4VHWPScw6zbSS6YqxYAgGXmSv0AAIMJZAAAgwlkAACDCWQAAIMJZAAAgwlkAACDCWQAAIMJZAAAgwlkAACDCWQAAIMJZAAAgwlkAACDCWQAAIMJZAAAgwlkAACDCWQAAIPtGl0AAKun6sHz3WPqgGWhhwwAYDCBDABgMIEMAGAwgQwAYDCD+pnNoYN2EwN3AeBw9JABAAwmkAEADCaQAQAMJpABAAwmkAEADCaQAQAM5rIXABwTl7SBraeHDABgMIEMAGAwgQwAYDBjyAA24dBxU8ZMAVtJDxkAwGACGQDAYAIZAMBgAhkAwGACGQDAYAIZAMBgAhkAwGACGQDAYAIZAMBgrtQP28ShV4pPXC0eYKcYEsiq6uYk9yS5L8m93b2vqp6Q5ANJ9iS5Ock53f31EfUBACzSyEOWP9vdZ3T3vmn+NUmu7u69Sa6e5gEAdrxlGkN2VpKLp+mLk5w9rhQAgMUZFcg6yR9V1TVVdf7UdmJ33z5N35HkxDGlAQAs1qhB/T/T3bdV1Y8muaqq/nz9wu7uqjrscOUpwJ2fJE984hPnrxQAYGZDesi6+7bp911J/jDJM5LcWVUnJcn0+64jPPai7t7X3ft27969qJIBAGaz8B6yqnp0kod19z3T9M8n+ddJrkhyXpI3Tr8/tOjajupw1xzgqA7fxbnoKrbGX3stA17HTno/t6Nl+AzM6Vhe31Z8Fnf6+7kVFv0eLfM2Wdj338BrCY04ZHlikj+stYCzK8n7uvvDVfXpJJdW1UuTfCXJOQNqAwBYuIUHsu7+UpKnHqb9/yZ5zqLr2TRX4DxmO+lCpoe+lhGvYye9n9vRMnwG5nQsr28rPos7/f3cCot+j5Z5m6zC998yXfYCAGAlCWQAAIMJZAAAgwlkAACDCWQAAINVb+PTFPbt29f79+9fyN/6kz9ZoguyAABb7swz581EVXVNd+873DI9ZAAAg426l+W2M3dq3ol20nVjluH6PDvp/dyOluEzMCfXIVs+rkP2gFX4/tNDBgAwmDFkm1S/bQwZAOxk/XpjyAAAVpYxZJs0d2reiXbSMf9lGFuxk97P7WgZPgNz2iljyHbSfmIM2QN20nY9Ej1kAACDCWQAAIMJZAAAgwlkAACDCWQAAIMJZAAAgwlkAACDuQ4ZAEttFa5BBXrIAAAG00MGzEbPBsDm6CEDABhMIAMAGEwgAwAYTCADABhMIAMAGEwgAwAYTCADABhMIAMAGEwgAwAYTCADABhMIAMAGEwgAwAYTCADABhs1+gCtoOqB893L/fzbvbvLeJvbtactS36fYZVdbh9bZm/d2CZ6CEDABhMDxlssWXpkVuWOmCnWMV9Sg/n4ughAwAYTA8ZwA9J7wFsvYfaE7ld90uBDFhJ2/VLG9iZBDK2hLEVa1blf3IAbC2BDADY8Za940AggwXQE7axY32Plv3Lle1rJ+2v9pPtY+nOsqyq51bVjVV1U1W9ZnQ9x6rqwT/bwXaseaezTbbWTnk/D30dG72WnfK6YRUsVSCrquOS/H6S5yU5PckLq+r0sVUd3rF+McJO4XP/w/Pe8VD5t2fnWrZDls9IclN3fylJqur9Sc5Kcv3QqpaY7uittZMOVRzOkV6fz9EDtuNnYFG3d9vK596Kv3csr/tI6851iYWHUttW1jGXOV73su9nc1uqHrIkJye5Zd38rVMbAMCOtWw9ZBuqqvOTnD/Nfquqblzgnz8hyVeP1EV8uPZj6U7eiq7nY32Oh1rzsdax6PfjEEfdfsdSx1Z8BuZ8jrn+3kOt4SE+9wlJvjrD8/5Q7cvyGV+Gz8sm6/jB9luG/WRZ/t6RLLrmTT5+yL+Bc32eBx3u/TtHWrBsgey2JKeumz9lavuB7r4oyUWLLOqgqtrf3ftG/G0eOttve7P9tjfbb/uzDee1bIcsP51kb1WdVlUPT3JukisG1wQAMKul6iHr7nur6mVJ/meS45K8q7uvG1wWAMCsliqQJUl3X5nkytF1HMGQQ6VsGdtve7P9tjfbb/uzDWdUvernmQIADLZsY8gAAFaOQLYJ2/12Tqumqk6tqo9W1fVVdV1VvWJqf0JVXVVVX5x+P350rRxZVR1XVZ+tqv8+zZ9WVZ+c9sMPTCf+sKSq6nFV9cGq+vOquqGqfto+uH1U1aum788vVNUlVfUj9sF5CWQb2E63c+IH7k3y6u4+Pckzk1wwbbPXJLm6u/cmuXqaZ3m9IskN6+b/bZK3dPdPJvl6kpcOqYrN+t0kH+7un0ry1KxtS/vgNlBVJyd5eZJ93f13s3aS3bmxD85KINvYD27n1N3fS3Lwdk4sqe6+vbs/M03fk7V/CE7O2na7eFrt4iRnDymQDVXVKUl+Mck7pvlK8uwkH5xWsf2WWFU9Nsk/SvLOJOnu73X3N2If3E52JXlkVe1K8qgkt8c+OCuBbGNu57SNVdWeJE9L8skkJ3b37dOiO5KcOKouNvTvkvyrJPdP838ryTe6+95p3n643E5LciDJf5oOO7+jqh4d++C20N23JfmdJH+ZtSB2d5JrYh+clUDGjlVVj0lyWZJXdvc31y/rtdOLnWK8hKrql5Lc1d3XjK6FH9quJE9P8vbuflqSb+eQw5P2weU1je07K2vB+seSPDrJc4cWtQIEso1teDsnlk9VHZ+1MPbe7r58ar6zqk6alp+U5K5R9XFUz0ryK1V1c9aGCDw7a+ORHjcdPknsh8vu1iS3dvcnp/kPZi2g2Qe3h59L8uXuPtDd309yedb2S/vgjASyjbmd0zYzjTd6Z5IbuvvN6xZdkeS8afq8JB9adG1srLtf292ndPeerO1vH+nuX0vy0ST/dFrN9lti3X1Hkluq6slT03OSXB/74Hbxl0meWVWPmr5PD24/++CMXBh2E6rq+Vkb03Lwdk4Xjq2Io6mqn0ny8SSfzwNjkF6XtXFklyZ5YpKvJDmnu782pEg2parOTPIvu/uXqurHs9Zj9oQkn03y69393YHlcRRVdUbWTsp4eJIvJXlJ1joB7IPbQFX9dpIXZO2s9c8m+edZGzNmH5yJQAYAMJhDlgAAgwlkAACDCWQAAIMJZAAAgwlkAACD7dp4FYDtr6ruy9qlUI7P2qn878najZLvP+oDARZAIANWxXe6+4wkqaofTfK+JH8zyetHFgWQOGQJrKDuvivJ+UleVmv2VNXHq+oz088/TJKqek9VnX3wcVX13qo6q6qeUlWfqqprq+pzVbV30EsBdggXhgVWQlV9q7sfc0jbN5I8Ock9Se7v7v83hatLuntfVf3jJK/q7rOr6rFJrk2yN8lbknyiu9873VLtuO7+ziJfD7CzOGQJsDau7Pem2/3cl+RJSdLdf1pVb6uq3Un+SZLLuvveqvrfSX6rqk5Jcnl3f3FU4cDO4JAlsJKme2Pel+SuJK9KcmeSpybZl7X7Lx70niS/nrV7Mb4rSbr7fUl+Jcl3klxZVc9eXOXATqSHDFg5U4/Xf0jye93d0+HIW7v7/qo6L8lx61Z/d5JPJbmju6+fHv/jSb7U3W+tqicm+XtJPrLQFwHsKAIZsCoeWVXX5oHLXvznJG+elr0tyWVV9eIkH07y7YMP6u47q+qGJP9t3XOdk+RFVfX9JHck+TezVw/saAb1AxxFVT0qa9cve3p33z26HmBnMoYM4Aiq6ueS3JDk3wtjwJz0kAEADKaHDABgMIEMAGAwgQwAYDCBDABgMIEMAGAwgQwAYLD/D5X8av0UGH62AAAAAElFTkSuQmCC\n", + "text/plain": [ + "
" + ] + }, + "metadata": { + "needs_background": "light" + }, + "output_type": "display_data" + } + ], + "source": [ + "col_scores = column_scores(x)\n", + "\n", + "plt.figure(figsize=(10, 5))\n", + "plt.bar(x = x.columns, height = col_scores, color = 'b')\n", + "plt.ylabel(\"Score\")\n", + "plt.xlabel(\"Days\")\n", + "x1 = 0\n", + "x2 = len(col_scores)\n", + "plt.plot([x1, x2], [out[idx1],out[idx1]], color='g', linestyle='-', linewidth=2)\n", + "plt.plot([x1, x2], [out[idx2],out[idx2]], color='y', linestyle='-', linewidth=2)\n", + "plt.plot([x1, x2], [out[idx3],out[idx3]], color='r', linestyle='-', linewidth=2)\n", + "plt.show()" + ] + }, + { + "cell_type": "markdown", + "id": "d139e386-cd02-4e34-bb83-eb2453bca3c6", + "metadata": {}, + "source": [ + "In this case the scores were so anomolous that the significance levels didn't end up making a big difference." + ] + }, + { + "cell_type": "code", + "execution_count": 622, + "id": "2ae357f5-2bde-4cfc-935e-ecc5971e6509", + "metadata": {}, + "outputs": [], + "source": [ + "### tbd: main function & analysis." + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3 (ipykernel)", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.8.3" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +}