Update.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Generate Passport Index datasets\n",
    "* Data by Passport Index 2024: https://www.passportindex.org/\n",
    "* In both tidy and matrix formats\n",
    "* Using ISO-2, ISO-3, and full country names"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import httpx # see https://www.python-httpx.org/\n",
    "import pandas as pd\n",
    "import json\n",
    "import time\n",
    "\n",
    "# Load tqdm!\n",
    "from tqdm import tqdm\n",
    "tqdm.pandas()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CSRF token: 24c9edeac4b4917f69de44005b0a27ea59a06e0f5ae4eb1739a8f616ef429e9b\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|█████████████████████████████████████████| 199/199 [22:38<00:00,  6.83s/it]\n"
     ]
    }
   ],
   "source": [
    "headers = {\n",
    "    'Host': 'www.passportindex.org',\n",
    "    'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10.15; rv:126.0) Gecko/20100101 Firefox/126.0',\n",
    "    'Accept': '*/*',\n",
    "    'Accept-Language': 'en-US,en;q=0.5',\n",
    "    'Accept-Encoding': 'gzip, deflate, br, zstd',\n",
    "    'Content-Type': 'application/x-www-form-urlencoded; charset=UTF-8',\n",
    "    'X-Requested-With': 'XMLHttpRequest',\n",
    "    'Origin': 'https://www.passportindex.org',\n",
    "    'Referer': 'https://www.passportindex.org/comparebyPassport.php'\n",
    "}\n",
    "\n",
    "mounts = {\n",
    "    'https://': httpx.HTTPTransport(http2=True)\n",
    "}\n",
    "\n",
    "\n",
    "with httpx.Client() as client:\n",
    "\n",
    "    csrf_token = (\n",
    "        client.request(\n",
    "            url='https://www.passportindex.org/comparebyPassport.php',\n",
    "            method='get',\n",
    "            headers=headers\n",
    "        ).text\n",
    "        .split('let csrf_token = \"')[1]\n",
    "        .split('\";')[0]\n",
    "        .strip()\n",
    "    )\n",
    "\n",
    "    print(f'CSRF token: {csrf_token}')\n",
    "\n",
    "\n",
    "    country_data = (\n",
    "        pd.read_csv(\n",
    "            'https://gist.githubusercontent.com/ilyankou/b2580c632bdea4af2309dcaa69860013/raw/420fb417bcd17d833156efdf64ce8a1c3ceb2691/country-codes',\n",
    "            dtype=str\n",
    "        )\n",
    "        .fillna('NA')\n",
    "    )\n",
    "\n",
    "    \n",
    "    def get_data(country_code):\n",
    "\n",
    "        time.sleep(6.1)\n",
    "        \n",
    "        res = client.request(\n",
    "            url='https://www.passportindex.org/incl/compare2.php',\n",
    "            method='post',\n",
    "            headers=headers,\n",
    "            data={\n",
    "                'compare': '3',\n",
    "                'cc': str.lower(country_code),\n",
    "                'year': '2024',\n",
    "                'csrf_token': csrf_token\n",
    "            }\n",
    "        ).json()\n",
    "        return res\n",
    "\n",
    "    \n",
    "    #print(get_data('by'))\n",
    "    country_data['travel'] = country_data.ISO2.progress_apply(get_data)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Get data from PassportIndex"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "assert country_data['travel'].isna().sum() == 0, \\\n",
    "    f\"Didn't fetch {country_data['travel'].isna().sum()} rows!\""
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Clean up the data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "obj = {}\n",
    "\n",
    "for passport in country_data.ISO2.tolist():\n",
    "    \n",
    "    # Add passport to the object\n",
    "    if passport not in obj:\n",
    "        obj[passport] = {}\n",
    "    \n",
    "    # Add destinations for the given passport\n",
    "    for dest in country_data.query(f'ISO2 == \"{passport}\"').iloc[0]['travel']:\n",
    "        \n",
    "        text = dest['text'].lower()\n",
    "        res = ''\n",
    "        \n",
    "        # ** Visa required, incl Cuba's tourist card **\n",
    "        if text == 'visa required' or text == 'tourist card':\n",
    "            res = 'visa required'\n",
    "        \n",
    "        # ** Visa on arrival **\n",
    "        elif 'visa on arrival' in text:\n",
    "            res = 'visa on arrival'\n",
    "            \n",
    "        # ** Covid-19 ban ** \n",
    "        elif text == 'covid-19 ban':\n",
    "            res = 'covid ban'\n",
    "            \n",
    "        # ** Visa-free, incl. Seychelles' tourist registration **\n",
    "        elif 'visa-free' in text or 'tourist registration' in text or 'visa waiver' in text:\n",
    "            res = dest['dur'] if dest['dur'] != '' else 'visa free'\n",
    "            \n",
    "        # ** eVisas, incl eVisitors (Australia), eTourist cards (Suriname),\n",
    "        # eTA (US), and pre-enrollment (Ivory Coast), or EVW (UK) **\n",
    "        elif 'evis' in text or 'etourist' in text or text == 'eta' or text == 'pre-enrollment' or text == 'evw':\n",
    "            res = 'e-visa'\n",
    "            \n",
    "        # ** No admission, including Trump ban **\n",
    "        elif text == 'trump ban' or text == 'not admitted':\n",
    "            res = 'no admission'\n",
    "\n",
    "        elif 'arrival card' in text or 'e-ticket' in text:\n",
    "            res = 'visa free'\n",
    "        \n",
    "        # Update the result!\n",
    "        obj[passport][ dest['code'] ] = res if res != '' else dest['text']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0 AFG\n",
      "1 ALB\n",
      "2 DZA\n",
      "3 AND\n",
      "4 AGO\n"
     ]
    }
   ],
   "source": [
    "for x, y in country_data.head(5).iterrows():\n",
    "    print(x, y['ISO3'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [],
   "source": [
    "#iso2to3\n",
    "#matrix"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Save"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [],
   "source": [
    "# ISO-2: Matrix\n",
    "matrix = pd.DataFrame(obj).T.fillna(-1)\n",
    "matrix.to_csv('passport-index-matrix-iso2.csv', index_label='Passport')\n",
    "\n",
    "# ISO-2: Tidy\n",
    "matrix.stack().to_csv(\n",
    "    'passport-index-tidy-iso2.csv',\n",
    "    index_label=['Passport', 'Destination'],\n",
    "    header=['Requirement'])\n",
    "\n",
    "# ISO-3: Matrix\n",
    "iso2to3 =  { y['ISO2']:y['ISO3'] for _,y in country_data.iterrows() }\n",
    "matrix.rename(columns=iso2to3, index=iso2to3).to_csv('passport-index-matrix-iso3.csv', index_label='Passport')\n",
    "\n",
    "# ISO-3: Tidy\n",
    "matrix.rename(columns=iso2to3, index=iso2to3).stack().to_csv(\n",
    "    'passport-index-tidy-iso3.csv',\n",
    "    index_label=['Passport', 'Destination'],\n",
    "    header=['Requirement'])\n",
    "\n",
    "\n",
    "# Country names: Matrix\n",
    "iso2name =  { y['ISO2']:y['Country'] for _,y in country_data.iterrows() }\n",
    "matrix.rename(columns=iso2name, index=iso2name).to_csv('passport-index-matrix.csv', index_label='Passport')\n",
    "\n",
    "# Country names: Tidy\n",
    "matrix.rename(columns=iso2name, index=iso2name).stack().to_csv(\n",
    "    'passport-index-tidy.csv',\n",
    "    index_label=['Passport', 'Destination'],\n",
    "    header=['Requirement'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "visa required      13429\n",
       "90                  7624\n",
       "e-visa              7367\n",
       "visa on arrival     5711\n",
       "visa free           1878\n",
       "30                  1658\n",
       "180                  734\n",
       "120                  233\n",
       "-1                   199\n",
       "60                   192\n",
       "15                   109\n",
       "21                   106\n",
       "14                   105\n",
       "360                   97\n",
       "42                    60\n",
       "no admission          34\n",
       "28                    23\n",
       "240                   15\n",
       "45                    15\n",
       "10                     6\n",
       "7                      4\n",
       "150                    1\n",
       "31                     1\n",
       "Name: count, dtype: int64"
      ]
     },
     "execution_count": 23,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Print all values\n",
    "tidy = matrix.rename(columns=iso2to3, index=iso2to3).stack()\n",
    "tidy.value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "ARM  AZE    no admission\n",
       "BGD  IRQ    no admission\n",
       "     LBY    no admission\n",
       "HTI  SUR    no admission\n",
       "IRN  LBY    no admission\n",
       "ISR  DZA    no admission\n",
       "     BGD    no admission\n",
       "     BRN    no admission\n",
       "     IRN    no admission\n",
       "     LBN    no admission\n",
       "     LBY    no admission\n",
       "     MYS    no admission\n",
       "     PAK    no admission\n",
       "     SAU    no admission\n",
       "     SYR    no admission\n",
       "     YEM    no admission\n",
       "XKX  ARM    no admission\n",
       "     KHM    no admission\n",
       "     CUB    no admission\n",
       "     HKG    no admission\n",
       "     SYC    no admission\n",
       "PRK  JPN    no admission\n",
       "     LKA    no admission\n",
       "PAK  LBY    no admission\n",
       "PSE  MDG    no admission\n",
       "     SYR    no admission\n",
       "PHL  KWT    no admission\n",
       "SOM  AUS    no admission\n",
       "     CAN    no admission\n",
       "SDN  LBY    no admission\n",
       "SYR  LBY    no admission\n",
       "TWN  GEO    no admission\n",
       "TJK  KGZ    no admission\n",
       "YEM  LBY    no admission\n",
       "dtype: object"
      ]
     },
     "execution_count": 24,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Which countries have no admission? (Typically war zones etc)\n",
    "tidy[ tidy == 'no admission' ]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [],
   "source": [
    "assert tidy.loc[('BLR', 'RUS')] == '90', \"Check data!\""
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Difference with previous run\n",
    "* Typically the difference between two consecutive months would be in low hundreds.\n",
    "* If difference is higher, double-check the data!"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Passport</th>\n",
       "      <th>Destination</th>\n",
       "      <th>Requirement_previous</th>\n",
       "      <th>Requirement_new</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>79</th>\n",
       "      <td>AFG</td>\n",
       "      <td>IRQ</td>\n",
       "      <td>e-visa</td>\n",
       "      <td>visa required</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>278</th>\n",
       "      <td>ALB</td>\n",
       "      <td>IRQ</td>\n",
       "      <td>e-visa</td>\n",
       "      <td>visa required</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>370</th>\n",
       "      <td>ALB</td>\n",
       "      <td>SYR</td>\n",
       "      <td>e-visa</td>\n",
       "      <td>visa required</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>374</th>\n",
       "      <td>ALB</td>\n",
       "      <td>THA</td>\n",
       "      <td>60</td>\n",
       "      <td>e-visa</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>477</th>\n",
       "      <td>DZA</td>\n",
       "      <td>IRQ</td>\n",
       "      <td>e-visa</td>\n",
       "      <td>visa required</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>39374</th>\n",
       "      <td>ZMB</td>\n",
       "      <td>SYR</td>\n",
       "      <td>e-visa</td>\n",
       "      <td>visa required</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>39394</th>\n",
       "      <td>ZMB</td>\n",
       "      <td>VUT</td>\n",
       "      <td>120</td>\n",
       "      <td>30</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>39481</th>\n",
       "      <td>ZWE</td>\n",
       "      <td>IRQ</td>\n",
       "      <td>e-visa</td>\n",
       "      <td>visa required</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>39573</th>\n",
       "      <td>ZWE</td>\n",
       "      <td>SYR</td>\n",
       "      <td>e-visa</td>\n",
       "      <td>visa required</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>39593</th>\n",
       "      <td>ZWE</td>\n",
       "      <td>VUT</td>\n",
       "      <td>120</td>\n",
       "      <td>30</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>622 rows × 4 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "      Passport Destination Requirement_previous Requirement_new\n",
       "79         AFG         IRQ               e-visa   visa required\n",
       "278        ALB         IRQ               e-visa   visa required\n",
       "370        ALB         SYR               e-visa   visa required\n",
       "374        ALB         THA                   60          e-visa\n",
       "477        DZA         IRQ               e-visa   visa required\n",
       "...        ...         ...                  ...             ...\n",
       "39374      ZMB         SYR               e-visa   visa required\n",
       "39394      ZMB         VUT                  120              30\n",
       "39481      ZWE         IRQ               e-visa   visa required\n",
       "39573      ZWE         SYR               e-visa   visa required\n",
       "39593      ZWE         VUT                  120              30\n",
       "\n",
       "[622 rows x 4 columns]"
      ]
     },
     "execution_count": 26,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "x = (tidy\n",
    " .reset_index()\n",
    " .rename(\n",
    "     columns={'level_0': 'Passport', 'level_1': 'Destination', 0: 'Requirement'}\n",
    " )\n",
    " .merge(\n",
    "     pd.read_csv('legacy/2024-07-02/passport-index-tidy-iso3.csv'), # SELECT VERSION TO COMPARE!\n",
    "     how='left',\n",
    "     left_on=['Passport', 'Destination'],\n",
    "     right_on=['Passport', 'Destination'],\n",
    "     suffixes=('_previous', '_new')\n",
    " )\n",
    " .assign(\n",
    "     is_different=lambda df_: df_.Requirement_previous.ne(df_.Requirement_new)\n",
    " )\n",
    " .query('is_different & (Passport != Destination)')\n",
    " .drop(columns=['is_different'])\n",
    ")\n",
    "\n",
    "x"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Passport</th>\n",
       "      <th>Destination</th>\n",
       "      <th>Requirement_previous</th>\n",
       "      <th>Requirement_new</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>37516</th>\n",
       "      <td>GBR</td>\n",
       "      <td>MYS</td>\n",
       "      <td>visa free</td>\n",
       "      <td>90</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>37568</th>\n",
       "      <td>GBR</td>\n",
       "      <td>SGP</td>\n",
       "      <td>visa free</td>\n",
       "      <td>90</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>37583</th>\n",
       "      <td>GBR</td>\n",
       "      <td>SYR</td>\n",
       "      <td>e-visa</td>\n",
       "      <td>visa required</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>37587</th>\n",
       "      <td>GBR</td>\n",
       "      <td>THA</td>\n",
       "      <td>60</td>\n",
       "      <td>30</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>37603</th>\n",
       "      <td>GBR</td>\n",
       "      <td>VUT</td>\n",
       "      <td>120</td>\n",
       "      <td>30</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "      Passport Destination Requirement_previous Requirement_new\n",
       "37516      GBR         MYS            visa free              90\n",
       "37568      GBR         SGP            visa free              90\n",
       "37583      GBR         SYR               e-visa   visa required\n",
       "37587      GBR         THA                   60              30\n",
       "37603      GBR         VUT                  120              30"
      ]
     },
     "execution_count": 28,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# What changed for the UK?\n",
    "x.query('Passport == \"GBR\"')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Passport</th>\n",
       "      <th>Destination</th>\n",
       "      <th>Requirement_previous</th>\n",
       "      <th>Requirement_new</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "Empty DataFrame\n",
       "Columns: [Passport, Destination, Requirement_previous, Requirement_new]\n",
       "Index: []"
      ]
     },
     "execution_count": 29,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "x.query('Destination == \"GBR\"')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.11.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}