pdc_to_crdch_transformation.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# PDC to CRDCH Transformation Workflow\n",
    "\n",
    "This notebook demonstrates the conversion of data from one of the CRDC nodes, specifically the PDC, into CRDCH instance data. The notebook reads in the node data as JSON and outputs it in the LinkML format."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import sys\n",
    "\n",
    "# Installing these packages is only necessary if you don't start this Notebook by running\n",
    "# `poetry run jupyter notebook`.\n",
    "\n",
    "# LinkML is a modeling language with built in generators\n",
    "# that can automagically generate data in many output formats\n",
    "# like JSON-LD, Python dataclasses modules, etc.\n",
    "#!{sys.executable} -m pip install linkml\n",
    "\n",
    "# Dataframes library to visualize node data in a tabular format\n",
    "#!{sys.executable} -m pip install pandas\n",
    "\n",
    "# Utilities to visualize data in LinkML YAML format\n",
    "#!{sys.executable} -m pip install linkml-runtime\n",
    "\n",
    "# Python dataclasses to load, validate and transform\n",
    "# CRDCH instance data\n",
    "#!{sys.executable} -m pip install crdch-model"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Load and Visualize PDC data"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "For the purposes of this demonstration, we have aggregated a dataset of 560 cases from the PDC by querying the [PDC public API](https://pdc.cancer.gov/data-dictionary/publicapi-documentation/#!/Case/allCases). The querying and download protocol has been documented in a different [Jupyter notebook](https://github.com/cancerDHC/example-data/blob/main/head-and-mouth/Head%20and%20Mouth%20Cancer%20Datasets.ipynb)."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>case_id</th>\n",
       "      <th>case_submitter_id</th>\n",
       "      <th>days_to_lost_to_followup</th>\n",
       "      <th>demographics</th>\n",
       "      <th>diagnoses</th>\n",
       "      <th>disease_type</th>\n",
       "      <th>externalReferences</th>\n",
       "      <th>index_date</th>\n",
       "      <th>lost_to_followup</th>\n",
       "      <th>primary_site</th>\n",
       "      <th>project_submitter_id</th>\n",
       "      <th>samples</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0232701d-6d00-440c-af6c-5899fbbf4142</td>\n",
       "      <td>OSCC_13</td>\n",
       "      <td>None</td>\n",
       "      <td>[{'cause_of_death': None, 'days_to_birth': Non...</td>\n",
       "      <td>[{'age_at_diagnosis': None, 'ajcc_clinical_m':...</td>\n",
       "      <td>Oral Squamous Cell Carcinoma</td>\n",
       "      <td>[]</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>Head and Neck</td>\n",
       "      <td>Oral Squamous Cell Carcinoma - Chang Gung Univ...</td>\n",
       "      <td>[{'aliquots': [{'aliquot_id': '426a2696-f073-4...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0e943de7-c277-48f2-8fa9-b2e836b03c2c</td>\n",
       "      <td>OSCC_25</td>\n",
       "      <td>None</td>\n",
       "      <td>[{'cause_of_death': None, 'days_to_birth': Non...</td>\n",
       "      <td>[{'age_at_diagnosis': None, 'ajcc_clinical_m':...</td>\n",
       "      <td>Oral Squamous Cell Carcinoma</td>\n",
       "      <td>[]</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>Head and Neck</td>\n",
       "      <td>Oral Squamous Cell Carcinoma - Chang Gung Univ...</td>\n",
       "      <td>[{'aliquots': [{'aliquot_id': '38404eb4-20a6-4...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>1104505a-9890-49ce-8d7d-7a8070261324</td>\n",
       "      <td>OSCC_23</td>\n",
       "      <td>None</td>\n",
       "      <td>[{'cause_of_death': None, 'days_to_birth': Non...</td>\n",
       "      <td>[{'age_at_diagnosis': None, 'ajcc_clinical_m':...</td>\n",
       "      <td>Oral Squamous Cell Carcinoma</td>\n",
       "      <td>[]</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>Head and Neck</td>\n",
       "      <td>Oral Squamous Cell Carcinoma - Chang Gung Univ...</td>\n",
       "      <td>[{'aliquots': [{'aliquot_id': '15218d5b-fc40-4...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>195cd133-0d53-402d-b31c-3d4fe0481858</td>\n",
       "      <td>OSCC_37</td>\n",
       "      <td>None</td>\n",
       "      <td>[{'cause_of_death': None, 'days_to_birth': Non...</td>\n",
       "      <td>[{'age_at_diagnosis': None, 'ajcc_clinical_m':...</td>\n",
       "      <td>Oral Squamous Cell Carcinoma</td>\n",
       "      <td>[]</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>Head and Neck</td>\n",
       "      <td>Oral Squamous Cell Carcinoma - Chang Gung Univ...</td>\n",
       "      <td>[{'aliquots': [{'aliquot_id': '47e8d70c-646d-4...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>1df726a4-8520-4474-8c00-d238a7384be1</td>\n",
       "      <td>OSCC_06</td>\n",
       "      <td>None</td>\n",
       "      <td>[{'cause_of_death': None, 'days_to_birth': Non...</td>\n",
       "      <td>[{'age_at_diagnosis': None, 'ajcc_clinical_m':...</td>\n",
       "      <td>Oral Squamous Cell Carcinoma</td>\n",
       "      <td>[]</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>Head and Neck</td>\n",
       "      <td>Oral Squamous Cell Carcinoma - Chang Gung Univ...</td>\n",
       "      <td>[{'aliquots': [{'aliquot_id': '333e24c9-ec45-4...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>143</th>\n",
       "      <td>df6bef95-c233-4b10-b321-36ef4e79b5d4</td>\n",
       "      <td>OSCC_40</td>\n",
       "      <td>None</td>\n",
       "      <td>[{'cause_of_death': None, 'days_to_birth': Non...</td>\n",
       "      <td>[{'age_at_diagnosis': None, 'ajcc_clinical_m':...</td>\n",
       "      <td>Oral Squamous Cell Carcinoma</td>\n",
       "      <td>[]</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>Head and Neck</td>\n",
       "      <td>Oral Squamous Cell Carcinoma - Chang Gung Univ...</td>\n",
       "      <td>[{'aliquots': [{'aliquot_id': 'a3402806-a9ec-4...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>144</th>\n",
       "      <td>e11e9155-4ac6-43dc-b8e5-1be822cd2dab</td>\n",
       "      <td>OSCC_47</td>\n",
       "      <td>None</td>\n",
       "      <td>[{'cause_of_death': None, 'days_to_birth': Non...</td>\n",
       "      <td>[{'age_at_diagnosis': None, 'ajcc_clinical_m':...</td>\n",
       "      <td>Oral Squamous Cell Carcinoma</td>\n",
       "      <td>[]</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>Head and Neck</td>\n",
       "      <td>Oral Squamous Cell Carcinoma - Chang Gung Univ...</td>\n",
       "      <td>[{'aliquots': [{'aliquot_id': '9d1789f8-d629-4...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>145</th>\n",
       "      <td>ea7c9fbd-8353-4f3c-9fea-2fba79140536</td>\n",
       "      <td>OSCC_56</td>\n",
       "      <td>None</td>\n",
       "      <td>[{'cause_of_death': None, 'days_to_birth': Non...</td>\n",
       "      <td>[{'age_at_diagnosis': None, 'ajcc_clinical_m':...</td>\n",
       "      <td>Oral Squamous Cell Carcinoma</td>\n",
       "      <td>[]</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>Head and Neck</td>\n",
       "      <td>Oral Squamous Cell Carcinoma - Chang Gung Univ...</td>\n",
       "      <td>[{'aliquots': [{'aliquot_id': '9c36e4e9-a971-4...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>146</th>\n",
       "      <td>f581075d-1b69-4812-9fe4-2bde4aad8bf2</td>\n",
       "      <td>OSCC_38</td>\n",
       "      <td>None</td>\n",
       "      <td>[{'cause_of_death': None, 'days_to_birth': Non...</td>\n",
       "      <td>[{'age_at_diagnosis': None, 'ajcc_clinical_m':...</td>\n",
       "      <td>Oral Squamous Cell Carcinoma</td>\n",
       "      <td>[]</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>Head and Neck</td>\n",
       "      <td>Oral Squamous Cell Carcinoma - Chang Gung Univ...</td>\n",
       "      <td>[{'aliquots': [{'aliquot_id': '4059003c-b576-4...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>147</th>\n",
       "      <td>ff5a631f-93b9-486e-beb9-d16737db8c9e</td>\n",
       "      <td>OSCC_58</td>\n",
       "      <td>None</td>\n",
       "      <td>[{'cause_of_death': None, 'days_to_birth': Non...</td>\n",
       "      <td>[{'age_at_diagnosis': None, 'ajcc_clinical_m':...</td>\n",
       "      <td>Oral Squamous Cell Carcinoma</td>\n",
       "      <td>[]</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>Head and Neck</td>\n",
       "      <td>Oral Squamous Cell Carcinoma - Chang Gung Univ...</td>\n",
       "      <td>[{'aliquots': [{'aliquot_id': '030c9e59-35a2-4...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>148 rows × 12 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                  case_id case_submitter_id  \\\n",
       "0    0232701d-6d00-440c-af6c-5899fbbf4142           OSCC_13   \n",
       "1    0e943de7-c277-48f2-8fa9-b2e836b03c2c           OSCC_25   \n",
       "2    1104505a-9890-49ce-8d7d-7a8070261324           OSCC_23   \n",
       "3    195cd133-0d53-402d-b31c-3d4fe0481858           OSCC_37   \n",
       "4    1df726a4-8520-4474-8c00-d238a7384be1           OSCC_06   \n",
       "..                                    ...               ...   \n",
       "143  df6bef95-c233-4b10-b321-36ef4e79b5d4           OSCC_40   \n",
       "144  e11e9155-4ac6-43dc-b8e5-1be822cd2dab           OSCC_47   \n",
       "145  ea7c9fbd-8353-4f3c-9fea-2fba79140536           OSCC_56   \n",
       "146  f581075d-1b69-4812-9fe4-2bde4aad8bf2           OSCC_38   \n",
       "147  ff5a631f-93b9-486e-beb9-d16737db8c9e           OSCC_58   \n",
       "\n",
       "    days_to_lost_to_followup  \\\n",
       "0                       None   \n",
       "1                       None   \n",
       "2                       None   \n",
       "3                       None   \n",
       "4                       None   \n",
       "..                       ...   \n",
       "143                     None   \n",
       "144                     None   \n",
       "145                     None   \n",
       "146                     None   \n",
       "147                     None   \n",
       "\n",
       "                                          demographics  \\\n",
       "0    [{'cause_of_death': None, 'days_to_birth': Non...   \n",
       "1    [{'cause_of_death': None, 'days_to_birth': Non...   \n",
       "2    [{'cause_of_death': None, 'days_to_birth': Non...   \n",
       "3    [{'cause_of_death': None, 'days_to_birth': Non...   \n",
       "4    [{'cause_of_death': None, 'days_to_birth': Non...   \n",
       "..                                                 ...   \n",
       "143  [{'cause_of_death': None, 'days_to_birth': Non...   \n",
       "144  [{'cause_of_death': None, 'days_to_birth': Non...   \n",
       "145  [{'cause_of_death': None, 'days_to_birth': Non...   \n",
       "146  [{'cause_of_death': None, 'days_to_birth': Non...   \n",
       "147  [{'cause_of_death': None, 'days_to_birth': Non...   \n",
       "\n",
       "                                             diagnoses  \\\n",
       "0    [{'age_at_diagnosis': None, 'ajcc_clinical_m':...   \n",
       "1    [{'age_at_diagnosis': None, 'ajcc_clinical_m':...   \n",
       "2    [{'age_at_diagnosis': None, 'ajcc_clinical_m':...   \n",
       "3    [{'age_at_diagnosis': None, 'ajcc_clinical_m':...   \n",
       "4    [{'age_at_diagnosis': None, 'ajcc_clinical_m':...   \n",
       "..                                                 ...   \n",
       "143  [{'age_at_diagnosis': None, 'ajcc_clinical_m':...   \n",
       "144  [{'age_at_diagnosis': None, 'ajcc_clinical_m':...   \n",
       "145  [{'age_at_diagnosis': None, 'ajcc_clinical_m':...   \n",
       "146  [{'age_at_diagnosis': None, 'ajcc_clinical_m':...   \n",
       "147  [{'age_at_diagnosis': None, 'ajcc_clinical_m':...   \n",
       "\n",
       "                     disease_type externalReferences index_date  \\\n",
       "0    Oral Squamous Cell Carcinoma                 []       None   \n",
       "1    Oral Squamous Cell Carcinoma                 []       None   \n",
       "2    Oral Squamous Cell Carcinoma                 []       None   \n",
       "3    Oral Squamous Cell Carcinoma                 []       None   \n",
       "4    Oral Squamous Cell Carcinoma                 []       None   \n",
       "..                            ...                ...        ...   \n",
       "143  Oral Squamous Cell Carcinoma                 []       None   \n",
       "144  Oral Squamous Cell Carcinoma                 []       None   \n",
       "145  Oral Squamous Cell Carcinoma                 []       None   \n",
       "146  Oral Squamous Cell Carcinoma                 []       None   \n",
       "147  Oral Squamous Cell Carcinoma                 []       None   \n",
       "\n",
       "    lost_to_followup   primary_site  \\\n",
       "0               None  Head and Neck   \n",
       "1               None  Head and Neck   \n",
       "2               None  Head and Neck   \n",
       "3               None  Head and Neck   \n",
       "4               None  Head and Neck   \n",
       "..               ...            ...   \n",
       "143             None  Head and Neck   \n",
       "144             None  Head and Neck   \n",
       "145             None  Head and Neck   \n",
       "146             None  Head and Neck   \n",
       "147             None  Head and Neck   \n",
       "\n",
       "                                  project_submitter_id  \\\n",
       "0    Oral Squamous Cell Carcinoma - Chang Gung Univ...   \n",
       "1    Oral Squamous Cell Carcinoma - Chang Gung Univ...   \n",
       "2    Oral Squamous Cell Carcinoma - Chang Gung Univ...   \n",
       "3    Oral Squamous Cell Carcinoma - Chang Gung Univ...   \n",
       "4    Oral Squamous Cell Carcinoma - Chang Gung Univ...   \n",
       "..                                                 ...   \n",
       "143  Oral Squamous Cell Carcinoma - Chang Gung Univ...   \n",
       "144  Oral Squamous Cell Carcinoma - Chang Gung Univ...   \n",
       "145  Oral Squamous Cell Carcinoma - Chang Gung Univ...   \n",
       "146  Oral Squamous Cell Carcinoma - Chang Gung Univ...   \n",
       "147  Oral Squamous Cell Carcinoma - Chang Gung Univ...   \n",
       "\n",
       "                                               samples  \n",
       "0    [{'aliquots': [{'aliquot_id': '426a2696-f073-4...  \n",
       "1    [{'aliquots': [{'aliquot_id': '38404eb4-20a6-4...  \n",
       "2    [{'aliquots': [{'aliquot_id': '15218d5b-fc40-4...  \n",
       "3    [{'aliquots': [{'aliquot_id': '47e8d70c-646d-4...  \n",
       "4    [{'aliquots': [{'aliquot_id': '333e24c9-ec45-4...  \n",
       "..                                                 ...  \n",
       "143  [{'aliquots': [{'aliquot_id': 'a3402806-a9ec-4...  \n",
       "144  [{'aliquots': [{'aliquot_id': '9d1789f8-d629-4...  \n",
       "145  [{'aliquots': [{'aliquot_id': '9c36e4e9-a971-4...  \n",
       "146  [{'aliquots': [{'aliquot_id': '4059003c-b576-4...  \n",
       "147  [{'aliquots': [{'aliquot_id': '030c9e59-35a2-4...  \n",
       "\n",
       "[148 rows x 12 columns]"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import json\n",
    "import pandas\n",
    "\n",
    "with open(\"head-and-mouth/pdc-head-and-mouth.json\") as file:\n",
    "    pdc_head_and_mouth = json.load(file)\n",
    "\n",
    "pandas.DataFrame(pdc_head_and_mouth)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Transform PDC Case into CRDCH Research Subject\n",
    "\n",
    "The transformation being demonstrated here is the transformation of a PDC case into a CCDH Research Subject. For the purposes of this demo, we will transform two components, specifically, the diagnosis information from a PDC case into a CRDCH Diagnosis. And also the top level specimen component into CCDH Specimens.\n",
    "\n",
    "TODO: Similar to specimen, other top level components like samples, aliquots, analytes, slides can also be transformed."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "CancerStageObservationSet(id=None, category=None, focus=[], subject=None, method_type=[], performed_by=None, observations=[CancerStageObservation(observation_type=CodeableConcept(coding=[Coding(code='Overall', system='https://example.org/CancerStageObservation', label=None, system_version=None, value_set=None, value_set_version=None, tag=[])], text=None), value_codeable_concept=CodeableConcept(coding=[Coding(code='Stage IV', system='https://example.org/CancerStageObservation', label=None, system_version=None, value_set=None, value_set_version=None, tag=[])], text=None), id=None, category=None, method_type=None, focus=None, subject=None, performed_by=None), CancerStageObservation(observation_type=CodeableConcept(coding=[Coding(code='Clinical Metastasis (M)', system='https://example.org/CancerStageObservation', label=None, system_version=None, value_set=None, value_set_version=None, tag=[])], text=None), value_codeable_concept=CodeableConcept(coding=[Coding(code='M0', system='https://example.org/CancerStageObservation', label=None, system_version=None, value_set=None, value_set_version=None, tag=[])], text=None), id=None, category=None, method_type=None, focus=None, subject=None, performed_by=None), CancerStageObservation(observation_type=CodeableConcept(coding=[Coding(code='Pathological Overall', system='https://example.org/CancerStageObservation', label=None, system_version=None, value_set=None, value_set_version=None, tag=[])], text=None), value_codeable_concept=CodeableConcept(coding=[Coding(code='Stage IVA', system='https://example.org/CancerStageObservation', label=None, system_version=None, value_set=None, value_set_version=None, tag=[])], text=None), id=None, category=None, method_type=None, focus=None, subject=None, performed_by=None), CancerStageObservation(observation_type=CodeableConcept(coding=[Coding(code='Pathological Tumor (T)', system='https://example.org/CancerStageObservation', label=None, system_version=None, value_set=None, value_set_version=None, tag=[])], text=None), value_codeable_concept=CodeableConcept(coding=[Coding(code='T4a', system='https://example.org/CancerStageObservation', label=None, system_version=None, value_set=None, value_set_version=None, tag=[])], text=None), id=None, category=None, method_type=None, focus=None, subject=None, performed_by=None), CancerStageObservation(observation_type=CodeableConcept(coding=[Coding(code='Pathological Node (N)', system='https://example.org/CancerStageObservation', label=None, system_version=None, value_set=None, value_set_version=None, tag=[])], text=None), value_codeable_concept=CodeableConcept(coding=[Coding(code='N0', system='https://example.org/CancerStageObservation', label=None, system_version=None, value_set=None, value_set_version=None, tag=[])], text=None), id=None, category=None, method_type=None, focus=None, subject=None, performed_by=None), CancerStageObservation(observation_type=CodeableConcept(coding=[Coding(code='Pathological Metastasis (M)', system='https://example.org/CancerStageObservation', label=None, system_version=None, value_set=None, value_set_version=None, tag=[])], text=None), value_codeable_concept=CodeableConcept(coding=[Coding(code='Unknown', system='https://example.org/CancerStageObservation', label=None, system_version=None, value_set=None, value_set_version=None, tag=[])], text=None), id=None, category=None, method_type=None, focus=None, subject=None, performed_by=None)])"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import crdch_model\n",
    "import utils\n",
    "\n",
    "\n",
    "def create_stage_observation(type, value):\n",
    "    \"\"\"Create a CCDHCancerStageObservation from a type of observation and a codeable concept.\"\"\"\n",
    "    # As with the body site example above, we need to map PDC values into the values\n",
    "    # allowed under the CRDCH model.\n",
    "    stage_mappings = {\n",
    "        \"not reported\": \"Not Reported\",\n",
    "        \"unknown\": \"Unknown\",\n",
    "        \"stage i\": \"Stage I\",\n",
    "        \"stage ii\": \"Stage II\",\n",
    "        \"stage iii\": \"Stage III\",\n",
    "        \"stage iva\": \"Stage IVA\",\n",
    "        \"stage ivb\": \"Stage IVB\",\n",
    "        \"stage ivc\": \"Stage IVC\",\n",
    "    }\n",
    "\n",
    "    # Check if any of the stages are named with lower case Roman numerals\n",
    "    # rather than upper case.\n",
    "    if value in stage_mappings:\n",
    "        return crdch_model.CancerStageObservation(\n",
    "            observation_type=utils.codeable_concept(\n",
    "                code=type, system=\"https://example.org/CancerStageObservation\"\n",
    "            ),\n",
    "            value_codeable_concept=utils.codeable_concept(\n",
    "                code=stage_mappings[value],\n",
    "                system=\"https://example.org/CancerStageObservation\",\n",
    "            ),\n",
    "        )\n",
    "\n",
    "    return crdch_model.CancerStageObservation(\n",
    "        observation_type=utils.codeable_concept(\n",
    "            code=type, system=\"https://example.org/CancerStageObservation\"\n",
    "        ),\n",
    "        value_codeable_concept=utils.codeable_concept(\n",
    "            code=value, system=\"https://example.org/CancerStageObservation\"\n",
    "        ),\n",
    "    )\n",
    "\n",
    "\n",
    "def create_stage_from_pdc(diagnosis):\n",
    "    # Create an observation set\n",
    "    # method_type information not present in case\n",
    "    obs = crdch_model.CancerStageObservationSet()\n",
    "\n",
    "    # Add observations for every type of observation in the PDC diagnosis.\n",
    "    if diagnosis.get(\"tumor_stage\") is not None:\n",
    "        obs.observations.append(\n",
    "            create_stage_observation(\"Overall\", diagnosis.get(\"tumor_stage\"))\n",
    "        )\n",
    "\n",
    "    if diagnosis.get(\"ajcc_clinical_stage\") is not None:\n",
    "        obs.observations.append(\n",
    "            create_stage_observation(\n",
    "                \"Clinical Overall\", diagnosis.get(\"ajcc_clinical_stage\")\n",
    "            )\n",
    "        )\n",
    "\n",
    "    if diagnosis.get(\"ajcc_clinical_t\") is not None:\n",
    "        obs.observations.append(\n",
    "            create_stage_observation(\n",
    "                \"Clinical Tumor (T)\", diagnosis.get(\"ajcc_clinical_t\")\n",
    "            )\n",
    "        )\n",
    "\n",
    "    if diagnosis.get(\"ajcc_clinical_n\") is not None:\n",
    "        obs.observations.append(\n",
    "            create_stage_observation(\n",
    "                \"Clinical Node (N)\", diagnosis.get(\"ajcc_clinical_n\")\n",
    "            )\n",
    "        )\n",
    "\n",
    "    if diagnosis.get(\"ajcc_clinical_m\") is not None:\n",
    "        obs.observations.append(\n",
    "            create_stage_observation(\n",
    "                \"Clinical Metastasis (M)\", diagnosis.get(\"ajcc_clinical_m\")\n",
    "            )\n",
    "        )\n",
    "\n",
    "    if diagnosis.get(\"ajcc_pathologic_stage\") is not None:\n",
    "        obs.observations.append(\n",
    "            create_stage_observation(\n",
    "                \"Pathological Overall\", diagnosis.get(\"ajcc_pathologic_stage\")\n",
    "            )\n",
    "        )\n",
    "\n",
    "    if diagnosis.get(\"ajcc_pathologic_t\") is not None:\n",
    "        obs.observations.append(\n",
    "            create_stage_observation(\n",
    "                \"Pathological Tumor (T)\", diagnosis.get(\"ajcc_pathologic_t\")\n",
    "            )\n",
    "        )\n",
    "\n",
    "    if diagnosis.get(\"ajcc_pathologic_n\") is not None:\n",
    "        obs.observations.append(\n",
    "            create_stage_observation(\n",
    "                \"Pathological Node (N)\", diagnosis.get(\"ajcc_pathologic_n\")\n",
    "            )\n",
    "        )\n",
    "\n",
    "    if diagnosis.get(\"ajcc_pathologic_m\") is not None:\n",
    "        obs.observations.append(\n",
    "            create_stage_observation(\n",
    "                \"Pathological Metastasis (M)\", diagnosis.get(\"ajcc_pathologic_m\")\n",
    "            )\n",
    "        )\n",
    "\n",
    "    return obs\n",
    "\n",
    "\n",
    "# Test transform with the diagnosis from the first loaded case.\n",
    "# Note that the resulting CancerStageObservationSet contains descriptions for the concepts included in it.\n",
    "example_observation_set = create_stage_from_pdc(pdc_head_and_mouth[131][\"diagnoses\"][0])\n",
    "example_observation_set"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Use the LinkML YAML dumper to format and display the Cancer Stage Observation Set as a YAML string."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "observations:\n",
      "- observation_type:\n",
      "    coding:\n",
      "    - code: Overall\n",
      "      system: https://example.org/CancerStageObservation\n",
      "  value_codeable_concept:\n",
      "    coding:\n",
      "    - code: Stage IV\n",
      "      system: https://example.org/CancerStageObservation\n",
      "- observation_type:\n",
      "    coding:\n",
      "    - code: Clinical Metastasis (M)\n",
      "      system: https://example.org/CancerStageObservation\n",
      "  value_codeable_concept:\n",
      "    coding:\n",
      "    - code: M0\n",
      "      system: https://example.org/CancerStageObservation\n",
      "- observation_type:\n",
      "    coding:\n",
      "    - code: Pathological Overall\n",
      "      system: https://example.org/CancerStageObservation\n",
      "  value_codeable_concept:\n",
      "    coding:\n",
      "    - code: Stage IVA\n",
      "      system: https://example.org/CancerStageObservation\n",
      "- observation_type:\n",
      "    coding:\n",
      "    - code: Pathological Tumor (T)\n",
      "      system: https://example.org/CancerStageObservation\n",
      "  value_codeable_concept:\n",
      "    coding:\n",
      "    - code: T4a\n",
      "      system: https://example.org/CancerStageObservation\n",
      "- observation_type:\n",
      "    coding:\n",
      "    - code: Pathological Node (N)\n",
      "      system: https://example.org/CancerStageObservation\n",
      "  value_codeable_concept:\n",
      "    coding:\n",
      "    - code: N0\n",
      "      system: https://example.org/CancerStageObservation\n",
      "- observation_type:\n",
      "    coding:\n",
      "    - code: Pathological Metastasis (M)\n",
      "      system: https://example.org/CancerStageObservation\n",
      "  value_codeable_concept:\n",
      "    coding:\n",
      "    - code: Unknown\n",
      "      system: https://example.org/CancerStageObservation\n",
      "\n"
     ]
    }
   ],
   "source": [
    "from linkml_runtime.dumpers import yaml_dumper\n",
    "\n",
    "print(yaml_dumper.dumps(example_observation_set))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Transform PDC Diagnosis into CRDCH Diagnosis\n",
    "\n",
    "We first define transformation methods for creating a CCDH BodySite from the `primary_site` in the PDC case, and also for transforming sample information from the case into a CCDH Specimen. Using these two methods we can define a combined diagnosis transformation method which can transform a full PDC diagnosis into a CCDH Diagnosis."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "id: 68e054bf-2850-11ec-b712-0a4e2186f121\n",
      "identifier:\n",
      "- value: C3N-03889-DX\n",
      "  system: PDC-submitter-id\n",
      "condition:\n",
      "  coding:\n",
      "  - code: Squamous cell carcinoma, NOS\n",
      "    system: https://example.org/PrimaryDiagnosis\n",
      "metastatic_site:\n",
      "- site:\n",
      "    coding:\n",
      "    - code: Head and Neck\n",
      "      system: https://example.org/BodySite\n",
      "stage:\n",
      "- observations:\n",
      "  - observation_type:\n",
      "      coding:\n",
      "      - code: Overall\n",
      "        system: https://example.org/CancerStageObservation\n",
      "    value_codeable_concept:\n",
      "      coding:\n",
      "      - code: Stage IV\n",
      "        system: https://example.org/CancerStageObservation\n",
      "  - observation_type:\n",
      "      coding:\n",
      "      - code: Clinical Metastasis (M)\n",
      "        system: https://example.org/CancerStageObservation\n",
      "    value_codeable_concept:\n",
      "      coding:\n",
      "      - code: M0\n",
      "        system: https://example.org/CancerStageObservation\n",
      "  - observation_type:\n",
      "      coding:\n",
      "      - code: Pathological Overall\n",
      "        system: https://example.org/CancerStageObservation\n",
      "    value_codeable_concept:\n",
      "      coding:\n",
      "      - code: Stage IVA\n",
      "        system: https://example.org/CancerStageObservation\n",
      "  - observation_type:\n",
      "      coding:\n",
      "      - code: Pathological Tumor (T)\n",
      "        system: https://example.org/CancerStageObservation\n",
      "    value_codeable_concept:\n",
      "      coding:\n",
      "      - code: T4a\n",
      "        system: https://example.org/CancerStageObservation\n",
      "  - observation_type:\n",
      "      coding:\n",
      "      - code: Pathological Node (N)\n",
      "        system: https://example.org/CancerStageObservation\n",
      "    value_codeable_concept:\n",
      "      coding:\n",
      "      - code: N0\n",
      "        system: https://example.org/CancerStageObservation\n",
      "  - observation_type:\n",
      "      coding:\n",
      "      - code: Pathological Metastasis (M)\n",
      "        system: https://example.org/CancerStageObservation\n",
      "    value_codeable_concept:\n",
      "      coding:\n",
      "      - code: Unknown\n",
      "        system: https://example.org/CancerStageObservation\n",
      "morphology:\n",
      "  coding:\n",
      "  - code: 8070/3\n",
      "    system: https://example.org/Morphology\n",
      "related_specimen:\n",
      "- id: df502f10-8f98-11ea-b1fd-0aad30af8a83\n",
      "  source_material_type: Primary Tumor\n",
      "  general_tissue_morphology: Tumor\n",
      "\n"
     ]
    }
   ],
   "source": [
    "def create_body_site(site_name):\n",
    "    \"\"\"Create a CCDH BodySite based on the name of a site in the human body.\"\"\"\n",
    "\n",
    "    # Accept 'None'.\n",
    "    if site_name is None:\n",
    "        return None\n",
    "\n",
    "    # Some body sites are not currently included in the CCDH model. We will need to translate these sites\n",
    "    # into values that *are* included in the CCDH model.\n",
    "    site_mappings = {\"Larynx, NOS\": crdch_model.EnumCRDCHBodySiteSite.Larynx}\n",
    "\n",
    "    # Map values if needed. Otherwise, pass them through unmapped.\n",
    "    if site_name in site_mappings:\n",
    "        return crdch_model.BodySite(\n",
    "            utils.codeable_concept(\n",
    "                code=site_mappings[site_name], system=\"https://example.org/BodySite\"\n",
    "            )\n",
    "        )\n",
    "\n",
    "    return crdch_model.BodySite(\n",
    "        utils.codeable_concept(code=site_name, system=\"https://example.org/BodySite\")\n",
    "    )\n",
    "\n",
    "\n",
    "def transform_sample_to_specimen(sample):\n",
    "    \"\"\"A method for transforming a PDC Sample into CCDH Specimen.\"\"\"\n",
    "\n",
    "    specimen = crdch_model.Specimen(id=sample.get(\"sample_id\"))\n",
    "    specimen.source_material_type = sample.get(\"sample_type\")\n",
    "    specimen.general_tissue_morphology = sample.get(\"tissue_type\")\n",
    "    specimen.specific_tissue_morphology = sample.get(\"tumor_code\")\n",
    "    specimen.tumor_status_at_collection = sample.get(\"tumor_descriptor\")\n",
    "\n",
    "    return specimen\n",
    "\n",
    "\n",
    "def transform_diagnosis(diagnosis, case):\n",
    "    ccdh_diagnosis = crdch_model.Diagnosis(\n",
    "        id=diagnosis.get(\"diagnosis_id\"),\n",
    "        condition=utils.codeable_concept(\n",
    "            code=diagnosis.get(\"primary_diagnosis\"),\n",
    "            system=\"https://example.org/PrimaryDiagnosis\",\n",
    "        ),\n",
    "        morphology=utils.codeable_concept(\n",
    "            code=diagnosis.get(\"morphology\"), system=\"https://example.org/Morphology\"\n",
    "        ),\n",
    "        grade=diagnosis.get(\"grade\"),\n",
    "        stage=create_stage_from_pdc(diagnosis),\n",
    "        # age_at_diagnosis=diagnosis.get('year_of_diagnosis'),\n",
    "        related_specimen=[\n",
    "            transform_sample_to_specimen(sample) for sample in case.get(\"samples\")\n",
    "        ],\n",
    "    )\n",
    "    ccdh_diagnosis.identifier = [\n",
    "        crdch_model.Identifier(\n",
    "            system=\"PDC-submitter-id\", value=diagnosis.get(\"diagnosis_submitter_id\")\n",
    "        )\n",
    "    ]\n",
    "\n",
    "    if \"primary_site\" in case and case[\"primary_site\"] != \"\":\n",
    "        body_site = create_body_site(case[\"primary_site\"])\n",
    "        if body_site is not None:\n",
    "            ccdh_diagnosis.metastatic_site.append(body_site)\n",
    "\n",
    "    return ccdh_diagnosis\n",
    "\n",
    "\n",
    "example_diagnosis = transform_diagnosis(\n",
    "    pdc_head_and_mouth[131][\"diagnoses\"][0], pdc_head_and_mouth[131]\n",
    ")\n",
    "print(yaml_dumper.dumps(example_diagnosis))"
   ]
  }
 ],
 "metadata": {
  "interpreter": {
   "hash": "aee8b7b246df8f9039afb4144a1f6fd8d2ca17a180786b69acc140d282b71a49"
  },
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.9"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}