Scraping_NHS_Speciality_Training_Data.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "provenance": []
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "language_info": {
      "name": "python"
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "source": [
        "#NHS Training Recruitment Data\n",
        "##Source: https://medical.hee.nhs.uk/medical-training-recruitment/medical-specialty-training/equality-and-diversity/equality-and-diversity-2023-recruitment-data/country-of-qualification-2023-recruitment-data"
      ],
      "metadata": {
        "id": "_VT2dm2j7xbr"
      }
    },
    {
      "cell_type": "markdown",
      "source": [
        "Setting things up"
      ],
      "metadata": {
        "id": "PaqgJu_XB0TX"
      }
    },
    {
      "cell_type": "code",
      "execution_count": 50,
      "metadata": {
        "id": "qFpq5QTpHFP9"
      },
      "outputs": [],
      "source": [
        "import requests\n",
        "from bs4 import BeautifulSoup as bs\n",
        "import pandas as pd"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "my_headers = {\"User-Agent\": \"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0.0.0 Safari/537.36\"}\n",
        "url = \"https://medical.hee.nhs.uk/medical-training-recruitment/medical-specialty-training/equality-and-diversity/equality-and-diversity-2023-recruitment-data/country-of-qualification-2023-recruitment-data\"\n",
        "req = requests.get(url)"
      ],
      "metadata": {
        "id": "eDJm30NkHaY4"
      },
      "execution_count": 51,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "print(req.status_code)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "7z3_dpadHh9U",
        "outputId": "d8290a19-43fd-4903-90f1-68d87cbaacc4"
      },
      "execution_count": 52,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "200\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "soup = bs(req.content)"
      ],
      "metadata": {
        "id": "whrYvhJQHi9u"
      },
      "execution_count": 53,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "soup"
      ],
      "metadata": {
        "id": "SpPFQ8mh7UJy"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "Drilling into the html, first into the spans, then into the rows"
      ],
      "metadata": {
        "id": "Nb8Gh4G0B4xw"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "for x in soup.find_all(\"span\", class_=\"nhs-details__summary-text\"):\n",
        "  for y in x.find_all('tr'):\n",
        "    print(y)"
      ],
      "metadata": {
        "id": "PGOiu4p87dpn"
      },
      "execution_count": 56,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "all = []\n",
        "for x in soup.find_all('tr'):\n",
        "  #print('---')\n",
        "  row = []\n",
        "  for y in x.find_all('td'):\n",
        "    term = y.text\n",
        "    row.append(term)\n",
        "  #print(row)\n",
        "  all.append(row)"
      ],
      "metadata": {
        "id": "RVRQ_bQg7hYp"
      },
      "execution_count": 57,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "print(all)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "MKvIskDn7hpf",
        "outputId": "3ab7c8ae-9fd7-471a-ee4a-999599d69e17"
      },
      "execution_count": 58,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "[[], ['ACCS Anaesthetics/Core Anaesthetics CT1', '2111', '181', '1063', 'N/A'], ['ACCS Emergency Medicine ST1/CT1', '745', '95', '963', 'N/A'], ['ACCS Internal Medicine/Internal Medicine CT1', '2085', '274', '2047', 'N/A'], ['Acute Internal Medicine ST4', '108', '8', '355', 'N/A'], ['Allergy ST3', '9', '<5', '25', 'N/A'], ['Anaesthetics ST4', '775', '19', '120', 'N/A'], ['Audiovestibular Medicine ST3', '<5', '<5', '10', 'N/A'], ['Cardiology ST4', '133', '26', '274', 'N/A'], ['Cardiothoracic Surgery ST1', '82', '27', '189', 'N/A'], ['Chemical Pathology ST3', '7', '<5', '16', 'N/A'], ['Clinical Genetics ST3', '34', '<5', '24', 'N/A'], ['Clinical Neurophysiology ST3', '10', '5', '35', 'N/A'], ['Clinical Oncology ST3', '121', '<5', '113', 'N/A'], ['Clinical Pharmacology and Therapeutics ST4', '16', '<5', '15', 'N/A'], ['Clinical Radiology ST1', '1219', '227', '1622', 'N/A'], ['Clinical Radiology ST3', '0', '<5', '36', 'N/A'], ['Combined Infection Training ST3', '109', '11', '94', 'N/A'], ['Combined Infection Training ST4', '57', '7', '58', 'N/A'], ['Community Sexual and Reproductive Health ST1', '122', '12', '138', 'N/A'], ['Core Psychiatry CT1', '909', '245', '2967', 'N/A'], ['Core Surgical Training CT1', '1437', '249', '853', 'N/A'], ['Dermatology ST3', '216', '40', '118', 'N/A'], ['Diagnostic Neuropathology ST3', '5', '0', '<5', 'N/A'], ['Emergency Medicine ST3', '24', '5', '240', 'N/A'], ['Emergency Medicine ST4', '45', '5', '81', 'N/A'], ['Endocrinology and Diabetes Mellitus ST4', '64', '20', '345', 'N/A'], ['Gastroenterology ST4', '132', '24', '295', 'N/A'], ['General (Internal) Medicine ST4', '8', '5', '192', 'N/A'], ['General Practice ST1', '4269', '1163', '10268', 'N/A'], ['General Surgery ST3', '160', '27', '328', 'N/A'], ['Genitourinary Medicine ST4', '22', '<5', '12', 'N/A'], ['Geriatric Medicine ST4', '168', '12', '184', 'N/A'], ['Haematology ST3', '101', '8', '183', 'N/A'], ['Histopathology ST1', '135', '26', '219', 'N/A'], ['Immunology ST3', '19', '<5', '45', 'N/A'], ['Intensive Care Medicine ST3', '254', '15', '123', 'N/A'], ['Medical Oncology ST3', '108', '8', '156', 'N/A'], ['Medical Ophthalmology ST3', '7', '<5', '7', 'N/A'], ['Neurology ST4', '59', '8', '152', 'N/A'], ['Neurosurgery ST1', '118', '34', '103', 'N/A'], ['Neurosurgery ST2', '9', '7', '23', 'N/A'], ['Nuclear Medicine ST3', '5', '0', '12', 'N/A'], ['Obstetrics and Gynaecology ST1', '581', '120', '1074', 'N/A'], ['Obstetrics and Gynaecology ST3', '5', '7', '333', 'N/A'], ['Occupational Medicine ST3', '39', '<5', '48', 'N/A'], ['Ophthalmology ST1', '394', '73', '504', 'N/A'], ['Ophthalmology ST3', '<5', '5', '64', 'N/A'], ['Oral and Maxillo Facial Surgery ST1', '26', '<5', '11', 'N/A'], ['Oral and Maxillo Facial Surgery ST3', '22', '<5', '<5', 'N/A'], ['Otolaryngology ST3', '105', '7', '40', 'N/A'], ['Paediatric and Perinatal Pathology ST3', '7', '0', '<5', 'N/A'], ['Paediatric Cardiology ST4', '17', '7', '34', 'N/A'], ['Paediatric Surgery ST3', '27', '5', '42', 'N/A'], ['Paediatrics ST1', '544', '63', '624', 'N/A'], ['Paediatrics ST3', '15', '8', '646', 'N/A'], ['Paediatrics ST4', '16', '7', '338', 'N/A'], ['Palliative Medicine ST4', '70', '7', '17', 'N/A'], ['Plastic Surgery ST3', '141', '16', '63', 'N/A'], ['Psychiatry (all specialties) ST4', '359', '41', '361', 'N/A'], ['Public Health Medicine ST1', '429', '38', '270', '521'], ['Rehabilitation Medicine ST3', '13', '<5', '54', 'N/A'], ['Renal Medicine ST4', '65', '5', '175', 'N/A'], ['Respiratory Medicine ST4', '174', '13', '292', 'N/A'], ['Rheumatology ST4', '66', '8', '214', 'N/A'], ['Sport and Exercise Medicine ST3', '40', '<5', '21', 'N/A'], ['Thoracic Surgery ST4', '<5', '<5', '22', 'N/A'], ['Trauma and Orthopaedic Surgery ST3', '222', '21', '240', 'N/A'], ['Urology ST3', '117', '12', '123', 'N/A'], ['Vascular Surgery ST3', '52', '7', '132', 'N/A'], [], ['ACCS Anaesthetics/Core Anaesthetics CT1', '1111', '18', '47', 'N/A'], ['ACCS Emergency Medicine ST1/CT1', '512', '24', '96', 'N/A'], ['ACCS Internal Medicine/Internal Medicine CT1', '1589', '165', '1111', 'N/A'], ['Acute Internal Medicine ST4', '64', '<5', '120', 'N/A'], ['Allergy ST3', '5', '0', '<5', 'N/A'], ['Anaesthetics ST4', '680', '13', '88', 'N/A'], ['Audiovestibular Medicine ST3', '<5', '0', '<5', 'N/A'], ['Cardiology ST4', '94', '13', '99', 'N/A'], ['Cardiothoracic Surgery ST1', '9', '<5', '<5', 'N/A'], ['Chemical Pathology ST3', '5', '0', '9', 'N/A'], ['Clinical Genetics ST3', '22', '<5', '7', 'N/A'], ['Clinical Neurophysiology ST3', '6', '<5', '<5', 'N/A'], ['Clinical Oncology ST3', '91', '<5', '45', 'N/A'], ['Clinical Pharmacology and Therapeutics ST4', '10', '0', '<5', 'N/A'], ['Clinical Radiology ST1', '425', '17', '56', 'N/A'], ['Clinical Radiology ST3', '0', '0', '14', 'N/A'], ['Combined Infection Training ST3', '79', '11', '20', 'N/A'], ['Combined Infection Training ST4', '36', '<5', '16', 'N/A'], ['Community Sexual and Reproductive Health ST1', '26', '0', '0', 'N/A'], ['Core Psychiatry CT1', '732', '109', '1444', 'N/A'], ['Core Surgical Training CT1', '833', '35', '55', 'N/A'], ['Dermatology ST3', '106', '14', '15', 'N/A'], ['Diagnostic Neuropathology ST3', '<5', '0', '0', 'N/A'], ['Emergency Medicine ST3', '6', '<5', '41', 'N/A'], ['Emergency Medicine ST4', '38', '<5', '15', 'N/A'], ['Endocrinology and Diabetes Mellitus ST4', '36', '10', '116', 'N/A'], ['Gastroenterology ST4', '83', '8', '74', 'N/A'], ['General (Internal) Medicine ST4', '<5', '<5', '42', 'N/A'], ['General Practice ST1', '3860', '612', '6196', 'N/A'], ['General Surgery ST3', '109', '17', '114', 'N/A'], ['Genitourinary Medicine ST4', '14', '<5', '<5', 'N/A'], ['Geriatric Medicine ST4', '117', '6', '56', 'N/A'], ['Haematology ST3', '80', '<5', '69', 'N/A'], ['Histopathology ST1', '80', '7', '57', 'N/A'], ['Immunology ST3', '10', '<5', '<5', 'N/A'], ['Intensive Care Medicine ST3', '199', '8', '60', 'N/A'], ['Medical Oncology ST3', '76', '6', '42', 'N/A'], ['Medical Ophthalmology ST3', '<5', '<5', '0', 'N/A'], ['Neurology ST4', '40', '6', '43', 'N/A'], ['Neurosurgery ST1', '38', '<5', '<5', 'N/A'], ['Neurosurgery ST2', '<5', '<5', '<5', 'N/A'], ['Nuclear Medicine ST3', '<5', '0', '<5', 'N/A'], ['Obstetrics and Gynaecology ST1', '350', '25', '133', 'N/A'], ['Obstetrics and Gynaecology ST3', '<5', '0', '102', 'N/A'], ['Occupational Medicine ST3', '11', '0', '8', 'N/A'], ['Ophthalmology ST1', '107', '9', '5', 'N/A'], ['Ophthalmology ST3', '<5', '<5', '17', 'N/A'], ['Oral and Maxillo Facial Surgery ST1', '13', '0', '0', 'N/A'], ['Oral and Maxillo Facial Surgery ST3', '17', '<5', '<5', 'N/A'], ['Otolaryngology ST3', '74', '<5', '9', 'N/A'], ['Paediatric and Perinatal Pathology ST3', '5', '0', '0', 'N/A'], ['Paediatric Cardiology ST4', '8', '<5', '6', 'N/A'], ['Paediatric Surgery ST3', '15', '<5', '6', 'N/A'], ['Paediatrics ST1', '420', '27', '192', 'N/A'], ['Paediatrics ST3', '9', '0', '162', 'N/A'], ['Paediatrics ST4', '8', '<5', '91', 'N/A'], ['Palliative Medicine ST4', '56', '<5', '<5', 'N/A'], ['Plastic Surgery ST3', '81', '7', '11', 'N/A'], ['Psychiatry (all specialties) ST4', '332', '32', '256', 'N/A'], ['Public Health Medicine ST1', '119', '<5', '5', '54'], ['Rehabilitation Medicine ST3', '6', '0', '9', 'N/A'], ['Renal Medicine ST4', '52', '<5', '53', 'N/A'], ['Respiratory Medicine ST4', '110', '<5', '111', 'N/A'], ['Rheumatology ST4', '39', '<5', '42', 'N/A'], ['Sport and Exercise Medicine ST3', '17', '0', '0', 'N/A'], ['Thoracic Surgery ST4', '<5', '0', '<5', 'N/A'], ['Trauma and Orthopaedic Surgery ST3', '174', '10', '95', 'N/A'], ['Urology ST3', '66', '5', '36', 'N/A'], ['Vascular Surgery ST3', '27', '5', '29', 'N/A'], [], ['ACCS Anaesthetics/Core Anaesthetics CT1', '790', '9', '24', 'N/A'], ['ACCS Emergency Medicine ST1/CT1', '473', '22\\xa0', '92', 'N/A'], ['ACCS Internal Medicine/Internal Medicine CT1', '1362', '114', '725', 'N/A'], ['Acute Internal Medicine ST4', '62', '<5', '117', 'N/A'], ['Allergy ST3', '<5', '0', '<5', 'N/A'], ['Anaesthetics ST4', '548', '10', '72', 'N/A'], ['Audiovestibular Medicine ST3', '<5', '0', '<5', 'N/A'], ['Cardiology ST4', '74', '10', '79', 'N/A'], ['Cardiothoracic Surgery ST1', '8', '<5', '<5', 'N/A'], ['Chemical Pathology ST3', '5', '0', '9', 'N/A'], ['Clinical Genetics ST3', '16', '<5', '7', 'N/A'], ['Clinical Neurophysiology ST3', '<5', '<5', '<5', 'N/A'], ['Clinical Oncology ST3', '91', '<5', '45', 'N/A'], ['Clinical Pharmacology and Therapeutics ST4', '10', '0', '<5', 'N/A'], ['Clinical Radiology ST1', '365', '14', '46', 'N/A'], ['Clinical Radiology ST3', '0', '0', '14', 'N/A'], ['Combined Infection Training ST3', '57', '9', '20', 'N/A'], ['Combined Infection Training ST4', '30', '<5', '14', 'N/A'], ['Community Sexual and Reproductive Health ST1', '10', '0', '0', 'N/A'], ['Core Psychiatry CT1', '675', '64', '573', 'N/A'], ['Core Surgical Training CT1', '692', '26', '42', 'N/A'], ['Dermatology ST3', '52', '9', '5', 'N/A'], ['Diagnostic Neuropathology ST3', '<5', '0', '0', 'N/A'], ['Emergency Medicine ST3', '6', '<5', '24', 'N/A'], ['Emergency Medicine ST4', '37', '<5', '15', 'N/A'], ['Endocrinology and Diabetes Mellitus ST4', '33', '7', '100', 'N/A'], ['Gastroenterology ST4', '74', '6', '63', 'N/A'], ['General (Internal) Medicine ST4', '<5', '<5', '42', 'N/A'], ['General Practice ST1', '3702', '408', '3375', 'N/A'], ['General Surgery ST3', '100', '13', '79', 'N/A'], ['Genitourinary Medicine ST4', '14', '<5', '<5', 'N/A'], ['Geriatric Medicine ST4', '110', '6', '56', 'N/A'], ['Haematology ST3', '67', '<5', '69', 'N/A'], ['Histopathology ST1', '77', '7', '47', 'N/A'], ['Immunology ST3', '7', '0', '<5', 'N/A'], ['Intensive Care Medicine ST3', '189', '6', '52', 'N/A'], ['Medical Oncology ST3', '76', '5', '42', 'N/A'], ['Medical Ophthalmology ST3', '<5', '0', '0', 'N/A'], ['Neurology ST4', '35', '5', '43', 'N/A'], ['Neurosurgery ST1', '25', '<5', '0', 'N/A'], ['Neurosurgery ST2', '<5', '0', '0', 'N/A'], ['Nuclear Medicine ST3', '<5', '0', '<5', 'N/A'], ['Obstetrics and Gynaecology ST1', '337', '25', '100', 'N/A'], ['Obstetrics and Gynaecology ST3', '<5', '0', '93', 'N/A'], ['Occupational Medicine ST3', '11', '0', '8', 'N/A'], ['Ophthalmology ST1', '104', '8', '5', 'N/A'], ['Ophthalmology ST3', '<5', '<5', '17', 'N/A'], ['Oral and Maxillo Facial Surgery ST1', '13', '0', '0', 'N/A'], ['Oral and Maxillo Facial Surgery ST3', '15', '<5', '<5', 'N/A'], ['Otolaryngology ST3', '65', '<5', '9', 'N/A'], ['Paediatric and Perinatal Pathology ST3', '5', '0', '0', 'N/A'], ['Paediatric Cardiology ST4', '7', '<5', '5', 'N/A'], ['Paediatric Surgery ST3', '14', '<5', '5', 'N/A'], ['Paediatrics ST1', '403', '26', '190', 'N/A'], ['Paediatrics ST3', '8', '0', '136', 'N/A'], ['Paediatrics ST4', '7', '<5', '78', 'N/A'], ['Palliative Medicine ST4', '48', '<5', '<5', 'N/A'], ['Plastic Surgery ST3', '52', '<5', '<5', 'N/A'], ['Psychiatry (all specialties) ST4', '296', '30', '238', 'N/A'], ['Public Health Medicine ST1', '106', '<5', '<5', '43'], ['Rehabilitation Medicine ST3', '6', '0', '9', 'N/A'], ['Renal Medicine ST4', '47', '<5', '53', 'N/A'], ['Respiratory Medicine ST4', '100', '<5', '97', 'N/A'], ['Rheumatology ST4', '35', '<5', '37', 'N/A'], ['Sport and Exercise Medicine ST3', '10', '0', '0', 'N/A'], ['Thoracic Surgery ST4', '<5', '0', '<5', 'N/A'], ['Trauma and Orthopaedic Surgery ST3', '133', '5', '43', 'N/A'], ['Urology ST3', '57', '<5', '28', 'N/A'], ['Vascular Surgery ST3', '27', '<5', '29', 'N/A'], [], ['ACCS Anaesthetics/Core Anaesthetics CT1', '606', '6', '18', 'N/A\\xa0'], ['ACCS Emergency Medicine ST1/CT1', '289', '12', '58', 'N/A'], ['ACCS Internal Medicine/Internal Medicine CT1', '1004', '84', '583', 'N/A'], ['Acute Internal Medicine ST4', '41', '<5', '53', 'N/A'], ['Allergy ST3', '<5', '0', '<5', 'N/A'], ['Anaesthetics ST4', '500', '10', '67', 'N/A'], ['Audiovestibular Medicine ST3', '<5', '0', '<5', 'N/A'], ['Cardiology ST4', '63', '7', '70', 'N/A'], ['Cardiothoracic Surgery ST1', '8', '<5', '<5', 'N/A'], ['Chemical Pathology ST3', '<5', '0', '7', 'N/A'], ['Clinical Genetics ST3', '15', '<5', '7', 'N/A'], ['Clinical Neurophysiology ST3', '<5', '<5', '<5', 'N/A'], ['Clinical Oncology ST3', '56', '0', '26', 'N/A'], ['Clinical Pharmacology and Therapeutics ST4', '5', '0', '<5', 'N/A'], ['Clinical Radiology ST1', '296', '11', '43', 'N/A'], ['Clinical Radiology ST3', '0', '0', '13', 'N/A'], ['Combined Infection Training ST3', '32', '7', '12', 'N/A'], ['Combined Infection Training ST4', '18', '<5', '<5', 'N/A'], ['Community Sexual and Reproductive Health ST1', '8', '0', '0', 'N/A'], ['Core Psychiatry CT1', '354', '33', '287', 'N/A'], ['Core Surgical Training CT1', '550', '20', '39', 'N/A'], ['Dermatology ST3', '43', '8', '5', 'N/A'], ['Diagnostic Neuropathology ST3', '<5', '0', '0', 'N/A'], ['Emergency Medicine ST3', '5', '<5', '19', 'N/A'], ['Emergency Medicine ST4', '24', '<5', '14', 'N/A'], ['Endocrinology and Diabetes Mellitus ST4', '26', '<5', '71', 'N/A'], ['Gastroenterology ST4', '73', '6', '54', 'N/A'], ['General (Internal) Medicine ST4', '<5', '0', '24', 'N/A'], ['General Practice ST1', '2048', '258', '2303', 'N/A'], ['General Surgery ST3', '82', '11', '69', 'N/A'], ['Genitourinary Medicine ST4', '7', '<5', '<5', 'N/A'], ['Geriatric Medicine ST4', '94', '5', '31', 'N/A'], ['Haematology ST3', '50', '<5', '52', 'N/A'], ['Histopathology ST1', '59', '6', '43', 'N/A'], ['Immunology ST3', '7', '0', '<5', 'N/A'], ['Intensive Care Medicine ST3', '133', '<5', '32', 'N/A'], ['Medical Oncology ST3', '40', '<5', '27', 'N/A'], ['Medical Ophthalmology ST3', '<5', '0', '0', 'N/A'], ['Neurology ST4', '25', '5', '35', 'N/A'], ['Neurosurgery ST1', '18', '<5', '0', 'N/A'], ['Neurosurgery ST2', '<5', '0', '0', 'N/A'], ['Nuclear Medicine ST3', '<5', '0', '<5', 'N/A'], ['Obstetrics and Gynaecology ST1', '226', '19', '61', 'N/A'], ['Obstetrics and Gynaecology ST3', '<5', '0', '87', 'N/A'], ['Occupational Medicine ST3', '<5', '0', '5', 'N/A'], ['Ophthalmology ST1', '84', '8', '<5', 'N/A'], ['Ophthalmology ST3', '<5', '<5', '16', 'N/A'], ['Oral and Maxillo Facial Surgery ST1', '11', '0', '0', 'N/A'], ['Oral and Maxillo Facial Surgery ST3', '11', '<5', '<5', 'N/A'], ['Otolaryngology ST3', '57', '<5', '6', 'N/A'], ['Paediatric and Perinatal Pathology ST3', '<5', '0', '0', 'N/A'], ['Paediatric Cardiology ST4', '6', '<5', '<5', 'N/A'], ['Paediatric Surgery ST3', '12', '0', '5', 'N/A'], ['Paediatrics ST1', '326', '22', '136', 'N/A'], ['Paediatrics ST3', '6', '0', '101', 'N/A'], ['Paediatrics ST4', '7', '<5', '61', 'N/A'], ['Palliative Medicine ST4', '40', '<5', '<5', 'N/A'], ['Plastic Surgery ST3', '48', '<5', '<5', 'N/A'], ['Psychiatry (all specialties) ST4', '253', '27', '220', 'N/A'], ['Public Health Medicine ST1', '82', '<5', '<5', '38'], ['Rehabilitation Medicine ST3', '5', '0', '6', 'N/A'], ['Renal Medicine ST4', '37', '<5', '36', 'N/A'], ['Respiratory Medicine ST4', '78', '<5', '69', 'N/A'], ['Rheumatology ST4', '28', '<5', '29', 'N/A'], ['Sport and Exercise Medicine ST3', '9', '0', '0', 'N/A'], ['Thoracic Surgery ST4', '<5', '0', '<5', 'N/A'], ['Trauma and Orthopaedic Surgery ST3', '116', '5', '39', 'N/A'], ['Urology ST3', '54', '<5', '26', 'N/A'], ['Vascular Surgery ST3', '13', '5', '24', 'N/A']]\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "all_ok = all[1:]"
      ],
      "metadata": {
        "id": "ibYGragB8kgd"
      },
      "execution_count": 59,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "Creating a dataframe"
      ],
      "metadata": {
        "id": "tfka_Gz4CA9i"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "cols = ['specialty_and_level', 'UK', 'EU', 'rest_of_world', 'non_medical']\n",
        "\n",
        "df = pd.DataFrame(all_ok, columns = cols)\n",
        "df"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 423
        },
        "id": "klQUkGRV7iBF",
        "outputId": "573ad1b4-3acf-4212-e5b7-657112eece7a"
      },
      "execution_count": 60,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "                              specialty_and_level    UK   EU rest_of_world  \\\n",
              "0         ACCS Anaesthetics/Core Anaesthetics CT1  2111  181          1063   \n",
              "1                 ACCS Emergency Medicine ST1/CT1   745   95           963   \n",
              "2    ACCS Internal Medicine/Internal Medicine CT1  2085  274          2047   \n",
              "3                     Acute Internal Medicine ST4   108    8           355   \n",
              "4                                     Allergy ST3     9   <5            25   \n",
              "..                                            ...   ...  ...           ...   \n",
              "274               Sport and Exercise Medicine ST3     9    0             0   \n",
              "275                          Thoracic Surgery ST4    <5    0            <5   \n",
              "276            Trauma and Orthopaedic Surgery ST3   116    5            39   \n",
              "277                                   Urology ST3    54   <5            26   \n",
              "278                          Vascular Surgery ST3    13    5            24   \n",
              "\n",
              "    non_medical  \n",
              "0           N/A  \n",
              "1           N/A  \n",
              "2           N/A  \n",
              "3           N/A  \n",
              "4           N/A  \n",
              "..          ...  \n",
              "274         N/A  \n",
              "275         N/A  \n",
              "276         N/A  \n",
              "277         N/A  \n",
              "278         N/A  \n",
              "\n",
              "[279 rows x 5 columns]"
            ],
            "text/html": [
              "\n",
              "  <div id=\"df-7b059133-372e-45d4-8279-f87d3d98e6b4\" class=\"colab-df-container\">\n",
              "    <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>specialty_and_level</th>\n",
              "      <th>UK</th>\n",
              "      <th>EU</th>\n",
              "      <th>rest_of_world</th>\n",
              "      <th>non_medical</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>ACCS Anaesthetics/Core Anaesthetics CT1</td>\n",
              "      <td>2111</td>\n",
              "      <td>181</td>\n",
              "      <td>1063</td>\n",
              "      <td>N/A</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>ACCS Emergency Medicine ST1/CT1</td>\n",
              "      <td>745</td>\n",
              "      <td>95</td>\n",
              "      <td>963</td>\n",
              "      <td>N/A</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>ACCS Internal Medicine/Internal Medicine CT1</td>\n",
              "      <td>2085</td>\n",
              "      <td>274</td>\n",
              "      <td>2047</td>\n",
              "      <td>N/A</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>Acute Internal Medicine ST4</td>\n",
              "      <td>108</td>\n",
              "      <td>8</td>\n",
              "      <td>355</td>\n",
              "      <td>N/A</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>Allergy ST3</td>\n",
              "      <td>9</td>\n",
              "      <td>&lt;5</td>\n",
              "      <td>25</td>\n",
              "      <td>N/A</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>...</th>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>274</th>\n",
              "      <td>Sport and Exercise Medicine ST3</td>\n",
              "      <td>9</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>N/A</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>275</th>\n",
              "      <td>Thoracic Surgery ST4</td>\n",
              "      <td>&lt;5</td>\n",
              "      <td>0</td>\n",
              "      <td>&lt;5</td>\n",
              "      <td>N/A</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>276</th>\n",
              "      <td>Trauma and Orthopaedic Surgery ST3</td>\n",
              "      <td>116</td>\n",
              "      <td>5</td>\n",
              "      <td>39</td>\n",
              "      <td>N/A</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>277</th>\n",
              "      <td>Urology ST3</td>\n",
              "      <td>54</td>\n",
              "      <td>&lt;5</td>\n",
              "      <td>26</td>\n",
              "      <td>N/A</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>278</th>\n",
              "      <td>Vascular Surgery ST3</td>\n",
              "      <td>13</td>\n",
              "      <td>5</td>\n",
              "      <td>24</td>\n",
              "      <td>N/A</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "<p>279 rows × 5 columns</p>\n",
              "</div>\n",
              "    <div class=\"colab-df-buttons\">\n",
              "\n",
              "  <div class=\"colab-df-container\">\n",
              "    <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-7b059133-372e-45d4-8279-f87d3d98e6b4')\"\n",
              "            title=\"Convert this dataframe to an interactive table.\"\n",
              "            style=\"display:none;\">\n",
              "\n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\" viewBox=\"0 -960 960 960\">\n",
              "    <path d=\"M120-120v-720h720v720H120Zm60-500h600v-160H180v160Zm220 220h160v-160H400v160Zm0 220h160v-160H400v160ZM180-400h160v-160H180v160Zm440 0h160v-160H620v160ZM180-180h160v-160H180v160Zm440 0h160v-160H620v160Z\"/>\n",
              "  </svg>\n",
              "    </button>\n",
              "\n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    .colab-df-buttons div {\n",
              "      margin-bottom: 4px;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "    <script>\n",
              "      const buttonEl =\n",
              "        document.querySelector('#df-7b059133-372e-45d4-8279-f87d3d98e6b4 button.colab-df-convert');\n",
              "      buttonEl.style.display =\n",
              "        google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "      async function convertToInteractive(key) {\n",
              "        const element = document.querySelector('#df-7b059133-372e-45d4-8279-f87d3d98e6b4');\n",
              "        const dataTable =\n",
              "          await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                    [key], {});\n",
              "        if (!dataTable) return;\n",
              "\n",
              "        const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "          '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "          + ' to learn more about interactive tables.';\n",
              "        element.innerHTML = '';\n",
              "        dataTable['output_type'] = 'display_data';\n",
              "        await google.colab.output.renderOutput(dataTable, element);\n",
              "        const docLink = document.createElement('div');\n",
              "        docLink.innerHTML = docLinkHtml;\n",
              "        element.appendChild(docLink);\n",
              "      }\n",
              "    </script>\n",
              "  </div>\n",
              "\n",
              "\n",
              "<div id=\"df-834108ff-1dd1-4b2b-9405-e77f3a18916f\">\n",
              "  <button class=\"colab-df-quickchart\" onclick=\"quickchart('df-834108ff-1dd1-4b2b-9405-e77f3a18916f')\"\n",
              "            title=\"Suggest charts\"\n",
              "            style=\"display:none;\">\n",
              "\n",
              "<svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "     width=\"24px\">\n",
              "    <g>\n",
              "        <path d=\"M19 3H5c-1.1 0-2 .9-2 2v14c0 1.1.9 2 2 2h14c1.1 0 2-.9 2-2V5c0-1.1-.9-2-2-2zM9 17H7v-7h2v7zm4 0h-2V7h2v10zm4 0h-2v-4h2v4z\"/>\n",
              "    </g>\n",
              "</svg>\n",
              "  </button>\n",
              "\n",
              "<style>\n",
              "  .colab-df-quickchart {\n",
              "      --bg-color: #E8F0FE;\n",
              "      --fill-color: #1967D2;\n",
              "      --hover-bg-color: #E2EBFA;\n",
              "      --hover-fill-color: #174EA6;\n",
              "      --disabled-fill-color: #AAA;\n",
              "      --disabled-bg-color: #DDD;\n",
              "  }\n",
              "\n",
              "  [theme=dark] .colab-df-quickchart {\n",
              "      --bg-color: #3B4455;\n",
              "      --fill-color: #D2E3FC;\n",
              "      --hover-bg-color: #434B5C;\n",
              "      --hover-fill-color: #FFFFFF;\n",
              "      --disabled-bg-color: #3B4455;\n",
              "      --disabled-fill-color: #666;\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart {\n",
              "    background-color: var(--bg-color);\n",
              "    border: none;\n",
              "    border-radius: 50%;\n",
              "    cursor: pointer;\n",
              "    display: none;\n",
              "    fill: var(--fill-color);\n",
              "    height: 32px;\n",
              "    padding: 0;\n",
              "    width: 32px;\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart:hover {\n",
              "    background-color: var(--hover-bg-color);\n",
              "    box-shadow: 0 1px 2px rgba(60, 64, 67, 0.3), 0 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "    fill: var(--button-hover-fill-color);\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart-complete:disabled,\n",
              "  .colab-df-quickchart-complete:disabled:hover {\n",
              "    background-color: var(--disabled-bg-color);\n",
              "    fill: var(--disabled-fill-color);\n",
              "    box-shadow: none;\n",
              "  }\n",
              "\n",
              "  .colab-df-spinner {\n",
              "    border: 2px solid var(--fill-color);\n",
              "    border-color: transparent;\n",
              "    border-bottom-color: var(--fill-color);\n",
              "    animation:\n",
              "      spin 1s steps(1) infinite;\n",
              "  }\n",
              "\n",
              "  @keyframes spin {\n",
              "    0% {\n",
              "      border-color: transparent;\n",
              "      border-bottom-color: var(--fill-color);\n",
              "      border-left-color: var(--fill-color);\n",
              "    }\n",
              "    20% {\n",
              "      border-color: transparent;\n",
              "      border-left-color: var(--fill-color);\n",
              "      border-top-color: var(--fill-color);\n",
              "    }\n",
              "    30% {\n",
              "      border-color: transparent;\n",
              "      border-left-color: var(--fill-color);\n",
              "      border-top-color: var(--fill-color);\n",
              "      border-right-color: var(--fill-color);\n",
              "    }\n",
              "    40% {\n",
              "      border-color: transparent;\n",
              "      border-right-color: var(--fill-color);\n",
              "      border-top-color: var(--fill-color);\n",
              "    }\n",
              "    60% {\n",
              "      border-color: transparent;\n",
              "      border-right-color: var(--fill-color);\n",
              "    }\n",
              "    80% {\n",
              "      border-color: transparent;\n",
              "      border-right-color: var(--fill-color);\n",
              "      border-bottom-color: var(--fill-color);\n",
              "    }\n",
              "    90% {\n",
              "      border-color: transparent;\n",
              "      border-bottom-color: var(--fill-color);\n",
              "    }\n",
              "  }\n",
              "</style>\n",
              "\n",
              "  <script>\n",
              "    async function quickchart(key) {\n",
              "      const quickchartButtonEl =\n",
              "        document.querySelector('#' + key + ' button');\n",
              "      quickchartButtonEl.disabled = true;  // To prevent multiple clicks.\n",
              "      quickchartButtonEl.classList.add('colab-df-spinner');\n",
              "      try {\n",
              "        const charts = await google.colab.kernel.invokeFunction(\n",
              "            'suggestCharts', [key], {});\n",
              "      } catch (error) {\n",
              "        console.error('Error during call to suggestCharts:', error);\n",
              "      }\n",
              "      quickchartButtonEl.classList.remove('colab-df-spinner');\n",
              "      quickchartButtonEl.classList.add('colab-df-quickchart-complete');\n",
              "    }\n",
              "    (() => {\n",
              "      let quickchartButtonEl =\n",
              "        document.querySelector('#df-834108ff-1dd1-4b2b-9405-e77f3a18916f button');\n",
              "      quickchartButtonEl.style.display =\n",
              "        google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "    })();\n",
              "  </script>\n",
              "</div>\n",
              "    </div>\n",
              "  </div>\n"
            ],
            "application/vnd.google.colaboratory.intrinsic+json": {
              "type": "dataframe",
              "variable_name": "df",
              "summary": "{\n  \"name\": \"df\",\n  \"rows\": 279,\n  \"fields\": [\n    {\n      \"column\": \"specialty_and_level\",\n      \"properties\": {\n        \"dtype\": \"category\",\n        \"num_unique_values\": 69,\n        \"samples\": [\n          \"Diagnostic Neuropathology ST3\",\n          \"ACCS Anaesthetics/Core Anaesthetics CT1\",\n          \"Oral and Maxillo Facial Surgery ST1\"\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    },\n    {\n      \"column\": \"UK\",\n      \"properties\": {\n        \"dtype\": \"category\",\n        \"num_unique_values\": 136,\n        \"samples\": [\n          \"83\",\n          \"544\",\n          \"512\"\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    },\n    {\n      \"column\": \"EU\",\n      \"properties\": {\n        \"dtype\": \"category\",\n        \"num_unique_values\": 51,\n        \"samples\": [\n          \"114\",\n          \"9\",\n          \"30\"\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    },\n    {\n      \"column\": \"rest_of_world\",\n      \"properties\": {\n        \"dtype\": \"category\",\n        \"num_unique_values\": 133,\n        \"samples\": [\n          \"646\",\n          \"9\",\n          \"184\"\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    },\n    {\n      \"column\": \"non_medical\",\n      \"properties\": {\n        \"dtype\": \"category\",\n        \"num_unique_values\": 6,\n        \"samples\": [\n          \"N/A\",\n          \"521\",\n          \"38\"\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    }\n  ]\n}"
            }
          },
          "metadata": {},
          "execution_count": 60
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "accepted_df = df[210:]"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 423
        },
        "id": "2kxfyM8G88Cc",
        "outputId": "52a9e374-61cb-4bb4-eb56-f7d617d55752"
      },
      "execution_count": 61,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "                              specialty_and_level    UK  EU rest_of_world  \\\n",
              "210       ACCS Anaesthetics/Core Anaesthetics CT1   606   6            18   \n",
              "211               ACCS Emergency Medicine ST1/CT1   289  12            58   \n",
              "212  ACCS Internal Medicine/Internal Medicine CT1  1004  84           583   \n",
              "213                   Acute Internal Medicine ST4    41  <5            53   \n",
              "214                                   Allergy ST3    <5   0            <5   \n",
              "..                                            ...   ...  ..           ...   \n",
              "274               Sport and Exercise Medicine ST3     9   0             0   \n",
              "275                          Thoracic Surgery ST4    <5   0            <5   \n",
              "276            Trauma and Orthopaedic Surgery ST3   116   5            39   \n",
              "277                                   Urology ST3    54  <5            26   \n",
              "278                          Vascular Surgery ST3    13   5            24   \n",
              "\n",
              "    non_medical  \n",
              "210        N/A   \n",
              "211         N/A  \n",
              "212         N/A  \n",
              "213         N/A  \n",
              "214         N/A  \n",
              "..          ...  \n",
              "274         N/A  \n",
              "275         N/A  \n",
              "276         N/A  \n",
              "277         N/A  \n",
              "278         N/A  \n",
              "\n",
              "[69 rows x 5 columns]"
            ],
            "text/html": [
              "\n",
              "  <div id=\"df-ad669e1a-2a9f-43cc-b906-ca0738f55459\" class=\"colab-df-container\">\n",
              "    <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>specialty_and_level</th>\n",
              "      <th>UK</th>\n",
              "      <th>EU</th>\n",
              "      <th>rest_of_world</th>\n",
              "      <th>non_medical</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>210</th>\n",
              "      <td>ACCS Anaesthetics/Core Anaesthetics CT1</td>\n",
              "      <td>606</td>\n",
              "      <td>6</td>\n",
              "      <td>18</td>\n",
              "      <td>N/A</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>211</th>\n",
              "      <td>ACCS Emergency Medicine ST1/CT1</td>\n",
              "      <td>289</td>\n",
              "      <td>12</td>\n",
              "      <td>58</td>\n",
              "      <td>N/A</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>212</th>\n",
              "      <td>ACCS Internal Medicine/Internal Medicine CT1</td>\n",
              "      <td>1004</td>\n",
              "      <td>84</td>\n",
              "      <td>583</td>\n",
              "      <td>N/A</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>213</th>\n",
              "      <td>Acute Internal Medicine ST4</td>\n",
              "      <td>41</td>\n",
              "      <td>&lt;5</td>\n",
              "      <td>53</td>\n",
              "      <td>N/A</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>214</th>\n",
              "      <td>Allergy ST3</td>\n",
              "      <td>&lt;5</td>\n",
              "      <td>0</td>\n",
              "      <td>&lt;5</td>\n",
              "      <td>N/A</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>...</th>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>274</th>\n",
              "      <td>Sport and Exercise Medicine ST3</td>\n",
              "      <td>9</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>N/A</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>275</th>\n",
              "      <td>Thoracic Surgery ST4</td>\n",
              "      <td>&lt;5</td>\n",
              "      <td>0</td>\n",
              "      <td>&lt;5</td>\n",
              "      <td>N/A</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>276</th>\n",
              "      <td>Trauma and Orthopaedic Surgery ST3</td>\n",
              "      <td>116</td>\n",
              "      <td>5</td>\n",
              "      <td>39</td>\n",
              "      <td>N/A</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>277</th>\n",
              "      <td>Urology ST3</td>\n",
              "      <td>54</td>\n",
              "      <td>&lt;5</td>\n",
              "      <td>26</td>\n",
              "      <td>N/A</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>278</th>\n",
              "      <td>Vascular Surgery ST3</td>\n",
              "      <td>13</td>\n",
              "      <td>5</td>\n",
              "      <td>24</td>\n",
              "      <td>N/A</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "<p>69 rows × 5 columns</p>\n",
              "</div>\n",
              "    <div class=\"colab-df-buttons\">\n",
              "\n",
              "  <div class=\"colab-df-container\">\n",
              "    <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-ad669e1a-2a9f-43cc-b906-ca0738f55459')\"\n",
              "            title=\"Convert this dataframe to an interactive table.\"\n",
              "            style=\"display:none;\">\n",
              "\n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\" viewBox=\"0 -960 960 960\">\n",
              "    <path d=\"M120-120v-720h720v720H120Zm60-500h600v-160H180v160Zm220 220h160v-160H400v160Zm0 220h160v-160H400v160ZM180-400h160v-160H180v160Zm440 0h160v-160H620v160ZM180-180h160v-160H180v160Zm440 0h160v-160H620v160Z\"/>\n",
              "  </svg>\n",
              "    </button>\n",
              "\n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    .colab-df-buttons div {\n",
              "      margin-bottom: 4px;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "    <script>\n",
              "      const buttonEl =\n",
              "        document.querySelector('#df-ad669e1a-2a9f-43cc-b906-ca0738f55459 button.colab-df-convert');\n",
              "      buttonEl.style.display =\n",
              "        google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "      async function convertToInteractive(key) {\n",
              "        const element = document.querySelector('#df-ad669e1a-2a9f-43cc-b906-ca0738f55459');\n",
              "        const dataTable =\n",
              "          await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                    [key], {});\n",
              "        if (!dataTable) return;\n",
              "\n",
              "        const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "          '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "          + ' to learn more about interactive tables.';\n",
              "        element.innerHTML = '';\n",
              "        dataTable['output_type'] = 'display_data';\n",
              "        await google.colab.output.renderOutput(dataTable, element);\n",
              "        const docLink = document.createElement('div');\n",
              "        docLink.innerHTML = docLinkHtml;\n",
              "        element.appendChild(docLink);\n",
              "      }\n",
              "    </script>\n",
              "  </div>\n",
              "\n",
              "\n",
              "<div id=\"df-5ba6f960-6056-442b-a024-a1bf89b9d59f\">\n",
              "  <button class=\"colab-df-quickchart\" onclick=\"quickchart('df-5ba6f960-6056-442b-a024-a1bf89b9d59f')\"\n",
              "            title=\"Suggest charts\"\n",
              "            style=\"display:none;\">\n",
              "\n",
              "<svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "     width=\"24px\">\n",
              "    <g>\n",
              "        <path d=\"M19 3H5c-1.1 0-2 .9-2 2v14c0 1.1.9 2 2 2h14c1.1 0 2-.9 2-2V5c0-1.1-.9-2-2-2zM9 17H7v-7h2v7zm4 0h-2V7h2v10zm4 0h-2v-4h2v4z\"/>\n",
              "    </g>\n",
              "</svg>\n",
              "  </button>\n",
              "\n",
              "<style>\n",
              "  .colab-df-quickchart {\n",
              "      --bg-color: #E8F0FE;\n",
              "      --fill-color: #1967D2;\n",
              "      --hover-bg-color: #E2EBFA;\n",
              "      --hover-fill-color: #174EA6;\n",
              "      --disabled-fill-color: #AAA;\n",
              "      --disabled-bg-color: #DDD;\n",
              "  }\n",
              "\n",
              "  [theme=dark] .colab-df-quickchart {\n",
              "      --bg-color: #3B4455;\n",
              "      --fill-color: #D2E3FC;\n",
              "      --hover-bg-color: #434B5C;\n",
              "      --hover-fill-color: #FFFFFF;\n",
              "      --disabled-bg-color: #3B4455;\n",
              "      --disabled-fill-color: #666;\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart {\n",
              "    background-color: var(--bg-color);\n",
              "    border: none;\n",
              "    border-radius: 50%;\n",
              "    cursor: pointer;\n",
              "    display: none;\n",
              "    fill: var(--fill-color);\n",
              "    height: 32px;\n",
              "    padding: 0;\n",
              "    width: 32px;\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart:hover {\n",
              "    background-color: var(--hover-bg-color);\n",
              "    box-shadow: 0 1px 2px rgba(60, 64, 67, 0.3), 0 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "    fill: var(--button-hover-fill-color);\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart-complete:disabled,\n",
              "  .colab-df-quickchart-complete:disabled:hover {\n",
              "    background-color: var(--disabled-bg-color);\n",
              "    fill: var(--disabled-fill-color);\n",
              "    box-shadow: none;\n",
              "  }\n",
              "\n",
              "  .colab-df-spinner {\n",
              "    border: 2px solid var(--fill-color);\n",
              "    border-color: transparent;\n",
              "    border-bottom-color: var(--fill-color);\n",
              "    animation:\n",
              "      spin 1s steps(1) infinite;\n",
              "  }\n",
              "\n",
              "  @keyframes spin {\n",
              "    0% {\n",
              "      border-color: transparent;\n",
              "      border-bottom-color: var(--fill-color);\n",
              "      border-left-color: var(--fill-color);\n",
              "    }\n",
              "    20% {\n",
              "      border-color: transparent;\n",
              "      border-left-color: var(--fill-color);\n",
              "      border-top-color: var(--fill-color);\n",
              "    }\n",
              "    30% {\n",
              "      border-color: transparent;\n",
              "      border-left-color: var(--fill-color);\n",
              "      border-top-color: var(--fill-color);\n",
              "      border-right-color: var(--fill-color);\n",
              "    }\n",
              "    40% {\n",
              "      border-color: transparent;\n",
              "      border-right-color: var(--fill-color);\n",
              "      border-top-color: var(--fill-color);\n",
              "    }\n",
              "    60% {\n",
              "      border-color: transparent;\n",
              "      border-right-color: var(--fill-color);\n",
              "    }\n",
              "    80% {\n",
              "      border-color: transparent;\n",
              "      border-right-color: var(--fill-color);\n",
              "      border-bottom-color: var(--fill-color);\n",
              "    }\n",
              "    90% {\n",
              "      border-color: transparent;\n",
              "      border-bottom-color: var(--fill-color);\n",
              "    }\n",
              "  }\n",
              "</style>\n",
              "\n",
              "  <script>\n",
              "    async function quickchart(key) {\n",
              "      const quickchartButtonEl =\n",
              "        document.querySelector('#' + key + ' button');\n",
              "      quickchartButtonEl.disabled = true;  // To prevent multiple clicks.\n",
              "      quickchartButtonEl.classList.add('colab-df-spinner');\n",
              "      try {\n",
              "        const charts = await google.colab.kernel.invokeFunction(\n",
              "            'suggestCharts', [key], {});\n",
              "      } catch (error) {\n",
              "        console.error('Error during call to suggestCharts:', error);\n",
              "      }\n",
              "      quickchartButtonEl.classList.remove('colab-df-spinner');\n",
              "      quickchartButtonEl.classList.add('colab-df-quickchart-complete');\n",
              "    }\n",
              "    (() => {\n",
              "      let quickchartButtonEl =\n",
              "        document.querySelector('#df-5ba6f960-6056-442b-a024-a1bf89b9d59f button');\n",
              "      quickchartButtonEl.style.display =\n",
              "        google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "    })();\n",
              "  </script>\n",
              "</div>\n",
              "    </div>\n",
              "  </div>\n"
            ],
            "application/vnd.google.colaboratory.intrinsic+json": {
              "type": "dataframe",
              "summary": "{\n  \"name\": \"df[210:]\",\n  \"rows\": 69,\n  \"fields\": [\n    {\n      \"column\": \"specialty_and_level\",\n      \"properties\": {\n        \"dtype\": \"string\",\n        \"num_unique_values\": 69,\n        \"samples\": [\n          \"Diagnostic Neuropathology ST3\",\n          \"ACCS Anaesthetics/Core Anaesthetics CT1\",\n          \"Oral and Maxillo Facial Surgery ST1\"\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    },\n    {\n      \"column\": \"UK\",\n      \"properties\": {\n        \"dtype\": \"string\",\n        \"num_unique_values\": 46,\n        \"samples\": [\n          \"37\",\n          \"50\",\n          \"59\"\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    },\n    {\n      \"column\": \"EU\",\n      \"properties\": {\n        \"dtype\": \"category\",\n        \"num_unique_values\": 16,\n        \"samples\": [\n          \"6\",\n          \"12\",\n          \"10\"\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    },\n    {\n      \"column\": \"rest_of_world\",\n      \"properties\": {\n        \"dtype\": \"string\",\n        \"num_unique_values\": 37,\n        \"samples\": [\n          \"14\",\n          \"287\",\n          \"<5\"\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    },\n    {\n      \"column\": \"non_medical\",\n      \"properties\": {\n        \"dtype\": \"category\",\n        \"num_unique_values\": 3,\n        \"samples\": [\n          \"N/A\\u00a0\",\n          \"N/A\",\n          \"38\"\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    }\n  ]\n}"
            }
          },
          "metadata": {},
          "execution_count": 61
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "Creating a csv file"
      ],
      "metadata": {
        "id": "wfKamfWPCDli"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "accepted_df.to_csv('nhs_training.csv', index = False)"
      ],
      "metadata": {
        "id": "3gfyrBmy8724"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "Testing out another method. This one drills into the correct table right away, that way you don't have all four tables in the dataframe."
      ],
      "metadata": {
        "id": "GQPePfDZATkk"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "mega_list = []\n",
        "for x in soup.find_all('table', summary = 'Table of 2023 Recruitment country of qualification accepted offers data split by specialty and level'):\n",
        "  for y in x.find_all('tr'):\n",
        "    small_list = []\n",
        "    for z in y.find_all('td'):\n",
        "      thing = z.text\n",
        "      small_list.append(thing)\n",
        "    mega_list.append(small_list)"
      ],
      "metadata": {
        "id": "oLOXNZ4oiA3M"
      },
      "execution_count": 68,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "mega_list"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "3aJDkKFEhUMN",
        "outputId": "fa3f0835-93ce-4344-fcc1-ce9269b6c021"
      },
      "execution_count": 69,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "[[],\n",
              " ['ACCS Anaesthetics/Core Anaesthetics CT1', '606', '6', '18', 'N/A\\xa0'],\n",
              " ['ACCS Emergency Medicine ST1/CT1', '289', '12', '58', 'N/A'],\n",
              " ['ACCS Internal Medicine/Internal Medicine CT1', '1004', '84', '583', 'N/A'],\n",
              " ['Acute Internal Medicine ST4', '41', '<5', '53', 'N/A'],\n",
              " ['Allergy ST3', '<5', '0', '<5', 'N/A'],\n",
              " ['Anaesthetics ST4', '500', '10', '67', 'N/A'],\n",
              " ['Audiovestibular Medicine ST3', '<5', '0', '<5', 'N/A'],\n",
              " ['Cardiology ST4', '63', '7', '70', 'N/A'],\n",
              " ['Cardiothoracic Surgery ST1', '8', '<5', '<5', 'N/A'],\n",
              " ['Chemical Pathology ST3', '<5', '0', '7', 'N/A'],\n",
              " ['Clinical Genetics ST3', '15', '<5', '7', 'N/A'],\n",
              " ['Clinical Neurophysiology ST3', '<5', '<5', '<5', 'N/A'],\n",
              " ['Clinical Oncology ST3', '56', '0', '26', 'N/A'],\n",
              " ['Clinical Pharmacology and Therapeutics ST4', '5', '0', '<5', 'N/A'],\n",
              " ['Clinical Radiology ST1', '296', '11', '43', 'N/A'],\n",
              " ['Clinical Radiology ST3', '0', '0', '13', 'N/A'],\n",
              " ['Combined Infection Training ST3', '32', '7', '12', 'N/A'],\n",
              " ['Combined Infection Training ST4', '18', '<5', '<5', 'N/A'],\n",
              " ['Community Sexual and Reproductive Health ST1', '8', '0', '0', 'N/A'],\n",
              " ['Core Psychiatry CT1', '354', '33', '287', 'N/A'],\n",
              " ['Core Surgical Training CT1', '550', '20', '39', 'N/A'],\n",
              " ['Dermatology ST3', '43', '8', '5', 'N/A'],\n",
              " ['Diagnostic Neuropathology ST3', '<5', '0', '0', 'N/A'],\n",
              " ['Emergency Medicine ST3', '5', '<5', '19', 'N/A'],\n",
              " ['Emergency Medicine ST4', '24', '<5', '14', 'N/A'],\n",
              " ['Endocrinology and Diabetes Mellitus ST4', '26', '<5', '71', 'N/A'],\n",
              " ['Gastroenterology ST4', '73', '6', '54', 'N/A'],\n",
              " ['General (Internal) Medicine ST4', '<5', '0', '24', 'N/A'],\n",
              " ['General Practice ST1', '2048', '258', '2303', 'N/A'],\n",
              " ['General Surgery ST3', '82', '11', '69', 'N/A'],\n",
              " ['Genitourinary Medicine ST4', '7', '<5', '<5', 'N/A'],\n",
              " ['Geriatric Medicine ST4', '94', '5', '31', 'N/A'],\n",
              " ['Haematology ST3', '50', '<5', '52', 'N/A'],\n",
              " ['Histopathology ST1', '59', '6', '43', 'N/A'],\n",
              " ['Immunology ST3', '7', '0', '<5', 'N/A'],\n",
              " ['Intensive Care Medicine ST3', '133', '<5', '32', 'N/A'],\n",
              " ['Medical Oncology ST3', '40', '<5', '27', 'N/A'],\n",
              " ['Medical Ophthalmology ST3', '<5', '0', '0', 'N/A'],\n",
              " ['Neurology ST4', '25', '5', '35', 'N/A'],\n",
              " ['Neurosurgery ST1', '18', '<5', '0', 'N/A'],\n",
              " ['Neurosurgery ST2', '<5', '0', '0', 'N/A'],\n",
              " ['Nuclear Medicine ST3', '<5', '0', '<5', 'N/A'],\n",
              " ['Obstetrics and Gynaecology ST1', '226', '19', '61', 'N/A'],\n",
              " ['Obstetrics and Gynaecology ST3', '<5', '0', '87', 'N/A'],\n",
              " ['Occupational Medicine ST3', '<5', '0', '5', 'N/A'],\n",
              " ['Ophthalmology ST1', '84', '8', '<5', 'N/A'],\n",
              " ['Ophthalmology ST3', '<5', '<5', '16', 'N/A'],\n",
              " ['Oral and Maxillo Facial Surgery ST1', '11', '0', '0', 'N/A'],\n",
              " ['Oral and Maxillo Facial Surgery ST3', '11', '<5', '<5', 'N/A'],\n",
              " ['Otolaryngology ST3', '57', '<5', '6', 'N/A'],\n",
              " ['Paediatric and Perinatal Pathology ST3', '<5', '0', '0', 'N/A'],\n",
              " ['Paediatric Cardiology ST4', '6', '<5', '<5', 'N/A'],\n",
              " ['Paediatric Surgery ST3', '12', '0', '5', 'N/A'],\n",
              " ['Paediatrics ST1', '326', '22', '136', 'N/A'],\n",
              " ['Paediatrics ST3', '6', '0', '101', 'N/A'],\n",
              " ['Paediatrics ST4', '7', '<5', '61', 'N/A'],\n",
              " ['Palliative Medicine ST4', '40', '<5', '<5', 'N/A'],\n",
              " ['Plastic Surgery ST3', '48', '<5', '<5', 'N/A'],\n",
              " ['Psychiatry (all specialties) ST4', '253', '27', '220', 'N/A'],\n",
              " ['Public Health Medicine ST1', '82', '<5', '<5', '38'],\n",
              " ['Rehabilitation Medicine ST3', '5', '0', '6', 'N/A'],\n",
              " ['Renal Medicine ST4', '37', '<5', '36', 'N/A'],\n",
              " ['Respiratory Medicine ST4', '78', '<5', '69', 'N/A'],\n",
              " ['Rheumatology ST4', '28', '<5', '29', 'N/A'],\n",
              " ['Sport and Exercise Medicine ST3', '9', '0', '0', 'N/A'],\n",
              " ['Thoracic Surgery ST4', '<5', '0', '<5', 'N/A'],\n",
              " ['Trauma and Orthopaedic Surgery ST3', '116', '5', '39', 'N/A'],\n",
              " ['Urology ST3', '54', '<5', '26', 'N/A'],\n",
              " ['Vascular Surgery ST3', '13', '5', '24', 'N/A']]"
            ]
          },
          "metadata": {},
          "execution_count": 69
        }
      ]
    }
  ]
}