add welsh pharms

Author: cjber

ahah/common/utils.py

@@ -68,6 +68,17 @@ class Config:
         "/d08bc753-c6dc-4dbd-8b37-ef439d3a7428/download"
         "/dispenser_contactdetails_oct2020_notabs.csv",
     }
+    NHS_WALES_URL = (
+        "https://nwssp.nhs.wales/ourservices/"
+        "primary-care-services/primary-care-services-documents/"
+    )
+
+    NHS_WALES_FILES = {
+        "pharmacy": (
+            "pharmacy-practice-dispensing-data-docs"
+            "/dispensing-data-report-november-2021"
+        )
+    }
 
 
 def combine_lsoa(eng, scot, wales):
@@ -246,7 +257,7 @@ def clean_gpp(
 
 
 def clean_pharmacies(
-    england: Path, scotland: Path, postcodes: cudf.DataFrame
+    england: Path, scotland: Path, wales: Path, postcodes: cudf.DataFrame
 ) -> cudf.DataFrame:
     logger.info("Cleaning pharmacies...")
 
@@ -270,7 +281,18 @@ def clean_pharmacies(
         .join(postcodes)
         .pipe(find_partial_pc, postcodes)
     )
-    return epharm.append(spharm).reset_index()
+
+    wpharm = (
+        cudf.from_pandas(pd.read_excel(wales, usecols=["Account Number", "Post Code"]))
+        .rename(columns={"Account Number": "pharmacy", "Post Code": "postcode"})
+        .astype(str)
+        .pipe(fix_postcodes)
+        .set_index("postcode")
+        .join(postcodes)
+        .pipe(find_partial_pc, postcodes)
+    )
+    breakpoint()
+    return epharm.append(spharm).append(wpharm).reset_index()
 
 
 def clean_hospitals(

ahah/create_index.py

@@ -1,20 +1,18 @@
-import matplotlib.colors as colors
-import matplotlib.pyplot as plt
 import numpy as np
 import pandas as pd
 from scipy.stats import norm
 
 from ahah.common.utils import Config, combine_lsoa
 
 
-def exp_default(x, df):
-    return norm.ppf((x - 0.5) / len(df))
-
-
 def exp_trans(x, df):
     return -23 * np.log(1 - (x / len(df)) * (1 - np.exp(-100 / 23)))
 
 
+def exp_default(x, df):
+    return norm.ppf((x - 0.5) / len(df))
+
+
 def read_v3():
     v3 = pd.read_csv("./data/out/weighted_mean_dists.csv")
     v3_secure = pd.read_csv(
@@ -31,20 +29,37 @@ def read_v2():
 
 
 def process(idx, low_dist, env_dist, air_qual, high_dist):
-    idx[low_dist] = idx[low_dist].rank(method="min").astype(int)
-    idx[env_dist] = idx[env_dist].rank(method="min").astype(int)
-    idx[air_qual] = idx[air_qual].rank(method="min").astype(int)
-    idx[high_dist] = idx[high_dist].rank(method="min", ascending=False).astype(int)
-
-    idx[low_dist + env_dist + air_qual + high_dist] = exp_default(
-        idx[low_dist + env_dist + air_qual + high_dist],
-        idx,
+    low_dist_ranked = [f"{asset}_ranked" for asset in low_dist]
+    env_dist_ranked = [f"{asset}_ranked" for asset in env_dist]
+    air_qual_ranked = [f"{asset}_ranked" for asset in air_qual]
+    high_dist_ranked = [f"{asset}_ranked" for asset in high_dist]
+
+    low_dist_expd = [f"{asset}_expd" for asset in low_dist]
+    env_dist_expd = [f"{asset}_expd" for asset in env_dist]
+    air_qual_expd = [f"{asset}_expd" for asset in air_qual]
+    high_dist_expd = [f"{asset}_expd" for asset in high_dist]
+
+    idx[low_dist_ranked] = idx[low_dist].rank(method="min").astype(int)
+    idx[env_dist_ranked] = idx[env_dist].rank(method="min").astype(int)
+    idx[air_qual_ranked] = idx[air_qual].rank(method="min").astype(int)
+    idx[high_dist_ranked] = (
+        idx[high_dist].rank(method="min", ascending=False).astype(int)
+    )
+
+    # higher values os gspassive are better (prop of pc that is gs)
+    idx[env_dist_ranked[1]] = (
+        idx[env_dist[1]].rank(method="min", ascending=False).astype(int)
     )
 
-    idx["r_domain"] = idx[high_dist].mean(axis=1)
-    idx["h_domain"] = idx[low_dist].mean(axis=1)
-    idx["g_domain"] = idx[env_dist].mean(axis=1)
-    idx["e_domain"] = idx[air_qual].mean(axis=1)
+    idx[low_dist_expd] = exp_default(idx[low_dist_ranked], idx)
+    idx[env_dist_expd] = exp_default(idx[env_dist_ranked], idx)
+    idx[air_qual_expd] = exp_default(idx[air_qual_ranked], idx)
+    idx[high_dist_expd] = exp_default(idx[high_dist_ranked], idx)
+
+    idx["h_domain"] = idx[low_dist_expd].mean(axis=1)
+    idx["g_domain"] = idx[env_dist_expd].mean(axis=1)
+    idx["e_domain"] = idx[air_qual_expd].mean(axis=1)
+    idx["r_domain"] = idx[high_dist_expd].mean(axis=1)
 
     idx["r_rank"] = idx["r_domain"].rank(method="min").astype(int)
     idx["h_rank"] = idx["h_domain"].rank(method="min").astype(int)
@@ -57,61 +72,34 @@ def process(idx, low_dist, env_dist, air_qual, high_dist):
     idx["e_exp"] = exp_trans(idx["e_rank"], idx)
 
     idx["ahah"] = idx[["r_exp", "h_exp", "g_exp", "e_exp"]].mean(axis=1)
+    idx["r_ahah"] = idx["ahah"].rank(method="min").astype(int)
+    idx["d_ahah"] = pd.qcut(idx["r_ahah"], 10, labels=False)
     return idx
 
 
-low_dist = ["gpp", "dentists", "pharmacies", "hospitals", "leisure"]
-env_dist = ["greenspace", "gspassive", "bluespace"]
-air_qual = ["no22019", "so22019", "pm102019g"]
-high_dist = ["gambling", "offlicences", "pubs", "tobacconists", "fastfood"]
-v3 = read_v3().dropna()
-v3 = process(v3, low_dist, env_dist, air_qual, high_dist)
-
-low_dist = ["gpp_dist", "ed_dist", "dent_dist", "pharm_dist", "leis_dist"]
-env_dist = ["green_act", "green_pas", "blue_dist"]
-air_qual = ["no2_mean", "pm10_mean", "so2_mean"]
-high_dist = ["gamb_dist", "ffood_dist", "pubs_dist", "off_dist", "tobac_dist"]
-v2 = read_v2()
-v2 = process(v2, low_dist, env_dist, air_qual, high_dist)
-
-lsoa = combine_lsoa(
-    eng=Config.RAW_DATA / "lsoa" / "england_lsoa_2011.shp",
-    scot=Config.RAW_DATA / "lsoa" / "SG_DataZone_Bdry_2011.shp",
-    wales=Config.RAW_DATA / "lsoa" / "lsoa_wales_2011.gpkg",
-)
-
-v3 = lsoa.merge(v3, on="lsoa11", how="outer")
-v2 = lsoa.merge(v2, on="lsoa11", how="outer")
-
-v3.to_file(Config.OUT_DATA / "v3_lsoa.gpkg", driver="GPKG")
-v2.to_file(Config.OUT_DATA / "v2_lsoa.gpkg", driver="GPKG")
-
-ax = plt.figure().subplots(1, 2)
-col = "ahah"
-v3.plot(
-    column=col,
-    legend=True,
-    cmap="RdYlBu_r",
-    norm=colors.TwoSlopeNorm(vcenter=50, vmin=0, vmax=100),
-    ax=ax[0],
-)
-col = "ahah"
-v2.plot(
-    column=col,
-    legend=True,
-    cmap="RdYlBu_r",
-    norm=colors.TwoSlopeNorm(vcenter=50, vmin=0, vmax=100),
-    ax=ax[1],
-)
-plt.show()
-
-test = v3.set_index("lsoa11").join(v2.set_index("lsoa11"), rsuffix="_v2")
-test["diff"] = test["ahah"] - test["ahah_v2"]
-
-test.plot(
-    column="diff",
-    legend=True,
-    cmap="RdYlBu_r",
-    # norm=colors.TwoSlopeNorm(vcenter=50, vmin=0, vmax=100),
-)
-plt.show()
+if __name__ == "__main__":
+    low_dist = ["gpp", "dentists", "pharmacies", "hospitals", "leisure"]
+    env_dist = ["greenspace", "gspassive", "bluespace"]
+    air_qual = ["no22019", "so22019", "pm102019g"]
+    high_dist = ["gambling", "offlicences", "pubs", "tobacconists", "fastfood"]
+    v3 = read_v3().dropna()
+    v3 = process(v3, low_dist, env_dist, air_qual, high_dist)
+
+    low_dist = ["gpp_dist", "ed_dist", "dent_dist", "pharm_dist", "leis_dist"]
+    env_dist = ["green_act", "green_pas", "blue_dist"]
+    air_qual = ["no2_mean", "pm10_mean", "so2_mean"]
+    high_dist = ["gamb_dist", "ffood_dist", "pubs_dist", "off_dist", "tobac_dist"]
+    v2 = read_v2()
+    v2 = process(v2, low_dist, env_dist, air_qual, high_dist)
+
+    lsoa = combine_lsoa(
+        eng=Config.RAW_DATA / "lsoa" / "england_lsoa_2011.shp",
+        scot=Config.RAW_DATA / "lsoa" / "SG_DataZone_Bdry_2011.shp",
+        wales=Config.RAW_DATA / "lsoa" / "lsoa_wales_2011.gpkg",
+    )
+
+    v3 = lsoa.merge(v3, on="lsoa11", how="outer")
+    v2 = lsoa.merge(v2, on="lsoa11", how="outer")
+
+    v3.to_file(Config.OUT_DATA / "v3_lsoa.gpkg", driver="GPKG")
+    v2.to_file(Config.OUT_DATA / "v2_lsoa.gpkg", driver="GPKG")

ahah/get_nhs.py

@@ -1,8 +1,10 @@
+from pathlib import Path
+from zipfile import ZipFile
+
 import requests
+
 from ahah.common.logger import logger
 from ahah.common.utils import Config
-from pathlib import Path
-from zipfile import ZipFile
 
 
 def download_url(url: str, save_path: Path, chunk_size: int = 128):
@@ -46,3 +48,10 @@ def download_url(url: str, save_path: Path, chunk_size: int = 128):
             logger.debug(f"{Config.NHS_SCOT_URL + url} saved to {file}")
         else:
             logger.warning(f"{file} exists: skipping {Config.NHS_SCOT_URL + url}")
+
+    for name, url in Config.NHS_WALES_FILES.items():
+        file = Config.RAW_DATA / "nhs" / "wales" / f"{name}.xls"
+        if not file.exists():
+            download_url(Config.NHS_WALES_URL + url, save_path=file)
+        else:
+            logger.warning(f"{file} exists: skipping {Config.NHS_WALES_URL + url}")

ahah/process_routing.py

@@ -142,6 +142,7 @@ def get_buffers(
     pharmacies: cudf.DataFrame = clean_pharmacies(
         england=Config.RAW_DATA / "nhs" / "edispensary.csv",
         scotland=Config.RAW_DATA / "nhs" / "scotland" / "pharmacies.csv",
+        wales=Config.RAW_DATA / "nhs" / "wales" / "pharmacy.xls",
         postcodes=pcs,
     )
     greenspace: cudf.DataFrame = clean_greenspace_access(

env.yml

@@ -13,6 +13,7 @@ dependencies:
   - pytables
   - pdoc3
   - openpyxl
+  - xlrd
   - isort
   - flake8
   - black