From a72008e99791e2f01af2698eaa24d8e47521461c Mon Sep 17 00:00:00 2001 From: Juan-Mateos Date: Fri, 15 Nov 2019 08:12:06 +0000 Subject: [PATCH] BEIS and IDBR indicators --- data/processed/2019_11_15_ashe_rankings.csv | 73 +++ .../2019_11_15_BRES_2018_industry_salary.csv | 40 ++ .../2019_11_15_IDBR_2018_industry_salary.csv | 41 ++ .../official/2019_11_15_ashe_rankings.csv | 73 +++ notebooks/dev/0-jmg-ashe_sectoral.ipynb | 506 ++++++++++++++++++ notebooks/dev/08_jmg_bres_idbr.ipynb | 197 +++++++ 6 files changed, 930 insertions(+) create mode 100644 data/processed/2019_11_15_ashe_rankings.csv create mode 100644 data/processed/industry/2019_11_15_BRES_2018_industry_salary.csv create mode 100644 data/processed/industry/2019_11_15_IDBR_2018_industry_salary.csv create mode 100644 data/processed/official/2019_11_15_ashe_rankings.csv create mode 100644 notebooks/dev/0-jmg-ashe_sectoral.ipynb create mode 100644 notebooks/dev/08_jmg_bres_idbr.ipynb diff --git a/data/processed/2019_11_15_ashe_rankings.csv b/data/processed/2019_11_15_ashe_rankings.csv new file mode 100644 index 00000000..c7b22680 --- /dev/null +++ b/data/processed/2019_11_15_ashe_rankings.csv @@ -0,0 +1,73 @@ +cluster,weighted_median_salary,ashe_median_salary_rank +construction_construction,29047.11773497602,7.0 +construction_infrastructure,24507.72037005299,5.0 +manufacture_aerospace,38885.13137962982,9.0 +manufacture_armaments,36166.25,9.0 +manufacture_automotive,33185.40257812628,8.0 +manufacture_casting,14890.141031239162,1.0 +manufacture_chemical,29529.909849907974,7.0 +manufacture_electrical,24704.329860951017,5.0 +manufacture_electronics,27963.020673337545,7.0 +manufacture_food,19082.802808406035,2.0 +manufacture_footwear_leather,10061.599568181691,0.0 +manufacture_furniture,19407.556238766745,2.0 +manufacture_instruments,22606.417389874074,3.0 +manufacture_light,, +manufacture_machinery_tools,26561.946611989417,6.0 +manufacture_materials,16160.182897012215,1.0 +manufacture_metal_equipment,27912.46491923181,7.0 +manufacture_oils,27119.957713815813,6.0 +manufacture_paper,29693.060223570992,8.0 +manufacture_pesticides,10642.034129993826,0.0 +manufacture_pharmaceuticals,31718.19215010201,8.0 +manufacture_plastics,23963.98384328097,4.0 +manufacture_rubber,25538.312495708356,5.0 +manufacture_ships,29687.078779243253,8.0 +manufacture_steel_metal,20565.043894998984,3.0 +manufacture_textiles,11251.493828799594,0.0 +primary_fishing,14501.878253214196,1.0 +primary_forestry,16877.363165876715,2.0 +primary_oil_gas,27251.841647576322,6.0 +primary_quarrying_concrete,26663.711268939915,6.0 +services_administrative,20080.105824797047,3.0 +services_agricultural,18297.3859981817,2.0 +services_air_transport,27922.81548333526,7.0 +services_computing,33474.99200755404,8.0 +services_consumer_retail,13300.796370529179,0.0 +services_content,22804.602339470286,4.0 +services_creative,21979.45507545414,3.0 +services_cultural,17905.405135874396,2.0 +services_education_post_primary,26909.041196711394,6.0 +services_education_primary,21512.899916948096,3.0 +services_electronics_machinery,9861.509367922288,0.0 +services_energy,36328.502158014715,9.0 +services_entertainment,14706.62583676456,1.0 +services_environmental,23422.50902496797,4.0 +services_facilities,10325.203804164368,0.0 +services_financial_legal,29134.219944499146,7.0 +services_health,21200.369912533246,3.0 +services_hospitality,12160.95054921591,0.0 +services_house_installation,27488.26353369079,6.0 +services_human_resources,16861.168283564348,1.0 +services_industrial_equipment,24218.37340950466,5.0 +services_insurance_finance,34348.53068425116,9.0 +services_kibs,29702.04901478667,8.0 +services_land_transport,34118.238233130636,9.0 +services_logistics,25278.45182399135,5.0 +services_motor,23576.4123047286,4.0 +services_passenger,22960.879454037622,4.0 +services_printing,23161.974480276935,4.0 +services_professional,25359.55122571418,5.0 +services_public_administration,28071.068957617226,7.0 +services_publishing,25745.065329240577,6.0 +services_r_&_d,35149.112341903376,9.0 +services_real_state,22616.2045848407,4.0 +services_recreation,14684.366104432187,1.0 +services_residential,15441.31049619675,1.0 +services_sea_transport,25640.257641776625,5.0 +services_security,18742.94714783385,2.0 +services_telecommunications,34780.15364849151,9.0 +services_textiles,11590.298934927581,0.0 +services_travelling,17072.784333703825,2.0 +services_utilities,33103.90177720155,8.0 +services_wholesale,21345.074667447287,3.0 diff --git a/data/processed/industry/2019_11_15_BRES_2018_industry_salary.csv b/data/processed/industry/2019_11_15_BRES_2018_industry_salary.csv new file mode 100644 index 00000000..65bd0cb1 --- /dev/null +++ b/data/processed/industry/2019_11_15_BRES_2018_industry_salary.csv @@ -0,0 +1,40 @@ +geo_nm,geo_cd,construction_construction,construction_infrastructure,manufacture_aerospace,manufacture_armaments,manufacture_automotive,manufacture_casting,manufacture_chemical,manufacture_electrical,manufacture_electronics,manufacture_food,manufacture_footwear_leather,manufacture_furniture,manufacture_instruments,manufacture_light,manufacture_machinery_tools,manufacture_materials,manufacture_metal_equipment,manufacture_oils,manufacture_paper,manufacture_pesticides,manufacture_pharmaceuticals,manufacture_plastics,manufacture_rubber,manufacture_ships,manufacture_steel_metal,manufacture_textiles,primary_fishing,primary_forestry,primary_oil_gas,primary_quarrying_concrete,services_administrative,services_agricultural,services_air_transport,services_computing,services_consumer_retail,services_content,services_creative,services_cultural,services_education_post_primary,services_education_primary,services_electronics_machinery,services_energy,services_entertainment,services_environmental,services_facilities,services_financial_legal,services_health,services_hospitality,services_house_installation,services_human_resources,services_industrial_equipment,services_insurance_finance,services_kibs,services_land_transport,services_logistics,services_motor,services_passenger,services_printing,services_professional,services_public_administration,services_publishing,services_r_&_d,services_real_state,services_recreation,services_residential,services_sea_transport,services_security,services_telecommunications,services_textiles,services_travelling,services_utilities,services_wholesale,top_20_salary_share,bottom_20_salary_share +Bedfordshire and Hertfordshire,UKH2,31600,5935,4500,2000,2270,20,1720,1410,4850,6230,160,3380,1870,435,7300,1155,4070,635,1040,255,2050,3700,675,10,855,790,595,325,0,1345,12750,5200,6285,51775,103700,3125,12250,1875,17450,58750,1630,2430,9490,5355,29800,36500,53225,30950,16600,67300,14625,11560,21000,6385,26000,28800,5850,3525,6995,20550,3365,9775,21000,22205,42900,100,10100,7405,1895,3800,14075,36990,0.1455797283366361,0.33420576122894824 +"Berkshire, Buckinghamshire and Oxfordshire",UKJ1,32400,3185,2350,6000,6170,150,3670,2635,6850,8490,135,4785,3650,60,11855,730,2725,570,840,235,700,5525,725,385,1280,870,780,415,40,1150,19600,8470,4050,90350,143075,4850,12250,3900,60550,77200,2225,5450,14100,11475,35800,42800,91250,51850,18900,46700,26400,14300,46200,8555,23000,41050,8550,4275,9725,27830,10000,19300,27600,38550,59500,485,8125,17975,2620,2105,22250,49415,0.18217406104618178,0.30055599682287526 +Cheshire,UKD6,11950,2025,325,250,7600,40,2780,855,2430,4775,20,2280,250,45,5895,245,1090,1540,1410,750,875,2525,230,90,1000,515,425,195,0,875,9650,3305,540,32270,64530,780,2750,1950,7525,25325,425,1545,4565,3570,10000,20250,36525,22550,8400,17350,5090,7480,12700,2055,16000,15400,2075,1335,2395,12760,670,4705,10350,13615,21625,350,2575,2835,2490,1935,14280,16140,0.17830027928736353,0.32091832104939455 +Cornwall and Isles of Scilly,UKK3,9175,930,270,0,215,40,80,110,365,4305,50,1350,200,90,1750,200,890,60,250,0,0,950,365,1150,180,340,1095,130,0,1100,1690,1915,550,2680,36260,290,2810,1550,5625,11900,580,885,3540,1525,3300,2320,19825,27200,3675,5050,1700,810,1875,830,2650,7320,1600,715,1155,5680,420,370,6100,6665,15350,680,705,755,490,400,2400,3795,0.05646856703664353,0.45283360083188284 +Cumbria,UKD1,8325,525,0,250,205,110,335,845,835,4075,45,2215,110,25,2935,90,670,185,1550,70,400,1690,1800,8035,120,485,415,295,0,900,1775,2575,165,3095,34250,300,690,1900,3720,15730,250,635,3890,1245,4575,3340,19015,22375,4900,2420,1815,725,2190,955,5300,8015,2175,375,705,8675,510,75,4125,4155,12550,595,2055,470,590,475,5410,3255,0.10452832654483675,0.3758140750582649 +Derbyshire and Nottinghamshire,UKF1,22300,2725,18500,30,4525,1195,2375,2725,2700,16815,195,11845,1455,145,15305,1435,5985,2100,2090,150,1600,9300,2075,685,3175,4910,300,435,170,3995,13725,3905,2150,32775,126925,490,6500,4650,28150,59925,1250,7695,14100,6145,11950,21285,82400,41600,21100,27500,13050,8315,10125,6150,21000,34100,7600,4150,4950,33315,760,2675,17850,21960,58800,305,5635,2950,3800,1775,15015,18200,0.11936125427967019,0.33347928613254973 +Devon,UKK4,16050,1675,1325,0,585,185,185,1420,1450,4305,215,3330,1325,70,4550,740,3365,230,525,10,1010,1970,405,9225,695,1375,1430,280,0,935,3865,5025,1840,8705,71720,695,5315,1525,16200,27740,960,2390,7425,3050,8600,8470,49925,39150,8800,9575,4280,3700,5400,2310,6000,16425,4800,1180,2520,19755,1425,195,12400,14370,31750,1515,2505,3625,1310,1250,6225,8345,0.09208744437995744,0.38450886356925396 +Dorset and Somerset,UKK2,16850,1615,5500,15,995,160,830,2895,2775,9405,990,4590,1120,70,10090,380,2490,2095,530,130,20,2725,545,2725,640,1210,445,195,170,2000,6980,4765,1160,17300,76800,550,4255,2000,11325,38425,1020,2990,6500,3765,8500,17945,58200,40450,9450,10625,5515,8585,8850,1525,9000,16900,4300,1120,2560,17675,1185,725,17300,18275,39750,695,945,1720,1250,1250,7275,12290,0.1020362089176399,0.35858984934658966 +East Anglia,UKH1,30200,4460,2400,150,3350,325,3065,2490,5925,27795,485,8275,1570,165,15075,1625,7675,1405,3300,975,1750,7400,1850,700,970,1745,1670,370,880,2460,10600,9775,6475,34720,145525,1665,12300,4450,40250,68250,2370,3900,14725,9210,33750,20575,95250,51200,18600,40250,12800,25035,14450,4230,26000,40725,7300,6725,7145,35420,5355,20250,20250,29650,61700,3675,5650,6540,2080,2250,26000,22025,0.1276931709927051,0.3376223970907446 +East Wales,UKL2,15755,3215,6800,400,2250,160,2500,1735,2535,10520,50,5560,1310,20,7160,240,4110,825,1750,370,1100,3450,920,215,2145,855,115,725,50,1065,10655,3845,1890,11605,69965,2940,2900,1850,19200,26500,1115,5230,7975,3360,10150,19895,53325,25350,7200,18250,3600,12800,5650,4200,8000,13775,3225,2645,4680,32515,710,1355,10975,12305,36000,375,3225,3410,1220,1725,11350,5865,0.12148424262961707,0.32838110185664093 +East Yorkshire and Northern Lincolnshire,UKE1,10175,1160,840,100,3710,295,2420,660,345,10270,85,8490,345,125,9435,430,5880,2250,1450,460,1700,5150,910,230,1400,490,6320,95,300,1865,5970,1945,2815,7705,52845,435,1085,840,9100,25900,355,975,3250,3770,5950,7525,31895,16150,6800,17550,3375,1535,5000,1375,13000,11050,3225,1925,2260,16365,335,215,6200,7930,25000,2535,1305,1095,1285,460,4950,5030,0.07806456505801755,0.3498622241322648 +Eastern Scotland,UKM2,28000,2625,475,5,1625,230,1635,1735,7100,11750,150,6345,1085,85,8680,490,4285,2005,1230,435,1800,2175,1630,1680,775,3360,3435,1900,495,1410,13590,6800,4325,37475,114325,805,4300,7000,40350,44850,1410,6475,13925,5070,11900,38240,94650,56300,16250,24150,5550,24900,12375,4400,12000,24525,8150,2225,7575,55545,1975,7825,18300,32250,60000,1165,4775,8825,2400,5000,14300,12325,0.12758499462321118,0.3358011415336256 +Essex,UKH3,25900,3630,1025,800,1650,75,1060,1020,3125,5930,90,4500,2285,215,9395,1075,3215,835,1150,100,25,3050,395,85,1505,930,710,125,0,1595,6275,5180,14350,24120,101975,875,9425,2150,16425,52675,1655,820,7325,8150,16600,15440,58300,32700,18500,18950,8625,14575,14625,2950,15000,28525,6850,3315,4145,19310,2370,3750,16350,14950,45000,1790,8225,1930,1785,1425,14925,17455,0.11214279397859994,0.3318436286213769 +"Gloucestershire, Wiltshire and Bath/Bristol area",UKK1,32150,3700,12600,800,7750,385,1130,3455,6920,10045,255,8250,2735,95,14385,840,7035,1310,2570,35,1750,4350,2425,230,1795,1800,1085,420,50,1800,13150,9560,4850,49725,139275,3320,10325,3600,35675,68225,2145,6650,13925,9775,27000,40950,109650,60500,21050,29150,14300,25380,26800,6550,24000,38850,8500,3895,8675,41325,5470,7100,30500,38530,64250,1300,6550,6850,2300,2700,26200,32745,0.14913976057527867,0.3144046318999444 +Greater Manchester,UKD3,30050,5425,1100,600,3275,300,5520,2700,4825,13045,30,9050,1275,85,14295,745,5675,4100,2950,260,545,10800,1910,75,2510,6925,845,365,40,1130,28950,3930,17275,52000,158125,7640,6500,1650,38250,74700,1750,11965,19700,7290,32750,52200,113100,61480,20650,51300,17850,23735,28050,8300,34000,36750,12000,4875,7010,51075,2490,1125,36500,31065,59000,210,12225,10600,15075,4675,22150,43930,0.1229232200611404,0.3244207168490541 +Hampshire and Isle of Wight,UKJ3,24600,3095,7500,350,1450,110,1405,2700,6815,4145,95,2370,4115,50,12285,600,3460,1865,1325,270,610,3750,1045,4800,910,1060,685,650,90,740,15000,5635,6780,40395,112875,1220,6925,3100,26150,54175,1875,3705,14225,4575,16650,22625,71700,45125,16100,25400,13450,16015,16750,2650,12000,29175,5500,2695,4505,25000,3765,7185,16450,25245,53500,5380,7750,8175,1670,2375,17450,19630,0.14529484008625326,0.3388835750952238 +"Herefordshire, Worcestershire and Warwickshire",UKG1,17725,1815,1600,20,17450,205,1635,3210,3350,11660,150,4725,1510,120,11485,740,5200,1325,475,30,0,4750,1075,180,7780,1370,440,500,0,1285,13250,5300,3585,22125,76325,410,6050,2600,11175,39850,1960,4200,7725,3800,11350,14240,46725,23150,13400,18750,16375,7615,12400,2650,16000,25700,3050,1400,5720,16105,1480,2620,14500,16500,37200,335,1875,2570,925,1675,9125,13420,0.13036583451091222,0.3090281048830272 +Highlands and Islands,UKM6,9795,660,30,0,10,0,25,50,155,2845,40,1695,40,120,2180,60,435,465,30,0,800,385,40,525,240,1265,6600,1220,260,960,2265,1820,1265,3575,27770,285,775,2350,3960,11820,345,1605,3030,1755,2075,2655,23145,19800,4375,2115,1425,905,785,920,2850,6560,2075,130,1200,12015,555,530,3765,8900,14400,1900,495,1125,125,1100,3610,1665,0.06732516423612729,0.4031356287564646 +Inner London - East,UKI4,21825,4070,10,10,435,55,560,545,500,3435,475,1745,275,430,1450,455,290,135,240,0,65,235,70,0,180,2830,1310,140,85,435,27450,1425,4180,107575,111675,11750,22850,4500,39100,57050,2855,3755,27655,3760,51250,97775,103750,72400,10200,40300,3245,30300,68500,7900,2750,9970,19050,3775,35250,53525,22700,4450,39800,28750,57750,935,19750,13750,5380,6950,17325,22435,0.19304970945568964,0.30602229467379716 +Inner London - West,UKI3,21750,2285,15,50,100,10,340,595,375,2840,360,1160,550,1320,4395,220,295,385,80,110,725,65,15,15,110,2370,540,200,1450,450,44900,1655,5610,136750,156650,52500,29300,9500,70750,29575,2175,12150,36300,1255,33700,218425,111700,148625,4700,88000,3145,133000,144000,9905,1750,7800,10450,2060,53000,86300,29925,9800,81750,40075,43250,1625,11475,17100,7925,12500,31120,44480,0.24513626387781084,0.2775859421058107 +Kent,UKJ4,24400,3595,375,0,1795,30,1040,1765,3925,9390,125,3055,1265,110,8915,705,2515,1500,2150,405,430,3750,225,210,680,545,345,275,0,1505,9525,5190,2690,18930,98600,1125,5550,3000,19500,52575,1510,2700,8365,5305,16450,13360,59125,33675,17050,21100,10375,13520,12075,6255,16000,25375,5950,3390,4310,23405,2855,2825,15050,17450,44250,2455,5175,2945,980,2265,9900,14110,0.10421075574241902,0.3438561213800086 +Lancashire,UKD4,16550,8620,13075,0,4650,120,2125,1390,2055,13875,630,9930,845,30,10450,830,3435,1310,2900,55,350,9300,1675,285,1180,3525,1155,135,160,645,7500,4130,615,15825,82025,615,2515,1175,19675,37550,1900,2455,8700,5250,8350,10970,61675,33150,11200,16500,6225,4855,7200,2345,14000,19550,4750,2800,1385,24025,485,700,11050,15605,38200,180,4900,4745,4740,2050,7600,20160,0.10456929657497893,0.33639045864685124 +"Leicestershire, Rutland and Northamptonshire",UKF2,16000,1730,1325,0,4300,330,2365,2275,3720,25805,2460,10265,1850,150,10400,1350,7395,705,4375,440,805,6800,1395,770,2280,5560,210,325,0,3460,15875,4200,6650,24250,95975,195,6450,2350,27750,47550,900,7675,14200,4980,9450,35055,63200,32500,17550,39800,20100,9035,11350,2300,37000,33900,3450,4100,5485,28585,900,4060,14850,22950,40700,100,4380,2475,4790,1500,10385,28110,0.09599136014230353,0.31373194494978374 +Lincolnshire,UKF3,6600,650,1150,75,1500,0,390,770,875,21375,50,2950,60,25,4210,60,2020,95,1620,135,250,1500,820,220,420,710,325,245,60,640,4125,3915,735,6520,39015,220,1820,1975,6430,17725,320,720,4480,3370,4400,4110,25745,18075,6175,12075,2640,1565,2795,1125,5800,11925,1725,1375,1055,7725,530,180,4950,5585,19000,60,970,730,710,350,3350,4600,0.07505077977071642,0.3612765517953661 +Merseyside,UKD7,10600,1870,375,0,7800,115,4775,1610,1275,5595,35,3465,515,55,5855,990,1920,1925,755,275,2050,2700,400,1125,710,845,225,35,300,380,11400,2860,3275,12090,82275,1275,2440,1525,19750,38300,740,1115,10075,4100,9000,25275,79575,30650,11750,17750,4865,7240,7825,2400,18000,13775,9250,2130,3040,33800,955,1785,13125,17590,38200,1880,8450,3850,1350,870,8365,17475,0.0887164822801425,0.32832520782548913 +North Eastern Scotland,UKM5,8425,730,135,0,500,10,585,600,710,2770,15,1345,90,10,6975,40,1545,270,520,10,10,255,410,1070,205,485,6155,400,25020,660,2700,1810,3310,5960,25790,230,1380,725,6250,11330,370,810,2640,1080,3875,4905,26225,12825,4150,5875,3905,1260,3285,630,3000,9150,2125,310,1435,11810,270,1600,4125,10775,13375,1255,1520,980,195,505,21575,3070,0.1377272400661062,0.29616655888481713 +North Yorkshire,UKE2,9125,1100,315,0,1930,150,695,1695,830,12585,80,2610,575,65,5910,325,510,35,675,95,725,1910,160,55,1025,480,745,475,20,1085,4925,3625,465,10650,52980,720,1810,2250,11975,23875,470,1710,5515,1785,8900,8590,26925,29150,5600,9475,4255,5990,8850,3555,8000,9125,2950,1365,3800,14215,285,2760,9400,10700,21500,160,1545,1075,820,875,6585,7495,0.11728346662484973,0.3695180596936909 +Northumberland and Tyne and Wear,UKC2,16450,2260,340,300,13600,90,2790,2005,2830,5905,70,3540,700,25,9235,580,5045,850,1525,10,1950,2775,765,260,1845,965,520,180,50,555,11050,2890,3795,17170,80300,445,2205,1850,23650,31100,950,4970,9525,3260,18000,18005,71625,33200,10350,17500,5320,7070,4850,2800,8000,16600,7700,4100,3375,32500,855,925,12700,14775,36400,680,4145,8650,1060,2400,9000,6345,0.11650901472828848,0.33955212036566784 +Outer London - East and North East,UKI5,18600,3320,40,50,3065,60,455,680,605,4890,150,1650,365,40,2170,985,550,570,470,0,450,680,80,0,445,1600,835,75,10,650,8575,1740,1420,19425,75690,1235,4425,1720,13150,50100,1185,970,9850,4625,28250,9195,49175,27790,12650,12750,4515,3275,10225,3350,11000,17035,11650,2450,2445,21010,910,110,12800,14900,37500,375,12550,2880,2650,975,4615,12095,0.08632782517402313,0.37794560067756533 +Outer London - South,UKI6,12850,1950,135,0,75,60,390,590,495,710,65,915,465,95,1445,585,315,150,225,10,295,650,50,0,140,460,680,65,0,255,7225,1635,620,20675,62075,1440,5400,350,12225,33600,1760,1280,7335,1425,21250,11750,39250,21675,10450,23750,2815,8240,12025,3325,2500,11275,6850,1715,2575,20055,2235,930,11000,13560,29750,75,3950,2080,940,3050,8750,12025,0.12376443491890855,0.3922992439994455 +Outer London - West and North West,UKI7,22450,4565,825,50,330,70,825,545,920,16495,130,2150,520,155,3070,1240,1010,320,215,5,900,1480,45,20,415,885,1000,70,630,985,13750,2005,56600,46125,97800,15375,9800,2825,19600,57675,3190,860,10840,3480,19400,23275,65225,49650,9700,30400,7975,5465,24900,3190,8000,21850,16500,2100,5600,22510,3445,3675,27500,27825,40700,640,12750,5700,4150,3800,9375,30545,0.11445657621334894,0.3235428268961129 +Shropshire and Staffordshire,UKG2,20275,1715,1000,250,7600,900,4150,3150,3050,13315,235,4685,515,165,15300,4900,7030,2150,1050,190,100,5600,1275,115,3815,1635,450,690,0,2255,5775,4235,1785,19820,84075,270,3140,1275,14200,41225,1015,735,13360,4700,9000,15250,62000,21550,18100,26625,14625,5775,9450,2600,29000,26250,2825,2560,2505,23050,535,620,12750,17905,37100,260,1655,4200,1210,1625,9425,22240,0.09176606527764573,0.322000307154506 +South Western Scotland,UKM3,35500,3515,4500,450,1535,140,1780,2450,1810,12390,800,4935,2855,155,9300,660,3845,705,2600,25,1300,3800,645,3155,1130,2485,2145,1025,20,1725,22850,5725,6475,29725,128100,3175,4475,3450,32150,42100,1600,9350,14775,8205,35700,34625,106800,54100,21150,24250,8700,16900,11775,6885,16000,29775,8850,2800,9275,58530,2145,3400,20800,32700,76500,1805,9350,7425,1885,3350,16425,17475,0.10901955598792923,0.35500658792638523 +South Yorkshire,UKE3,12450,2335,125,0,3510,895,860,2175,1230,5665,105,6805,2700,260,11610,520,5945,265,535,40,10,5200,855,20,4980,790,185,120,25,860,12900,1720,830,17000,76875,680,1650,1125,26025,34285,725,1250,6235,4200,8400,16935,58450,22140,8650,18000,8025,6920,8500,4165,19000,15025,4550,1970,3670,22775,795,730,8400,12765,34500,50,3060,5180,1255,675,7600,11450,0.09745591718571806,0.32183524870602687 +"Surrey, East and West Sussex",UKJ2,31900,5890,2450,200,5700,60,1060,2920,9400,6135,165,4115,2705,205,9245,1600,3695,1385,295,110,1525,2550,800,350,885,1250,915,700,155,1270,17300,11265,20440,61775,157100,3750,12375,3300,35600,82200,2275,5590,19275,5575,29250,37100,113100,64750,23500,36500,21200,31150,39000,6460,11000,37300,9000,3925,10365,33305,5700,6750,29200,44645,81500,665,6925,5425,2005,6545,24425,35455,0.14774682135001294,0.34127238778563396 +Tees Valley and Durham,UKC1,15650,1420,1200,0,4350,275,2950,1265,1845,6140,100,2325,240,15,8100,490,5725,1225,750,390,1780,3425,295,255,2230,755,255,25,315,605,4000,2050,1065,6680,57920,50,1330,1475,15525,27575,1040,1710,4835,2590,7400,10645,40525,18500,7500,9875,4605,2840,3395,2325,11000,12950,4825,735,975,26925,205,1520,8625,9875,31100,1315,1860,2035,755,805,8700,4390,0.08762224867539621,0.3350955815419088 +West Midlands,UKG3,27600,2725,2850,250,28300,1540,3825,2235,1675,10310,1500,8770,1990,1125,27875,1420,11150,625,1750,10,0,7000,1315,35,10795,2780,770,360,0,750,17275,2855,7350,43450,142125,1560,7975,2025,50425,81200,1915,5000,19625,8505,23850,54345,117250,37825,22300,52600,32425,16125,16850,10720,24000,41950,10250,2975,7575,46335,750,1420,27450,23325,69500,410,8620,6110,3100,2650,20295,26945,0.12160563425014292,0.30321809693196977 +West Wales and The Valleys,UKL1,28350,2370,4000,0,7350,65,1910,1485,2940,11270,40,5525,1895,40,11005,1190,7675,1950,1800,195,2050,7525,1025,475,2310,2075,620,1000,100,1715,8490,7120,905,9790,106470,850,2825,2950,26300,43600,770,2170,10800,6825,10850,15075,82675,42725,10200,9250,5015,6695,4240,1830,8500,20650,5675,3675,2295,48425,525,1500,13925,14300,54500,1185,3180,4395,780,3250,10050,10025,0.07664265845082396,0.3463576203981419 +West Yorkshire,UKE4,21850,2215,40,10,4025,515,3385,3500,2420,14710,150,13550,2215,150,15215,1380,9325,2700,3100,810,950,7850,1065,25,2805,9445,445,200,60,2745,17350,5330,6260,42650,123625,2785,5300,2600,31650,70300,1090,5350,15775,7110,28000,54015,85400,42350,16800,43600,12725,19175,22050,4755,32000,27800,8100,9650,7250,38050,1185,1025,20350,20170,60500,320,7495,5275,4110,1410,17790,28750,0.11644126275392536,0.32469695012823113 diff --git a/data/processed/industry/2019_11_15_IDBR_2018_industry_salary.csv b/data/processed/industry/2019_11_15_IDBR_2018_industry_salary.csv new file mode 100644 index 00000000..eeadf359 --- /dev/null +++ b/data/processed/industry/2019_11_15_IDBR_2018_industry_salary.csv @@ -0,0 +1,41 @@ +geo_nm,geo_cd,construction_construction,construction_infrastructure,manufacture_aerospace,manufacture_armaments,manufacture_automotive,manufacture_casting,manufacture_chemical,manufacture_electrical,manufacture_electronics,manufacture_food,manufacture_footwear_leather,manufacture_furniture,manufacture_instruments,manufacture_light,manufacture_machinery_tools,manufacture_materials,manufacture_metal_equipment,manufacture_oils,manufacture_paper,manufacture_pesticides,manufacture_pharmaceuticals,manufacture_plastics,manufacture_rubber,manufacture_ships,manufacture_steel_metal,manufacture_textiles,primary_fishing,primary_forestry,primary_oil_gas,primary_quarrying_concrete,services_administrative,services_agricultural,services_air_transport,services_computing,services_consumer_retail,services_content,services_creative,services_cultural,services_education_post_primary,services_education_primary,services_electronics_machinery,services_energy,services_entertainment,services_environmental,services_facilities,services_financial_legal,services_health,services_hospitality,services_house_installation,services_human_resources,services_industrial_equipment,services_insurance_finance,services_kibs,services_land_transport,services_logistics,services_motor,services_passenger,services_printing,services_professional,services_public_administration,services_publishing,services_r_&_d,services_real_state,services_recreation,services_residential,services_sea_transport,services_security,services_telecommunications,services_textiles,services_travelling,services_utilities,services_wholesale,top_20_salary_share,bottom_20_salary_share +Bedfordshire and Hertfordshire,UKH2,5355,1175,110,10,80,0,95,125,305,325,30,680,140,55,960,60,155,40,40,5,35,200,45,10,60,215,60,95,0,90,3680,855,265,9940,7960,1080,2920,80,990,1175,265,160,1030,375,1250,1850,3875,2700,4925,1675,1575,1240,8350,430,2545,3660,565,460,1715,475,455,235,4995,3115,2300,35,435,505,410,305,2580,4175,0.24182040566710836,0.2148099072469677 +"Berkshire, Buckinghamshire and Oxfordshire",UKJ1,5820,1065,90,5,125,10,170,155,435,530,40,795,180,50,1105,55,185,60,35,10,50,220,50,45,105,190,85,195,5,135,5095,1395,460,16855,11045,1440,4100,225,1610,1540,395,250,1495,440,2020,2625,5090,3690,5160,1800,2035,1445,13740,275,2005,4680,755,520,2260,710,755,565,6180,4405,3115,60,515,785,475,435,3915,4385,0.2877274953841516,0.2172651569388447 +Cheshire,UKD6,2000,320,55,0,75,5,115,35,110,300,15,300,40,10,605,25,90,25,25,15,10,80,30,30,25,100,35,40,0,65,2860,595,105,3595,4930,180,1435,100,490,595,125,80,545,190,665,1165,2220,1540,1505,950,700,670,4230,150,1005,1890,205,165,655,330,150,105,2340,1360,1265,30,135,195,205,175,2960,1480,0.2493857493857494,0.24119574119574116 +Cornwall and Isles of Scilly,UKK3,1655,295,35,0,20,0,15,25,40,245,15,225,20,25,260,20,40,10,5,0,0,75,10,80,10,60,400,45,0,75,685,440,35,880,3655,105,625,65,210,280,90,150,350,110,300,340,975,1950,1100,120,300,185,730,65,245,1145,140,85,370,240,95,35,1220,900,840,100,70,80,85,80,540,500,0.11593304401735895,0.36309154784046294 +Cumbria,UKD1,1440,140,10,0,25,0,25,20,40,325,10,230,25,10,405,5,30,10,20,0,0,35,10,15,5,40,75,90,0,75,900,445,25,700,3215,50,520,85,285,410,60,75,385,115,270,370,1035,1680,1120,100,295,435,890,90,390,1075,160,55,260,325,65,25,925,695,705,30,35,60,70,85,1155,405,0.15099223468507333,0.315358067299396 +Derbyshire and Nottinghamshire,UKF1,4090,715,70,5,140,35,160,125,185,495,35,955,110,25,1390,70,235,40,55,10,25,310,70,35,105,320,60,80,5,210,2570,815,155,4595,10190,275,1970,215,1045,1305,230,125,1195,390,1150,1750,4105,3215,4135,715,1485,1015,4160,330,2115,3990,410,390,950,890,270,145,4095,2200,2990,35,275,365,460,255,2560,2765,0.16528173712622066,0.27749135682659065 +Devon,UKK4,3100,495,50,0,50,5,40,50,95,490,25,445,65,25,540,25,90,20,15,0,5,115,20,115,20,130,325,140,0,75,1265,920,80,2120,6610,240,1140,100,555,660,165,100,840,220,755,905,2030,2845,2415,330,680,560,1950,110,575,2420,280,190,710,505,245,70,2570,1510,1820,120,160,150,185,165,1095,1255,0.13266895048271568,0.3232637807536593 +Dorset and Somerset,UKK2,3785,565,150,5,85,10,50,85,155,550,35,620,75,40,885,40,135,40,15,10,10,135,45,75,50,125,200,145,0,140,1735,975,95,3610,7175,325,1490,210,615,765,175,145,845,295,945,1065,2485,2875,3150,465,770,775,3150,125,770,2820,320,290,980,550,295,75,3260,1780,2005,85,175,225,220,225,1610,1830,0.16759729977498122,0.2817734811234269 +East Anglia,UKH1,5880,1140,200,5,185,20,155,155,340,965,70,1005,120,40,1475,80,240,60,65,25,40,230,75,115,85,190,270,155,30,195,3250,1580,495,6450,11950,490,2655,365,1310,1495,300,150,1535,525,2200,1975,4420,4430,5275,1060,1805,1195,6000,310,2990,5220,595,485,1530,1350,585,565,5035,3595,3465,115,355,490,390,375,3255,3035,0.1757992977268527,0.27333210127518015 +East Wales,UKL2,2230,620,115,0,50,0,60,40,90,390,20,370,55,15,585,20,105,25,20,5,10,115,15,20,40,95,35,135,0,90,1830,780,110,2350,5550,440,1140,110,660,610,125,95,810,235,860,1145,2300,2305,1775,370,605,730,2365,255,1000,2110,275,160,700,545,145,95,2120,1250,1885,50,160,170,150,155,1295,1235,0.16311139840551606,0.29045464339581983 +East Yorkshire and Northern Lincolnshire,UKE1,1855,260,45,5,60,15,65,30,40,285,15,370,30,5,645,20,90,20,25,10,15,120,35,40,30,65,370,25,5,85,1000,390,195,1185,4535,100,680,70,370,495,95,95,495,220,445,595,1745,1590,1820,260,480,270,1215,130,1470,1840,180,135,315,460,65,35,1430,1005,1305,70,115,80,110,90,1110,865,0.12740171445462606,0.3054980786284363 +Eastern Scotland,UKM2,3380,415,35,5,55,0,80,50,135,540,20,830,85,40,775,40,175,30,20,10,10,90,35,40,50,210,390,405,20,125,2420,1045,125,5480,10020,290,1860,335,800,1035,200,290,1555,305,1350,2120,3405,4610,3135,570,885,1045,5195,235,905,2845,495,215,1075,645,275,250,3350,2645,3095,80,145,285,255,400,2890,1645,0.20337785769329567,0.3207038273824814 +Essex,UKH3,6140,1555,100,0,90,10,90,100,205,370,30,800,115,60,1055,65,180,25,55,5,10,210,40,50,75,185,150,75,0,130,2865,880,480,6380,8230,470,2150,150,740,1015,315,70,950,445,1385,1400,3625,2870,5840,1015,1295,1220,5470,240,2020,3655,530,435,1200,475,350,120,4100,2040,2055,90,510,320,340,245,2560,2680,0.19597134132017852,0.23073760864458537 +"Gloucestershire, Wiltshire and Bath/Bristol area",UKK1,5890,1040,160,5,155,15,115,125,265,650,45,940,125,50,1340,70,260,25,50,10,25,250,85,40,105,200,95,255,5,210,4175,1545,305,9665,12125,1050,3455,240,1405,1475,325,245,1555,460,1895,2455,4855,4370,5125,1265,1805,1960,9005,260,2460,4730,545,505,2025,795,615,235,5620,3685,3625,50,380,580,345,390,3855,3240,0.223785766065413,0.2526544710246898 +Greater Manchester,UKD3,4510,1665,130,0,120,10,210,150,255,430,30,925,145,35,1345,60,305,85,85,5,25,340,85,15,105,515,80,40,0,110,5095,675,455,7165,13465,850,2770,135,1210,1430,295,135,1685,545,1805,3770,6070,4700,4360,1400,2065,2325,6275,905,5675,4535,515,485,1435,565,360,175,6215,2820,3470,35,505,510,1190,435,3500,8285,0.17501535941019863,0.258283841900471 +Hampshire and Isle of Wight,UKJ3,4975,875,270,10,100,10,90,135,305,345,25,595,100,30,1195,50,195,35,25,5,10,190,55,300,75,175,155,140,10,85,2900,995,300,7610,8980,530,2510,165,945,1180,280,105,1185,335,1625,1735,3740,3425,4740,1035,1320,995,6475,225,1390,3540,420,330,1240,540,355,200,4125,2770,2560,220,335,510,250,300,2940,3825,0.22040793580026752,0.25111457868925546 +"Herefordshire, Worcestershire and Warwickshire",UKG1,3020,565,50,5,180,10,80,95,170,565,30,600,70,35,1240,35,230,20,30,0,5,195,60,35,175,130,55,130,0,90,3365,1050,185,4315,6990,290,2385,110,800,840,165,115,780,290,1675,1110,2865,2175,2705,630,1345,725,4640,185,3350,3065,295,260,1380,645,235,175,3590,2205,1760,35,255,265,235,195,2665,2035,0.1906049822064057,0.2402135231316726 +Highlands and Islands,UKM6,1545,135,10,0,0,0,15,5,30,235,0,210,10,20,330,10,10,15,0,0,0,20,5,55,0,90,1385,345,15,125,640,380,85,505,2975,65,505,225,245,480,40,170,420,90,250,305,810,1965,995,90,170,120,535,60,290,940,215,25,240,450,90,45,715,830,925,205,35,90,35,100,1025,270,0.11247311827956989,0.3877419354838709 +Inner London - East,UKI4,4150,1120,20,5,35,10,60,65,135,385,100,495,75,140,395,45,105,55,30,0,20,65,20,10,25,575,195,35,10,50,7590,370,325,15950,11140,4660,7040,170,1730,1000,265,330,2150,235,1920,3455,4875,5700,2340,1885,1015,1395,14445,270,735,1770,735,495,6030,370,1290,290,7155,2880,3340,80,615,590,1125,670,2425,3975,0.2690530744093453,0.2353604026593547 +Inner London - West,UKI3,2330,650,25,0,80,0,95,70,145,400,95,365,120,285,240,45,105,80,20,5,50,35,10,20,30,500,80,45,55,60,9285,335,515,12775,12760,5315,6280,195,1760,695,330,1085,1905,120,2310,15735,5550,7235,1215,5785,710,5335,19565,315,350,1150,445,330,7490,600,1570,480,14430,3905,2325,170,670,610,980,1085,2420,5175,0.2560909907213409,0.2290332235857528 +Kent,UKJ4,5050,1160,35,0,80,0,70,90,150,435,20,570,95,65,900,55,195,40,50,10,15,125,25,40,55,130,125,95,5,130,2830,885,295,5210,8320,535,2285,155,835,1050,220,95,985,380,1375,1380,3295,3045,4740,895,1280,1080,5205,315,1510,3290,420,390,1295,525,420,135,3515,2400,2430,125,350,275,245,255,2335,2515,0.18853810067199187,0.25681501204513757 +Lancashire,UKD4,3060,685,70,0,120,10,80,80,105,465,25,715,70,10,925,55,145,25,50,5,15,205,55,20,45,205,140,40,10,95,2020,705,135,2885,7780,140,1180,90,585,925,160,75,865,330,840,1255,3105,2690,2585,435,995,750,2560,290,1650,2950,295,280,520,440,145,70,2550,1535,2105,30,195,210,395,210,1695,1985,0.1514786795048143,0.29650962861072905 +"Leicestershire, Rutland and Northamptonshire",UKF2,3905,1210,50,0,190,5,120,115,210,525,80,780,80,45,1385,45,225,35,85,15,15,275,80,50,95,805,50,85,0,120,4255,900,440,5175,8385,270,2115,130,890,1085,240,135,995,340,990,1630,3530,2650,3605,915,1870,1720,5370,475,4245,3865,355,490,950,610,240,125,3845,3030,2385,30,240,280,695,255,2155,2900,0.1852176966292135,0.24912219101123595 +Lincolnshire,UKF3,1715,245,45,5,45,0,30,30,40,420,10,250,10,15,470,20,65,15,20,5,5,80,35,20,20,65,60,50,5,60,745,565,80,1095,3735,75,590,65,325,480,65,50,380,180,415,435,1280,1310,1430,290,450,275,1240,95,1470,1545,175,120,350,355,105,25,1335,810,1005,10,95,75,130,80,760,860,0.1304045841291891,0.2882444868900851 +Merseyside,UKD7,2180,715,40,0,55,10,125,70,75,180,20,370,55,15,645,35,115,40,35,5,20,105,25,30,40,120,35,10,0,55,2365,355,185,2435,6545,200,1195,95,690,735,115,50,995,250,790,1560,3020,2365,2200,710,855,630,2635,335,1980,1840,275,165,590,285,140,85,2120,1360,2150,90,210,200,170,140,1545,1950,0.1562288912477082,0.29769371803531797 +North Eastern Scotland,UKM5,1095,85,15,0,5,0,35,25,50,190,5,160,10,0,510,0,75,15,0,0,0,20,20,45,25,45,760,100,155,55,980,340,110,960,2045,50,790,80,195,300,50,135,335,70,400,450,755,980,865,260,390,125,1680,35,300,905,130,30,235,175,50,65,850,760,725,75,55,80,35,55,4070,445,0.28952648475120385,0.2568218298555377 +North Yorkshire,UKE2,1850,270,20,0,55,0,60,55,75,455,15,455,40,15,415,30,45,10,15,5,10,80,15,10,30,80,150,100,0,95,1080,775,60,1995,5190,165,1050,145,430,525,110,75,550,165,550,825,1655,2330,1550,350,685,495,2500,150,625,1710,225,165,580,530,120,85,2240,1345,1340,30,70,110,175,135,1090,1085,0.16732059233008478,0.30932793317301605 +Northern Ireland,UKN0,5600,375,230,0,105,0,90,75,80,935,30,760,85,25,1080,65,270,30,35,10,25,205,40,30,40,195,330,60,0,350,635,1190,150,2260,10600,295,830,220,645,2605,205,485,2210,400,1095,1985,3590,3365,3285,495,1020,965,1470,150,1645,3610,380,210,705,605,145,90,3455,1385,2900,55,145,180,235,195,1310,1720,0.10441281138790034,0.3291103202846975 +Northumberland and Tyne and Wear,UKC2,2005,310,25,5,65,0,70,70,90,230,20,380,50,15,735,45,120,15,20,0,15,125,30,35,40,120,160,120,10,70,1585,420,100,2260,6850,190,1140,110,595,655,125,80,895,210,1080,1105,2750,2800,2210,375,740,535,1895,140,560,1900,300,210,540,340,155,80,2105,1295,2050,50,150,210,180,195,1695,1170,0.15018081259306532,0.3402467560093597 +Outer London - East and North East,UKI5,6100,1715,15,0,30,5,45,45,90,365,40,370,70,25,510,55,115,35,25,0,10,85,20,10,30,275,90,30,0,60,3335,340,290,8925,7325,820,2230,105,895,890,165,40,1045,345,1470,1395,4125,2970,4495,850,1040,720,5905,225,1530,2475,620,295,1265,310,265,65,3480,1720,2390,60,505,320,530,265,1630,2630,0.2224981375713931,0.23503849019120934 +Outer London - South,UKI6,3625,1015,25,0,15,0,40,50,85,135,25,260,60,30,325,30,80,20,10,0,20,40,20,5,10,125,50,35,0,40,2720,420,120,8300,5520,820,2155,50,710,640,160,35,805,160,1120,1255,3105,2395,3230,840,720,750,6270,155,405,1735,540,265,1350,175,330,75,2845,1635,1770,30,300,265,285,295,1800,2405,0.27225251516780585,0.22671069810306427 +Outer London - West and North West,UKI7,6470,1915,40,5,55,5,85,70,165,525,45,475,95,60,510,55,140,40,15,0,35,95,25,15,35,285,130,35,15,80,5020,465,775,15185,10140,2685,3750,95,1265,1050,310,50,1330,290,1560,2645,5315,4025,3605,1600,1460,1085,10995,235,1650,3210,1145,370,2840,225,595,210,8255,2830,2445,90,505,510,815,635,2550,5260,0.25706226407267596,0.21217903513502304 +Shropshire and Staffordshire,UKG2,3385,630,40,5,130,30,135,90,160,595,30,670,60,25,1180,115,225,35,45,5,5,240,50,20,105,155,60,125,0,140,1870,1010,155,3315,7925,165,1360,115,705,935,160,80,810,360,975,1135,3060,2335,3240,545,1315,675,3150,215,2190,3410,285,280,640,620,175,70,2840,2325,1965,40,160,270,235,185,1940,2050,0.15622304133009174,0.2771547329621206 +South Western Scotland,UKM3,3545,650,105,5,65,10,85,65,135,505,35,620,85,40,1050,40,185,25,45,0,20,130,45,50,45,210,205,255,10,155,2320,850,230,4030,11150,525,1760,280,740,1235,210,210,1770,365,1655,1955,4175,4455,3550,570,1400,985,3515,380,1360,3075,490,265,1030,1315,210,115,3500,2635,3220,85,215,330,280,295,3085,1955,0.16143721539517186,0.329923273657289 +South Yorkshire,UKE3,2525,540,15,0,105,25,45,65,95,180,15,445,70,30,920,30,155,10,15,0,0,155,40,10,130,95,50,35,5,95,1560,325,125,2125,6045,225,1010,75,590,655,105,60,705,295,690,885,2415,1965,2240,410,1260,485,1795,240,1575,2210,240,160,560,465,140,65,1895,1205,1520,15,130,210,205,135,1265,1405,0.14027000329272307,0.2867961804412249 +"Surrey, East and West Sussex",UKJ2,7475,1575,140,5,150,5,135,170,420,610,50,1060,195,100,1145,90,255,70,30,10,60,200,60,60,100,285,255,260,15,140,5830,1955,565,14515,14275,2025,5185,220,1700,1650,505,200,1930,500,2320,3065,6185,5285,6935,1990,1660,2220,14515,405,1315,5185,795,635,3410,750,990,315,7925,5290,4090,105,540,600,500,650,4245,5475,0.24407884020198728,0.24029972308193515 +Tees Valley and Durham,UKC1,1930,270,20,0,45,10,65,30,85,185,15,310,35,10,695,15,125,25,15,5,5,130,20,25,35,90,60,40,10,75,1250,340,70,1330,5135,70,895,90,460,595,75,85,710,215,590,710,2070,1755,1780,290,580,320,1465,105,800,1775,280,115,370,490,65,70,1255,1195,1565,40,140,135,85,120,2520,775,0.16527590847913864,0.31211305518169585 +West Midlands,UKG3,4260,790,40,10,265,55,135,125,190,1680,80,850,125,170,2150,90,480,25,60,0,20,425,130,15,385,335,90,30,0,90,3550,455,360,6415,12865,370,2055,140,1205,1345,280,105,1635,575,1845,2775,5585,4075,4020,1290,3055,1195,4650,380,3450,4690,505,405,995,470,220,125,4575,2395,3600,30,440,440,600,345,2805,3565,0.16073339169341502,0.28435949810329736 +West Wales and The Valleys,UKL1,4155,690,175,0,110,10,65,70,125,610,25,585,75,30,1010,50,160,30,30,5,20,240,35,50,65,155,245,200,0,215,1905,1215,105,2295,9850,290,1290,230,700,1165,155,175,1320,415,930,1170,3385,4385,2945,355,800,685,1720,210,1230,3460,565,225,620,870,175,125,2230,1845,3240,100,175,220,195,230,1625,2605,0.11165303519171606,0.34163727770691077 +West Yorkshire,UKE4,3775,660,30,5,115,20,155,145,185,365,30,1020,110,45,1320,65,265,65,55,5,15,280,90,10,115,460,60,55,0,170,2720,645,240,5415,11240,380,2010,155,1000,1160,250,120,1375,440,1320,2515,4500,3820,3910,1135,2095,1225,4740,240,3105,4150,405,535,1010,465,265,105,4355,2195,3075,25,345,315,655,350,2085,2840,0.1634208298052498,0.29020604007902906 diff --git a/data/processed/official/2019_11_15_ashe_rankings.csv b/data/processed/official/2019_11_15_ashe_rankings.csv new file mode 100644 index 00000000..c7b22680 --- /dev/null +++ b/data/processed/official/2019_11_15_ashe_rankings.csv @@ -0,0 +1,73 @@ +cluster,weighted_median_salary,ashe_median_salary_rank +construction_construction,29047.11773497602,7.0 +construction_infrastructure,24507.72037005299,5.0 +manufacture_aerospace,38885.13137962982,9.0 +manufacture_armaments,36166.25,9.0 +manufacture_automotive,33185.40257812628,8.0 +manufacture_casting,14890.141031239162,1.0 +manufacture_chemical,29529.909849907974,7.0 +manufacture_electrical,24704.329860951017,5.0 +manufacture_electronics,27963.020673337545,7.0 +manufacture_food,19082.802808406035,2.0 +manufacture_footwear_leather,10061.599568181691,0.0 +manufacture_furniture,19407.556238766745,2.0 +manufacture_instruments,22606.417389874074,3.0 +manufacture_light,, +manufacture_machinery_tools,26561.946611989417,6.0 +manufacture_materials,16160.182897012215,1.0 +manufacture_metal_equipment,27912.46491923181,7.0 +manufacture_oils,27119.957713815813,6.0 +manufacture_paper,29693.060223570992,8.0 +manufacture_pesticides,10642.034129993826,0.0 +manufacture_pharmaceuticals,31718.19215010201,8.0 +manufacture_plastics,23963.98384328097,4.0 +manufacture_rubber,25538.312495708356,5.0 +manufacture_ships,29687.078779243253,8.0 +manufacture_steel_metal,20565.043894998984,3.0 +manufacture_textiles,11251.493828799594,0.0 +primary_fishing,14501.878253214196,1.0 +primary_forestry,16877.363165876715,2.0 +primary_oil_gas,27251.841647576322,6.0 +primary_quarrying_concrete,26663.711268939915,6.0 +services_administrative,20080.105824797047,3.0 +services_agricultural,18297.3859981817,2.0 +services_air_transport,27922.81548333526,7.0 +services_computing,33474.99200755404,8.0 +services_consumer_retail,13300.796370529179,0.0 +services_content,22804.602339470286,4.0 +services_creative,21979.45507545414,3.0 +services_cultural,17905.405135874396,2.0 +services_education_post_primary,26909.041196711394,6.0 +services_education_primary,21512.899916948096,3.0 +services_electronics_machinery,9861.509367922288,0.0 +services_energy,36328.502158014715,9.0 +services_entertainment,14706.62583676456,1.0 +services_environmental,23422.50902496797,4.0 +services_facilities,10325.203804164368,0.0 +services_financial_legal,29134.219944499146,7.0 +services_health,21200.369912533246,3.0 +services_hospitality,12160.95054921591,0.0 +services_house_installation,27488.26353369079,6.0 +services_human_resources,16861.168283564348,1.0 +services_industrial_equipment,24218.37340950466,5.0 +services_insurance_finance,34348.53068425116,9.0 +services_kibs,29702.04901478667,8.0 +services_land_transport,34118.238233130636,9.0 +services_logistics,25278.45182399135,5.0 +services_motor,23576.4123047286,4.0 +services_passenger,22960.879454037622,4.0 +services_printing,23161.974480276935,4.0 +services_professional,25359.55122571418,5.0 +services_public_administration,28071.068957617226,7.0 +services_publishing,25745.065329240577,6.0 +services_r_&_d,35149.112341903376,9.0 +services_real_state,22616.2045848407,4.0 +services_recreation,14684.366104432187,1.0 +services_residential,15441.31049619675,1.0 +services_sea_transport,25640.257641776625,5.0 +services_security,18742.94714783385,2.0 +services_telecommunications,34780.15364849151,9.0 +services_textiles,11590.298934927581,0.0 +services_travelling,17072.784333703825,2.0 +services_utilities,33103.90177720155,8.0 +services_wholesale,21345.074667447287,3.0 diff --git a/notebooks/dev/0-jmg-ashe_sectoral.ipynb b/notebooks/dev/0-jmg-ashe_sectoral.ipynb new file mode 100644 index 00000000..c14690bd --- /dev/null +++ b/notebooks/dev/0-jmg-ashe_sectoral.ipynb @@ -0,0 +1,506 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Read the ASHE data\n", + "\n", + "ASHE is the Annual Survey of Hours and Earnings. It contains information about salaries in various regions and sectors of the UK. Here we want to calculate median salaries in Nesta sectors which we can then use to benchmark industries.\n", + "\n", + "\n", + "**Tasks**\n", + "\n", + "* Scrape the data\n", + "* Parse it\n", + "* Merge with out segments at the finest level of granularity possible\n", + "* Create weighted median taking into account distribution of employment in the segment" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Preamble" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "%run ../notebook_preamble.ipy" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "from zipfile import ZipFile\n", + "from io import BytesIO" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "def get_ashe_data(path,ons_path):\n", + " '''\n", + " Function to collect the ASHE data from the ONS website.\n", + " \n", + " Arguments:\n", + " path (str) is the path for the file we are interested in\n", + " ons_path (str) is the parent for all ashe files\n", + " \n", + " This will return a doanloaded and parsed file\n", + " \n", + " '''\n", + " \n", + " file = requests.get(ons_path+path)\n", + " \n", + " #Create a zipfile with its content\n", + " z = ZipFile(BytesIO(file.content))\n", + " \n", + " #Extract names\n", + " names = z.namelist()\n", + " \n", + " #Select the names (they will meantion hourly gross but not the confidence intervals)\n", + " \n", + " my_name = [x for x in names if (all(names in x for names in ['Annual','Gross'])) & ('CV' not in x)]\n", + " \n", + " print(my_name)\n", + "\n", + " #if len(my_name)>1:\n", + " # print('Too many options')\n", + " # break\n", + " \n", + " #Read into excel\n", + " infile = pd.read_excel(BytesIO(z.open(my_name[0]).read()),sheet_name=1,skiprows=4,\n", + " na_values=['x','..',':'])\n", + " \n", + " #Drop missing values in the matching code or median (these don't interest us)\n", + " infile.dropna(axis=0,subset=['Code'],inplace=True)\n", + " \n", + " infile['Code'] = [x.strip() for x in infile['Code']]\n", + " \n", + " #container.append(infile.reset_index(drop=True))\n", + " \n", + " return(infile.reset_index(drop=True))" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "#Processing files\n", + "def add_zeros(container):\n", + " '''\n", + " This adds pre-zeroes to codes in categories A and B\n", + " \n", + " Args:\n", + " Container (df) is one of the dataframes we have created before\n", + " \n", + " '''\n", + " \n", + " new_cont = container.copy()\n", + " \n", + " for pid,row in new_cont.iterrows():\n", + " \n", + " if row['Code']=='C':\n", + " break\n", + " else:\n", + " if row['Code'] not in ['A','B']:\n", + " \n", + " #print(row['Code'])\n", + " new_cont.loc[pid,'Code']='0'+row['Code']\n", + " \n", + " return(new_cont)\n", + " \n", + "\n", + "def year_ashe_lookups(ashe_table):\n", + " '''\n", + " \n", + " Takes an ashe table and outputs a list of code lookups depending on the level of resolution at which they are available\n", + " \n", + " Args:\n", + " ashe_table: an ashe table as above\n", + " \n", + " returns three dicts with code - salary lookups with decreasing levels of resolution\n", + " \n", + " '''\n", + " \n", + " #Containers\n", + " ashe_4 = {}\n", + " ashe_3 = {}\n", + " ashe_2 = {}\n", + "\n", + " #In each row it gets the length of a code (sic4,3 etc) and assigns the median salary to the right dict.\n", + " #We we will use this later to assign the median to \n", + " \n", + " for pid, row in ashe_table.iterrows():\n", + "\n", + " code = row['Code'].strip()\n", + " med_sal = row['Median']\n", + "\n", + " if len(code)==4:\n", + " ashe_4[code]= med_sal\n", + "\n", + " elif len(code)==3:\n", + " ashe_3[code] = med_sal\n", + "\n", + " elif len(code)==2:\n", + " ashe_2[code] = med_sal\n", + "\n", + " else:\n", + " pass\n", + " \n", + " return([ashe_4,ashe_3,ashe_2])\n", + " \n", + " " + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "def map_salaries(lookup,four_digit, ashe):\n", + " '''\n", + " \n", + " Assigns each 4-digit sic code a median according to ASHE at its finest level of resolution\n", + " \n", + " Args:\n", + " lookup (df) a lookup with the 4 digit sic code we want to query against our ashe lookups\n", + " four_digit (str) the name of the variable with the four digits\n", + " ashe_lookups (list of dicts) the list of ashe code-median key-value pairs to query\n", + " \n", + " Returns\n", + " a table with four digit sics, names and salaries.\n", + " \n", + " \n", + " '''\n", + " \n", + " cont = []\n", + "\n", + " #Is loo\n", + " for sic in lookup[four_digit]:\n", + " \n", + " if sic in ashe[0].keys():\n", + " #cont.append({sic:ashe_lookups[0][sic]})\n", + " cont.append([sic,ashe[0][sic]])\n", + " \n", + " elif sic[:-1] in ashe[1].keys():\n", + " \n", + " #cont.append({sic:ashe_lookups[1][sic[:-1]]})\n", + " cont.append([sic,ashe[1][sic[:-1]]])\n", + " \n", + " elif sic[:-2] in ashe[2].keys():\n", + " #cont.append({sic:ashe_lookups[2][sic[:-2]]})\n", + " cont.append([sic,ashe[2][sic[:-2]]])\n", + " \n", + " else:\n", + " #cont.append({sic:np.nan})\n", + " cont.append([sic,np.nan])\n", + " \n", + " return(pd.DataFrame(cont,columns=['sic_4','median_salary_thGBP']).set_index('sic_4'))\n", + " \n", + " \n", + " " + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## 1. Collect data" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "standard_path = 'https://www.ons.gov.uk/file?uri=/employmentandlabourmarket/peopleinwork/earningsandworkinghours/datasets/industry4digitsic2007ashetable16/'\n", + "\n", + "#Ashe paths\n", + "ashe_paths = ['2018provisional/table162018provisional.zip', '2017revised/table162017revised.zip',\n", + " '2016revised/table162016revised.zip','2015/table162015revised.zip']" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "ashes = [get_ashe_data(p,standard_path) for p in ashe_paths]" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "We have lost a few codes with zero at the beginning" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "ashes[0].head()['Code']" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "new_containers = [add_zeros(x) for x in ashes]" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Cluster lookup" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "cl = pd.read_csv('../../data/aux/sic_4_industry_segment_lookup.csv',dtype={'sic_4':str})" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## 2. Processing\n", + "\n", + "We need to assign salaries to their more detailed code. If a salary is available at the 4-digit then we are not interested in the salary at the 3 digit" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Create 2 digit and four digit lookups from Ashe" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "all_ashe_lookups = [year_ashe_lookups(cont) for cont in new_containers]" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "all_salaries = pd.concat([map_salaries(cl,'sic_4',tab) for tab in all_ashe_lookups],axis=1)\n", + "\n", + "all_salaries.columns = [2018,2017,2016,2015]" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "all_salaries.head()" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "#### Create weighted medians\n", + "\n", + "We are not going to be working with sic codes, but our own segments. This requires creating weighted medians of salaries across SIC codes. We use levels of employment to create the weights.\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "#Melt the salaries file from above\n", + "salaries_long = all_salaries.reset_index(drop=False).melt(id_vars=['sic_4'],var_name='year',value_name='median_salary')" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "#We read for the four years\n", + "bres_data = pd.concat([pd.read_csv(f'../../data/processed/official/nomis_BRES_{y}_TYPE450.csv',dtype={'SIC4':str}) for y in [2015,2016,2017,2018]],axis=0)\n", + "\n", + "bres_data.head()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "#Group them by year to get the total level of employment by SIC4\n", + "sic_yearly_long = bres_data.groupby(['year','SIC4'])['value'].sum().reset_index(drop=False)\n", + "\n", + "sic_yearly_long.rename(columns={'value':'employment'},inplace=True)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "salary_empl_merge = pd.merge(salaries_long,sic_yearly_long,left_on=['sic_4','year'],right_on=['SIC4','year'])" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "segment_merged = pd.merge(cl[['sic_4','cluster']],salary_empl_merge,left_on='sic_4',right_on='sic_4')" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "segment_merged.head()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "#Weighted salary: takes all the sics in a segment and applies a weight based on their importance in the segment\n", + "weighted_sal = segment_merged.groupby(\n", + " ['cluster','year']).apply(lambda x: np.sum(x['median_salary']*x['employment'])/np.sum(x['employment'])).reset_index(drop=False)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "ashe_out = weighted_sal.rename(columns={0:'weighted_median_salary'})\n", + "\n", + "ashe_out.pivot_table(index='cluster',columns='year',values='weighted_median_salary').corr()" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "**Remove some outliers**" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "for pid,row in ashe_out.iterrows():\n", + " \n", + " if row['weighted_median_salary']<1000:\n", + " \n", + " ashe_out.loc[pid,'weighted_median_salary'] = np.nan\n", + " " + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "**Calculate averages for all years**" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "ashe_out_grouped = pd.DataFrame(ashe_out.groupby(['cluster'])['weighted_median_salary'].mean())" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "ashe_out_grouped['ashe_median_salary_rank'] = pd.qcut(ashe_out_grouped['weighted_median_salary'],np.arange(0,1.1,0.1),labels=False)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "ashe_out_grouped.sort_values('ashe_median_salary_rank',ascending=False).tail()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "ashe_out_grouped.to_csv(f'../../data/processed/official/{today_str}_ashe_rankings.csv')" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + } + ], + "metadata": { + "language_info": { + "name": "python", + "pygments_lexer": "ipython3" + } + }, + "nbformat": 4, + "nbformat_minor": 2 +} diff --git a/notebooks/dev/08_jmg_bres_idbr.ipynb b/notebooks/dev/08_jmg_bres_idbr.ipynb new file mode 100644 index 00000000..b4a7aaa1 --- /dev/null +++ b/notebooks/dev/08_jmg_bres_idbr.ipynb @@ -0,0 +1,197 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Official data indicators\n", + "\n", + "Here we create indicators of industrial activity in NUTS2 areas based on BRES and NOMIS data.\n", + "\n", + "We have already collected the data by running `make data` in the terminal. This has stored the BRES and IDBR data in the `data/external/` folder, and processed it into Nesta segments (a shorter number of industrial categories) in `data/processed`\n", + "\n", + "Here we create a clean table for the most recent year, and an indicator of the share of employment working in high median salary occupations according to the ashe data we calculated in `0-jmg-ashe_sectoral`.\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Preamble" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "%run ../notebook_preamble.ipy" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "#Need to put this in utils\n", + "def make_dirs(name,dirs = ['raw','processed']):\n", + " '''\n", + " Utility that creates directories to save the data\n", + " \n", + " '''\n", + " \n", + " for d in dirs:\n", + " if name not in os.listdir(f'../../data/{d}'):\n", + " os.mkdir(f'../../data/{d}/{name}')" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "make_dirs('industry')" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Read data" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### metadata (ASHE)\n", + "\n", + "This is a lookup indicating the position in the salary distribution of various industries based on the analysis in the `ashe` notebook" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "#Read ashe and turn it into a lookup\n", + "ashe = pd.read_csv('../../data/processed/official/2019_11_15_ashe_rankings.csv')\n", + "\n", + "ashe_lookup = ashe.set_index('cluster')['ashe_median_salary_rank'].to_dict()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "#bres\n", + "bres_2018 = pd.read_csv('../../data/processed/official/nomis_BRES_2018_TYPE450.csv',dtype={'SIC4':str})\n", + "\n", + "bres_2018['sal'] = bres_2018['cluster_name'].map(ashe_lookup)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "def create_local_industry_dataset(path,salary_lookup,cluster_name,save=True):\n", + " '''\n", + " This creates a long dataset with industry activity per NUTS area and extra variables with share of activity in top two deciles\n", + " of salary, and bottom two deciles of salary\n", + " \n", + " Arguments:\n", + " path (str) path to a tidy dataframe with the industrial activity information (could be employment or establishments)\n", + " salary_lookup (dict) a lookup between industry segments and position in the salary distribution\n", + " cluster_name (str) name of the cluster variable in the industry df\n", + " \n", + " '''\n", + " #Read the data\n", + " industry = pd.read_csv(path,dtype={'SIC4':str})\n", + " \n", + " #Label with salary info\n", + " industry['median_salary_decile'] = industry[cluster_name].map(ashe_lookup)\n", + " \n", + " #Create wide dataset with industry activity per geography\n", + " industry_long = industry.groupby(\n", + " ['geo_nm','geo_cd',cluster_name])['value'].sum().reset_index(drop=False).pivot_table(\n", + " index=['geo_nm','geo_cd'],columns=cluster_name,values='value')\n", + " \n", + " #Share of activity in top and bottom of salary distribution\n", + " salary_long = industry.groupby(\n", + " ['geo_nm','geo_cd','median_salary_decile'])['value'].sum().reset_index(drop=False).pivot_table(\n", + " index=['geo_nm','geo_cd'],columns='median_salary_decile',values='value')\n", + " \n", + " #Top of distro\n", + " high_salary = salary_long.apply(lambda x: x/x.sum(),axis=1)[[8,9]].sum(axis=1)\n", + " \n", + " #Bottom of distro\n", + " low_salary = salary_long.apply(lambda x: x/x.sum(),axis=1)[[0,1]].sum(axis=1)\n", + " \n", + " salary_stats = pd.concat([high_salary,low_salary],axis=1)\n", + " \n", + " #Names\n", + " salary_stats.columns = ['top_20_salary_share','bottom_20_salary_share']\n", + " \n", + " #Concatenate\n", + " combined = pd.concat([industry_long,salary_stats],axis=1)\n", + " \n", + " if save==True:\n", + " \n", + " #Take the informative bit of the name\n", + " name = '_'.join(path.split('_')[1:3])\n", + " \n", + " combined.to_csv(f'../../data/processed/industry/{today_str}_{name}_industry_salary.csv')\n", + " \n", + " \n", + " #Return everything\n", + " return(pd.concat([industry_long,salary_stats],axis=1))\n", + " \n", + " #return(salary_long)\n", + " " + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "bres_nuts = create_local_industry_dataset('../../data/processed/official/nomis_BRES_2018_TYPE450.csv',ashe_lookup,'cluster_name')\n", + "\n", + "idbr_nuts = create_local_industry_dataset('../../data/processed/official/nomis_IDBR_2018_TYPE450.csv',ashe_lookup,'cluster_name')" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "bres_nuts.head()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + } + ], + "metadata": { + "language_info": { + "name": "python", + "pygments_lexer": "ipython3" + } + }, + "nbformat": 4, + "nbformat_minor": 2 +}