Import additional names from Wikidata

osmontrouge · Nov 16, 2017 · 741be6d · 741be6d
1 parent d3a5985
commit 741be6d
Show file tree

Hide file tree

Showing 18 changed files with 50 additions and 20 deletions.
diff --git a/.gitignore b/.gitignore
@@ -11,5 +11,8 @@ quickstart.log
 # imput / output data
 data/*
 
+# input wikidata
+wikidata/*
+
 # generated source files
 build/*
diff --git a/Makefile b/Makefile
@@ -126,6 +126,9 @@ list:
 download-geofabrik-list:
 	docker-compose run --rm import-osm  ./download-geofabrik-list.sh
 
+download-wikidata:
+	mkdir -p wikidata && docker-compose run --rm --entrypoint /usr/src/app/download-gz.sh import-wikidata
+
 start-tileserver:
 	@echo " "
 	@echo "***********************************************************"

diff --git a/docker-compose.yml b/docker-compose.yml
@@ -68,14 +68,22 @@ services:
      - ./build:/mapping
      - cache:/cache
   import-sql:
-    image: "openmaptiles/import-sql:0.6"
+    image: "openmaptiles/import-sql:0.7"
     env_file: .env
     networks:
     - postgres_conn
     volumes:
      - ./build:/sql
+  import-wikidata:
+    image: "openmaptiles/import-wikidata:0.1"
+    env_file: .env
+    command: import-wikidata
+    networks:
+    - postgres_conn
+    volumes:
+     - ./wikidata:/import
   openmaptiles-tools:
-    image: "openmaptiles/openmaptiles-tools:0.7"
+    image: "openmaptiles/openmaptiles-tools:0.8"
     env_file: .env
     networks:
     - postgres_conn

diff --git a/layers/aerodrome_label/update.sql b/layers/aerodrome_label/update.sql
@@ -9,7 +9,7 @@ BEGIN
   WHERE ST_GeometryType(geometry) <> 'ST_Point';
 
   UPDATE osm_aerodrome_label_point
-  SET tags = delete_empty_keys(tags) || get_basic_names(tags, geometry)
+  SET tags = update_tags(tags, geometry)
   WHERE COALESCE(tags->'name:latin', tags->'name:nonlatin', tags->'name_int') IS NULL;
 END;
 $$ LANGUAGE plpgsql;

diff --git a/layers/mountain_peak/update_peak_point.sql b/layers/mountain_peak/update_peak_point.sql
@@ -5,7 +5,7 @@ DROP TRIGGER IF EXISTS trigger_refresh ON mountain_peak_point.updates;
 CREATE OR REPLACE FUNCTION update_osm_peak_point() RETURNS VOID AS $$
 BEGIN
   UPDATE osm_peak_point
-  SET tags = delete_empty_keys(tags) || get_basic_names(tags, geometry)
+  SET tags = update_tags(tags, geometry)
   WHERE COALESCE(tags->'name:latin', tags->'name:nonlatin', tags->'name_int') IS NULL;
 
 END;

diff --git a/layers/place/update_city_point.sql b/layers/place/update_city_point.sql
@@ -41,7 +41,7 @@ BEGIN
   WHERE osm.osm_id = ne.osm_id;
 
   UPDATE osm_city_point
-  SET tags = delete_empty_keys(tags) || get_basic_names(tags, geometry)
+  SET tags = update_tags(tags, geometry)
   WHERE COALESCE(tags->'name:latin', tags->'name:nonlatin', tags->'name_int') IS NULL;
 
 END;

diff --git a/layers/place/update_continent_point.sql b/layers/place/update_continent_point.sql
@@ -5,7 +5,7 @@ DROP TRIGGER IF EXISTS trigger_refresh ON place_continent_point.updates;
 CREATE OR REPLACE FUNCTION update_osm_continent_point() RETURNS VOID AS $$
 BEGIN
   UPDATE osm_continent_point
-  SET tags = delete_empty_keys(tags) || get_basic_names(tags, geometry)
+  SET tags = update_tags(tags, geometry)
   WHERE COALESCE(tags->'name:latin', tags->'name:nonlatin', tags->'name_int') IS NULL;
 
 END;

diff --git a/layers/place/update_country_point.sql b/layers/place/update_country_point.sql
@@ -37,7 +37,7 @@ BEGIN
   WHERE "rank" = 0;
 
   UPDATE osm_country_point
-  SET tags = delete_empty_keys(tags) || get_basic_names(tags, geometry)
+  SET tags = update_tags(tags, geometry)
   WHERE COALESCE(tags->'name:latin', tags->'name:nonlatin', tags->'name_int') IS NULL;
 
 END;

diff --git a/layers/place/update_island_point.sql b/layers/place/update_island_point.sql
@@ -5,7 +5,7 @@ DROP TRIGGER IF EXISTS trigger_refresh ON place_island_point.updates;
 CREATE OR REPLACE FUNCTION update_osm_island_point() RETURNS VOID AS $$
 BEGIN
   UPDATE osm_island_point
-  SET tags = delete_empty_keys(tags) || get_basic_names(tags, geometry)
+  SET tags = update_tags(tags, geometry)
   WHERE COALESCE(tags->'name:latin', tags->'name:nonlatin', tags->'name_int') IS NULL;
 
 END;

diff --git a/layers/place/update_island_polygon.sql b/layers/place/update_island_polygon.sql
@@ -7,7 +7,7 @@ BEGIN
   UPDATE osm_island_polygon  SET geometry=ST_PointOnSurface(geometry) WHERE ST_GeometryType(geometry) <> 'ST_Point';
 
   UPDATE osm_island_polygon
-  SET tags = delete_empty_keys(tags) || get_basic_names(tags, geometry)
+  SET tags = update_tags(tags, geometry)
   WHERE COALESCE(tags->'name:latin', tags->'name:nonlatin', tags->'name_int') IS NULL;
 
   ANALYZE osm_island_polygon;

diff --git a/layers/place/update_state_point.sql b/layers/place/update_state_point.sql
@@ -33,7 +33,7 @@ BEGIN
   DELETE FROM osm_state_point WHERE "rank" IS NULL;
 
   UPDATE osm_state_point
-  SET tags = delete_empty_keys(tags) || get_basic_names(tags, geometry)
+  SET tags = update_tags(tags, geometry)
   WHERE COALESCE(tags->'name:latin', tags->'name:nonlatin', tags->'name_int') IS NULL;
 
 END;

diff --git a/layers/poi/update_poi_point.sql b/layers/poi/update_poi_point.sql
@@ -13,7 +13,7 @@ BEGIN
     WHERE funicular = 'yes' and subclass='station';
 
   UPDATE osm_poi_point
-  SET tags = delete_empty_keys(tags) || get_basic_names(tags, geometry)
+  SET tags = update_tags(tags, geometry)
   WHERE COALESCE(tags->'name:latin', tags->'name:nonlatin', tags->'name_int') IS NULL;
 
 END;

diff --git a/layers/poi/update_poi_polygon.sql b/layers/poi/update_poi_polygon.sql
@@ -22,7 +22,7 @@ BEGIN
     WHERE funicular = 'yes' and subclass='station';
 
   UPDATE osm_poi_polygon
-  SET tags = delete_empty_keys(tags) || get_basic_names(tags, geometry)
+  SET tags = update_tags(tags, geometry)
   WHERE COALESCE(tags->'name:latin', tags->'name:nonlatin', tags->'name_int') IS NULL;
 
   ANALYZE osm_poi_polygon;

diff --git a/layers/water_name/update_marine_point.sql b/layers/water_name/update_marine_point.sql
@@ -22,7 +22,7 @@ BEGIN
   WHERE osm.osm_id = ne.osm_id;
 
   UPDATE osm_marine_point
-  SET tags = delete_empty_keys(tags) || get_basic_names(tags, geometry)
+  SET tags = update_tags(tags, geometry)
   WHERE COALESCE(tags->'name:latin', tags->'name:nonlatin', tags->'name_int') IS NULL;
 
 END;

diff --git a/layers/water_name/update_water_lakeline.sql b/layers/water_name/update_water_lakeline.sql
@@ -9,7 +9,7 @@ CREATE MATERIALIZED VIEW osm_water_lakeline AS (
 	SELECT wp.osm_id,
 		ll.wkb_geometry AS geometry,
 		name, name_en, name_de,
-		delete_empty_keys(tags) || get_basic_names(tags, ll.wkb_geometry) AS tags,
+		update_tags(tags, ll.wkb_geometry) AS tags,
 		ST_Area(wp.geometry) AS area
     FROM osm_water_polygon AS wp
     INNER JOIN lake_centerline ll ON wp.osm_id = ll.osm_id

diff --git a/layers/water_name/update_water_point.sql b/layers/water_name/update_water_point.sql
@@ -9,7 +9,7 @@ CREATE MATERIALIZED VIEW osm_water_point AS (
     SELECT
         wp.osm_id, ST_PointOnSurface(wp.geometry) AS geometry,
         wp.name, wp.name_en, wp.name_de,
-        delete_empty_keys(wp.tags) || get_basic_names(wp.tags, ST_PointOnSurface(wp.geometry)) AS tags,
+        update_tags(wp.tags, ST_PointOnSurface(wp.geometry)) AS tags,
         ST_Area(wp.geometry) AS area
     FROM osm_water_polygon AS wp
     LEFT JOIN lake_centerline ll ON wp.osm_id = ll.osm_id

diff --git a/layers/waterway/update_waterway_linestring.sql b/layers/waterway/update_waterway_linestring.sql
@@ -3,10 +3,17 @@ DROP TRIGGER IF EXISTS trigger_refresh ON osm_waterway_linestring;
 
 DO $$
 BEGIN
-  update osm_waterway_linestring SET tags = delete_empty_keys(tags) || get_basic_names(tags, geometry);
-  update osm_waterway_linestring_gen1 SET tags = delete_empty_keys(tags) || get_basic_names(tags, geometry);
-  update osm_waterway_linestring_gen2 SET tags = delete_empty_keys(tags) || get_basic_names(tags, geometry);
-  update osm_waterway_linestring_gen3 SET tags = delete_empty_keys(tags) || get_basic_names(tags, geometry);
+  update osm_waterway_linestring
+      SET tags = update_tags(tags, geometry);
+
+  update osm_waterway_linestring_gen1
+      SET tags = update_tags(tags, geometry);
+
+  update osm_waterway_linestring_gen2
+      SET tags = update_tags(tags, geometry);
+
+  update osm_waterway_linestring_gen3
+      SET tags = update_tags(tags, geometry);
 END $$;
 
 
@@ -17,7 +24,7 @@ CREATE OR REPLACE FUNCTION waterway_linestring.refresh() RETURNS trigger AS
   $BODY$
   BEGIN
 --     RAISE NOTICE 'Refresh waterway_linestring %', NEW.osm_id;
-    NEW.tags = delete_empty_keys(NEW.tags) || get_basic_names(NEW.tags, NEW.geometry);
+    NEW.tags = update_tags(NEW.tags, NEW.geometry);
     RETURN NEW;
   END;
   $BODY$

diff --git a/quickstart.sh b/quickstart.sh
@@ -129,6 +129,7 @@ echo "====> : Making directories - if they don't exist ( ./build ./data ./pgdata
 mkdir -p pgdata
 mkdir -p build
 mkdir -p data
+mkdir -p wikidata
 
 echo " "
 echo "-------------------------------------------------------------------------------------"
@@ -231,6 +232,14 @@ echo "      : The OpenstreetMap data license: https://www.openstreetmap.org/copy
 echo "      : Thank you OpenStreetMap Contributors ! "
 docker-compose run --rm import-osm
 
+echo " "
+echo "-------------------------------------------------------------------------------------"
+echo "====> : Start importing Wikidata: ./wikidata/latest-all.json.gz -> PostgreSQL"
+echo "      : Source code: https://github.com/openmaptiles/import-wikidata "
+echo "      : The Wikidata license: https://www.wikidata.org/wiki/Wikidata:Database_download/en#License "
+echo "      : Thank you Wikidata Contributors ! "
+docker-compose run --rm import-wikidata
+
 echo " "
 echo "-------------------------------------------------------------------------------------"
 echo "====> : Start SQL postprocessing:  ./build/tileset.sql -> PostgreSQL "