Multilinguality (openmaptiles#279)

Improve multilinguality: names in 57 languages, name:latin, name:nonlatin, name_int. Fixes openmaptiles#211 openmaptiles#252 openmaptiles#80.

See openmaptiles#279 for more info.

Makefile

@@ -83,7 +83,7 @@ psql-pg-stat-reset:
 
 forced-clean-sql:
 	docker-compose run --rm import-osm /usr/src/app/psql.sh -c "DROP SCHEMA IF EXISTS public CASCADE ; CREATE SCHEMA IF NOT EXISTS public; "
-	docker-compose run --rm import-osm /usr/src/app/psql.sh -c "CREATE EXTENSION hstore; CREATE EXTENSION postgis; CREATE EXTENSION pg_stat_statements;"
+	docker-compose run --rm import-osm /usr/src/app/psql.sh -c "CREATE EXTENSION hstore; CREATE EXTENSION postgis; CREATE EXTENSION unaccent; CREATE EXTENSION fuzzystrmatch; CREATE EXTENSION osml10n; CREATE EXTENSION pg_stat_statements;"
 	docker-compose run --rm import-osm /usr/src/app/psql.sh -c "GRANT ALL ON SCHEMA public TO public;COMMENT ON SCHEMA public IS 'standard public schema';"
 
 pgclimb-list-views:

README.md

@@ -134,7 +134,7 @@ docker-compose run generate-vectortiles
 
 ## License
 
-All code in this repository is under the [BSD license](./LICENSE.md) and the cartography decisions encoded in the schema and SQL are licensed under [CC-BY](./LICENSE.md). 
+All code in this repository is under the [BSD license](./LICENSE.md) and the cartography decisions encoded in the schema and SQL are licensed under [CC-BY](./LICENSE.md).
 
 Products or services using maps derived from OpenMapTiles schema need to visibly credit "OpenMapTiles.org" or reference "OpenMapTiles" with a link to http://openmaptiles.org/. Exceptions to attribution requirement can be granted on request.
 

docker-compose.yml

@@ -4,7 +4,7 @@ volumes:
   cache:
 services:
   postgres:
-    image: "openmaptiles/postgis:2.5"
+    image: "openmaptiles/postgis:2.8"
     volumes:
     - pgdata:/var/lib/postgresql/data
     networks:
@@ -68,14 +68,14 @@ services:
      - ./build:/mapping
      - cache:/cache
   import-sql:
-    image: "openmaptiles/import-sql:0.1"
+    image: "openmaptiles/import-sql:0.5"
     env_file: .env
     networks:
     - postgres_conn
     volumes:
      - ./build:/sql
   openmaptiles-tools:
-    image: "openmaptiles/openmaptiles-tools:0.5"
+    image: "openmaptiles/openmaptiles-tools:0.6"
     env_file: .env
     networks:
     - postgres_conn

layers/mountain_peak/layer.sql

@@ -3,13 +3,14 @@
 -- etldoc:     style="rounded,filled", label="layer_mountain_peak | <z7_> z7+" ] ;
 
 CREATE OR REPLACE FUNCTION layer_mountain_peak(bbox geometry, zoom_level integer, pixel_width numeric)
-RETURNS TABLE(osm_id bigint, geometry geometry, name text, name_en text, name_de text, ele int, ele_ft int, "rank" int) AS $$
+RETURNS TABLE(osm_id bigint, geometry geometry, name text, name_en text, name_de text, tags hstore, ele int, ele_ft int, "rank" int) AS $$
    -- etldoc: osm_peak_point -> layer_mountain_peak:z7_
-   SELECT osm_id, geometry, name, name_en, name_de, ele::int, ele_ft::int, rank::int
+   SELECT osm_id, geometry, name, name_en, name_de, tags, ele::int, ele_ft::int, rank::int
    FROM (
      SELECT osm_id, geometry, name,
      COALESCE(NULLIF(name_en, ''), name) AS name_en,
      COALESCE(NULLIF(name_de, ''), name, name_en) AS name_de,
+     tags,
      substring(ele from E'^(-?\\d+)(\\D|$)')::int AS ele,
      round(substring(ele from E'^(-?\\d+)(\\D|$)')::int*3.2808399)::int AS ele_ft,
        row_number() OVER (

layers/mountain_peak/mapping.yaml

@@ -17,6 +17,8 @@ tables:
     - name: name_de
       key: name:de
       type: string
+    - name: tags
+      type: hstore_tags
     - name: ele
       key: ele
       type: string

layers/mountain_peak/mountain_peak.yaml

@@ -14,8 +14,9 @@ layer:
   datasource:
     geometry_field: geometry
     srid: 900913
-    query: (SELECT osm_id, geometry, name, name_en, name_de, ele, ele_ft, rank FROM layer_mountain_peak(!bbox!, z(!scale_denominator!), !pixel_width!)) AS t
+    query: (SELECT osm_id, geometry, name, name_en, name_de, {name_languages}, ele, ele_ft, rank FROM layer_mountain_peak(!bbox!, z(!scale_denominator!), !pixel_width!)) AS t
 schema:
+  - ./name.sql
   - ./layer.sql
 datasources:
   - type: imposm3

layers/mountain_peak/name.sql

@@ -0,0 +1,4 @@
+DO $$
+BEGIN
+  update osm_peak_point SET tags = slice_language_tags(tags) || get_basic_names(tags, geometry);
+END $$;

layers/place/city.sql

@@ -4,10 +4,11 @@
 
 -- etldoc: osm_city_point -> layer_city:z2_14
 CREATE OR REPLACE FUNCTION layer_city(bbox geometry, zoom_level int, pixel_width numeric)
-RETURNS TABLE(osm_id bigint, geometry geometry, name text, name_en text, name_de text, place city_place, "rank" int, capital int) AS $$
+RETURNS TABLE(osm_id bigint, geometry geometry, name text, name_en text, name_de text, tags hstore, place city_place, "rank" int, capital int) AS $$
     SELECT osm_id, geometry, name,
     COALESCE(NULLIF(name_en, ''), name) AS name_en,
     COALESCE(NULLIF(name_de, ''), name, name_en) AS name_de,
+    tags,
     place, "rank", normalize_capital_level(capital) AS capital
     FROM osm_city_point
     WHERE geometry && bbox
@@ -18,13 +19,15 @@ RETURNS TABLE(osm_id bigint, geometry geometry, name text, name_en text, name_de
     SELECT osm_id, geometry, name,
         COALESCE(NULLIF(name_en, ''), name) AS name_en,
         COALESCE(NULLIF(name_de, ''), name, name_en) AS name_de,
+        tags,
         place,
         COALESCE("rank", gridrank + 10),
         normalize_capital_level(capital) AS capital
     FROM (
       SELECT osm_id, geometry, name,
       COALESCE(NULLIF(name_en, ''), name) AS name_en,
       COALESCE(NULLIF(name_de, ''), name, name_en) AS name_de,
+      tags,
       place, "rank", capital,
       row_number() OVER (
         PARTITION BY LabelGrid(geometry, 128 * pixel_width)

layers/place/layer.sql

@@ -3,13 +3,14 @@
 -- etldoc:     label="layer_place | <z0_3> z0-3|<z4_7> z4-7|<z8_11> z8-11| <z12_14> z12-z14+" ] ;
 
 CREATE OR REPLACE FUNCTION layer_place(bbox geometry, zoom_level int, pixel_width numeric)
-RETURNS TABLE(osm_id bigint, geometry geometry, name text, name_en text, name_de text, class text, "rank" int, capital INT) AS $$
+RETURNS TABLE(osm_id bigint, geometry geometry, name text, name_en text, name_de text, tags hstore, class text, "rank" int, capital INT) AS $$
 
     -- etldoc: osm_continent_point -> layer_place:z0_3
     SELECT
         osm_id, geometry, name,
         COALESCE(NULLIF(name_en, ''), name) AS name_en,
         COALESCE(NULLIF(name_de, ''), name, name_en) AS name_de,
+        tags,
         'continent' AS class, 1 AS "rank", NULL::int AS capital
     FROM osm_continent_point
     WHERE geometry && bbox AND zoom_level < 4
@@ -23,6 +24,7 @@ RETURNS TABLE(osm_id bigint, geometry geometry, name text, name_en text, name_de
         osm_id, geometry, name,
         COALESCE(NULLIF(name_en, ''), name) AS name_en,
         COALESCE(NULLIF(name_de, ''), name, name_en) AS name_de,
+        tags,
         'country' AS class, "rank", NULL::int AS capital
     FROM osm_country_point
     WHERE geometry && bbox AND "rank" <= zoom_level + 1 AND name <> ''
@@ -36,6 +38,7 @@ RETURNS TABLE(osm_id bigint, geometry geometry, name text, name_en text, name_de
         osm_id, geometry, name,
         COALESCE(NULLIF(name_en, ''), name) AS name_en,
         COALESCE(NULLIF(name_de, ''), name, name_en) AS name_de,
+        tags,
         'state' AS class, "rank", NULL::int AS capital
     FROM osm_state_point
     WHERE geometry && bbox AND
@@ -51,6 +54,7 @@ RETURNS TABLE(osm_id bigint, geometry geometry, name text, name_en text, name_de
         osm_id, geometry, name,
         COALESCE(NULLIF(name_en, ''), name) AS name_en,
         COALESCE(NULLIF(name_de, ''), name, name_en) AS name_de,
+        tags,
         'island' AS class, 7 AS "rank", NULL::int AS capital
     FROM osm_island_point
     WHERE zoom_level >= 12
@@ -63,6 +67,7 @@ RETURNS TABLE(osm_id bigint, geometry geometry, name text, name_en text, name_de
         osm_id, geometry, name,
         COALESCE(NULLIF(name_en, ''), name) AS name_en,
         COALESCE(NULLIF(name_de, ''), name, name_en) AS name_de,
+        tags,
         'island' AS class, island_rank(area) AS "rank", NULL::int AS capital
     FROM osm_island_polygon
     WHERE geometry && bbox AND
@@ -77,6 +82,7 @@ RETURNS TABLE(osm_id bigint, geometry geometry, name text, name_en text, name_de
     -- etldoc: layer_city          -> layer_place:z12_14
     SELECT
         osm_id, geometry, name, name_en, name_de,
+        tags,
         place::text AS class, "rank", capital
     FROM layer_city(bbox, zoom_level, pixel_width)
     ORDER BY "rank" ASC

layers/place/mapping.yaml

@@ -6,38 +6,10 @@ name_en_field: &name_en
   name: name_en
   key: name:en
   type: string
-name_fr_field: &name_fr
-  name: name_fr
-  key: name:fr
-  type: string
 name_de_field: &name_de
   name: name_de
   key: name:de
   type: string
-name_es_field: &name_es
-  name: name_es
-  key: name:es
-  type: string
-name_pt_field: &name_pt
-  name: name_pt
-  key: name:pt
-  type: string
-name_ru_field: &name_ru
-  name: name_ru
-  key: name:ru
-  type: string
-name_zh_field: &name_zh
-  name: name_zh
-  key: name:zh
-  type: string
-name_ar_field: &name_ar
-  name: name_ar
-  key: name:ar
-  type: string
-name_ja_field: &name_ja
-  name: name_ja
-  key: name:ja
-  type: string
 rank_field: &rank
   name: rank
   key: rank
@@ -56,6 +28,8 @@ tables:
     - *name
     - *name_en
     - *name_de
+    - name: tags
+      type: hstore_tags
     filters:
       require:
         name: ["__any__"]
@@ -74,14 +48,9 @@ tables:
     - *name
     - *name_en
     - *name_de
-    - *name_fr
-    - *name_es
-    - *name_pt
-    - *name_ru
-    - *name_zh
-    - *name_ar
-    - *name_ja
     - *rank
+    - name: tags
+      type: hstore_tags
     filters:
       require:
         name: ["__any__"]
@@ -104,13 +73,8 @@ tables:
     - *name
     - *name_en
     - *name_de
-    - *name_fr
-    - *name_es
-    - *name_pt
-    - *name_ru
-    - *name_zh
-    - *name_ar
-    - *name_ja
+    - name: tags
+      type: hstore_tags
     - *rank
     filters:
       require:
@@ -130,13 +94,8 @@ tables:
     - *name
     - *name_en
     - *name_de
-    - *name_fr
-    - *name_es
-    - *name_pt
-    - *name_ru
-    - *name_zh
-    - *name_ar
-    - *name_ja
+    - name: tags
+      type: hstore_tags
     - *rank
     filters:
       require:
@@ -156,13 +115,8 @@ tables:
     - *name
     - *name_en
     - *name_de
-    - *name_fr
-    - *name_es
-    - *name_pt
-    - *name_ru
-    - *name_zh
-    - *name_ar
-    - *name_ja
+    - name: tags
+      type: hstore_tags
     - name: is_in_country
       key: is_in:country
       type: string
@@ -191,13 +145,8 @@ tables:
     - *name
     - *name_en
     - *name_de
-    - *name_fr
-    - *name_es
-    - *name_pt
-    - *name_ru
-    - *name_zh
-    - *name_ar
-    - *name_ja
+    - name: tags
+      type: hstore_tags
     - name: place
       key: place
       type: string

layers/place/name.sql

@@ -0,0 +1,9 @@
+DO $$
+BEGIN
+  update osm_continent_point SET tags = slice_language_tags(tags) || get_basic_names(tags, geometry);
+  update osm_country_point SET tags = slice_language_tags(tags) || get_basic_names(tags, geometry);
+  update osm_island_polygon SET tags = slice_language_tags(tags) || get_basic_names(tags, geometry);
+  update osm_island_point SET tags = slice_language_tags(tags) || get_basic_names(tags, geometry);
+  update osm_state_point SET tags = slice_language_tags(tags) || get_basic_names(tags, geometry);
+  update osm_city_point SET tags = slice_language_tags(tags) || get_basic_names(tags, geometry);
+END $$;

layers/place/place.yaml

@@ -53,8 +53,9 @@ layer:
   buffer_size: 256
   datasource:
     geometry_field: geometry
-    query: (SELECT geometry, name, name_en, name_de, class, rank, capital FROM layer_place(!bbox!, z(!scale_denominator!), !pixel_width!)) AS t
+    query: (SELECT geometry, name, name_en, name_de, {name_languages}, class, rank, capital FROM layer_place(!bbox!, z(!scale_denominator!), !pixel_width!)) AS t
 schema:
+  - ./name.sql
   - ./types.sql
   - ./capital.sql
   - ./city.sql

layers/poi/layer.sql

@@ -3,10 +3,11 @@
 -- etldoc:     label="layer_poi | <z14_> z14+" ] ;
 
 CREATE OR REPLACE FUNCTION layer_poi(bbox geometry, zoom_level integer, pixel_width numeric)
-RETURNS TABLE(osm_id bigint, geometry geometry, name text, name_en text, name_de text, class text, subclass text, "rank" int) AS $$
+RETURNS TABLE(osm_id bigint, geometry geometry, name text, name_en text, name_de text, tags hstore, class text, subclass text, "rank" int) AS $$
     SELECT osm_id, geometry, NULLIF(name, '') AS name,
     COALESCE(NULLIF(name_en, ''), name) AS name_en,
     COALESCE(NULLIF(name_de, ''), name, name_en) AS name_de,
+    tags,
     poi_class(subclass, mapping_key) AS class, subclass,
         row_number() OVER (
             PARTITION BY LabelGrid(geometry, 100 * pixel_width)

layers/poi/mapping.yaml

@@ -315,6 +315,8 @@ tables:
     - name: name_de
       key: name:de
       type: string
+    - name: tags
+      type: hstore_tags
     - name: subclass
       type: mapping_value
     - name: mapping_key
@@ -354,6 +356,8 @@ tables:
     - name: name_de
       key: name:de
       type: string
+    - name: tags
+      type: hstore_tags
     - name: subclass
       type: mapping_value
     - name: mapping_key

layers/poi/name.sql

@@ -0,0 +1,5 @@
+DO $$
+BEGIN
+  update osm_poi_point SET tags = slice_language_tags(tags) || get_basic_names(tags, geometry);
+  update osm_poi_polygon SET tags = slice_language_tags(tags) || get_basic_names(tags, geometry);
+END $$;

layers/poi/poi.yaml

@@ -34,9 +34,10 @@ layer:
   datasource:
     geometry_field: geometry
     srid: 900913
-    query: (SELECT geometry, name, name_en, name_de, class, subclass, rank FROM layer_poi(!bbox!, z(!scale_denominator!), !pixel_width!)) AS t
+    query: (SELECT geometry, name, name_en, name_de, {name_languages}, class, subclass, rank FROM layer_poi(!bbox!, z(!scale_denominator!), !pixel_width!)) AS t
 schema:
   - ./poi_update.sql
+  - ./name.sql
   - ./poi_polygon_update.sql
   - ./class.sql
   - ./layer.sql

layers/transportation_name/layer.sql

@@ -3,11 +3,12 @@
 -- etldoc:     label="layer_transportation_name | <z6> z6 | <z7> z7 | <z8> z8 |<z9> z9 |<z10> z10 |<z11> z11 |<z12> z12|<z13> z13|<z14_> z14+" ] ;
 
 CREATE OR REPLACE FUNCTION layer_transportation_name(bbox geometry, zoom_level integer)
-RETURNS TABLE(osm_id bigint, geometry geometry, name text, name_en text, name_de text, ref text, ref_length int, network text, class text) AS $$
+RETURNS TABLE(osm_id bigint, geometry geometry, name text, name_en text, name_de text, tags hstore, ref text, ref_length int, network text, class text) AS $$
     SELECT osm_id, geometry,
       NULLIF(name, '') AS name,
       COALESCE(NULLIF(name_en, ''), name) AS name_en,
       COALESCE(NULLIF(name_de, ''), name, name_en) AS name_de,
+      tags,
       NULLIF(ref, ''), NULLIF(LENGTH(ref), 0) AS ref_length,
       --TODO: The road network of the road is not yet implemented
       case