Skip to content

Commit d01d0c1

Browse files
lonvialonvia
committed
Merge pull request osm-search#425 from lonvia/disable-aux-tables
make tables for external data (Tiger and aux) configurable
2 parents be268f9 + a86563c commit d01d0c1

10 files changed

+125
-79
lines changed

‎lib/Geocode.php

+62-53
Original file line numberDiff line numberDiff line change
@@ -431,56 +431,62 @@ function getDetails($aPlaceIDs)
431431

432432
if (30 >= $this->iMinAddressRank && 30 <= $this->iMaxAddressRank)
433433
{
434-
//query also location_property_tiger and location_property_aux
435-
//Tiger search only if a housenumber was searched and if it was found (i.e. aPlaceIDs[placeID] = housenumber != -1) (realized through a join)
436-
//only Tiger housenumbers need to be interpolated, because they are saved as lines with start- and endnumber, the common osm housenumbers are usually saved as points
437-
$sHousenumbers = "";
438-
$i = 0;
439-
$length = count($aPlaceIDs);
440-
foreach($aPlaceIDs as $placeID => $housenumber)
441-
{
442-
$i++;
443-
$sHousenumbers .= "(".$placeID.", ".$housenumber.")";
444-
if($i<$length)
445-
$sHousenumbers .= ", ";
434+
if (CONST_Use_US_Tiger_Data)
435+
{
436+
//query also location_property_tiger and location_property_aux
437+
//Tiger search only if a housenumber was searched and if it was found (i.e. aPlaceIDs[placeID] = housenumber != -1) (realized through a join)
438+
//only Tiger housenumbers need to be interpolated, because they are saved as lines with start- and endnumber, the common osm housenumbers are usually saved as points
439+
$sHousenumbers = "";
440+
$i = 0;
441+
$length = count($aPlaceIDs);
442+
foreach($aPlaceIDs as $placeID => $housenumber)
443+
{
444+
$i++;
445+
$sHousenumbers .= "(".$placeID.", ".$housenumber.")";
446+
if($i<$length)
447+
$sHousenumbers .= ", ";
448+
}
449+
450+
$sSQL .= "union ";
451+
$sSQL .= "select 'T' as osm_type, place_id as osm_id, 'place' as class, 'house' as type, null as admin_level, 30 as rank_search, 30 as rank_address, min(place_id) as place_id, min(parent_place_id) as parent_place_id, 'us' as country_code";
452+
$sSQL .= ", get_address_by_language(place_id, housenumber_for_place, $sLanguagePrefArraySQL) as langaddress ";
453+
$sSQL .= ", null as placename";
454+
$sSQL .= ", null as ref";
455+
if ($this->bIncludeExtraTags) $sSQL .= ", null as extra";
456+
if ($this->bIncludeNameDetails) $sSQL .= ", null as names";
457+
$sSQL .= ", avg(st_x(centroid)) as lon, avg(st_y(centroid)) as lat,";
458+
$sSQL .= $sImportanceSQL."-1.15 as importance ";
459+
$sSQL .= ", (select max(p.importance*(p.rank_address+2)) from place_addressline s, placex p where s.place_id = min(blub.parent_place_id) and p.place_id = s.address_place_id and s.isaddress and p.importance is not null) as addressimportance ";
460+
$sSQL .= ", null as extra_place ";
461+
$sSQL .= " from (select place_id";
462+
//interpolate the Tiger housenumbers here
463+
$sSQL .= ", ST_LineInterpolatePoint(linegeo, (housenumber_for_place-startnumber::float)/(endnumber-startnumber)::float) as centroid, parent_place_id, housenumber_for_place ";
464+
$sSQL .= "from (location_property_tiger ";
465+
$sSQL .= " join (values ".$sHousenumbers.") as housenumbers(place_id, housenumber_for_place) using(place_id)) ";
466+
$sSQL .= " where housenumber_for_place>=0 and 30 between $this->iMinAddressRank and $this->iMaxAddressRank) as blub"; //postgres wants an alias here
467+
$sSQL .= " group by place_id, housenumber_for_place"; //is this group by really needed?, place_id + housenumber (in combination) are unique
468+
if (!$this->bDeDupe) $sSQL .= ", place_id ";
446469
}
447470

448-
$sSQL .= "union ";
449-
$sSQL .= "select 'T' as osm_type, place_id as osm_id, 'place' as class, 'house' as type, null as admin_level, 30 as rank_search, 30 as rank_address, min(place_id) as place_id, min(parent_place_id) as parent_place_id, 'us' as country_code";
450-
$sSQL .= ", get_address_by_language(place_id, housenumber_for_place, $sLanguagePrefArraySQL) as langaddress ";
451-
$sSQL .= ", null as placename";
452-
$sSQL .= ", null as ref";
453-
if ($this->bIncludeExtraTags) $sSQL .= ", null as extra";
454-
if ($this->bIncludeNameDetails) $sSQL .= ", null as names";
455-
$sSQL .= ", avg(st_x(centroid)) as lon, avg(st_y(centroid)) as lat,";
456-
$sSQL .= $sImportanceSQL."-1.15 as importance ";
457-
$sSQL .= ", (select max(p.importance*(p.rank_address+2)) from place_addressline s, placex p where s.place_id = min(blub.parent_place_id) and p.place_id = s.address_place_id and s.isaddress and p.importance is not null) as addressimportance ";
458-
$sSQL .= ", null as extra_place ";
459-
$sSQL .= " from (select place_id";
460-
//interpolate the Tiger housenumbers here
461-
$sSQL .= ", ST_LineInterpolatePoint(linegeo, (housenumber_for_place-startnumber::float)/(endnumber-startnumber)::float) as centroid, parent_place_id, housenumber_for_place ";
462-
$sSQL .= "from (location_property_tiger ";
463-
$sSQL .= " join (values ".$sHousenumbers.") as housenumbers(place_id, housenumber_for_place) using(place_id)) ";
464-
$sSQL .= " where housenumber_for_place>=0 and 30 between $this->iMinAddressRank and $this->iMaxAddressRank) as blub"; //postgres wants an alias here
465-
$sSQL .= " group by place_id, housenumber_for_place"; //is this group by really needed?, place_id + housenumber (in combination) are unique
466-
if (!$this->bDeDupe) $sSQL .= ", place_id ";
467-
468-
$sSQL .= " union ";
469-
$sSQL .= "select 'L' as osm_type, place_id as osm_id, 'place' as class, 'house' as type, null as admin_level, 0 as rank_search, 0 as rank_address, min(place_id) as place_id, min(parent_place_id) as parent_place_id, 'us' as country_code, ";
470-
$sSQL .= "get_address_by_language(place_id, -1, $sLanguagePrefArraySQL) as langaddress, ";
471-
$sSQL .= "null as placename, ";
472-
$sSQL .= "null as ref, ";
473-
if ($this->bIncludeExtraTags) $sSQL .= "null as extra, ";
474-
if ($this->bIncludeNameDetails) $sSQL .= "null as names, ";
475-
$sSQL .= "avg(ST_X(centroid)) as lon, avg(ST_Y(centroid)) as lat, ";
476-
$sSQL .= $sImportanceSQL."-1.10 as importance, ";
477-
$sSQL .= "(select max(p.importance*(p.rank_address+2)) from place_addressline s, placex p where s.place_id = min(location_property_aux.parent_place_id) and p.place_id = s.address_place_id and s.isaddress and p.importance is not null) as addressimportance, ";
478-
$sSQL .= "null as extra_place ";
479-
$sSQL .= "from location_property_aux where place_id in ($sPlaceIDs) ";
480-
$sSQL .= "and 30 between $this->iMinAddressRank and $this->iMaxAddressRank ";
481-
$sSQL .= "group by place_id";
482-
if (!$this->bDeDupe) $sSQL .= ", place_id";
483-
$sSQL .= ", get_address_by_language(place_id, -1, $sLanguagePrefArraySQL) ";
471+
if (CONST_Use_Aux_Location_data)
472+
{
473+
$sSQL .= " union ";
474+
$sSQL .= "select 'L' as osm_type, place_id as osm_id, 'place' as class, 'house' as type, null as admin_level, 0 as rank_search, 0 as rank_address, min(place_id) as place_id, min(parent_place_id) as parent_place_id, 'us' as country_code, ";
475+
$sSQL .= "get_address_by_language(place_id, -1, $sLanguagePrefArraySQL) as langaddress, ";
476+
$sSQL .= "null as placename, ";
477+
$sSQL .= "null as ref, ";
478+
if ($this->bIncludeExtraTags) $sSQL .= "null as extra, ";
479+
if ($this->bIncludeNameDetails) $sSQL .= "null as names, ";
480+
$sSQL .= "avg(ST_X(centroid)) as lon, avg(ST_Y(centroid)) as lat, ";
481+
$sSQL .= $sImportanceSQL."-1.10 as importance, ";
482+
$sSQL .= "(select max(p.importance*(p.rank_address+2)) from place_addressline s, placex p where s.place_id = min(location_property_aux.parent_place_id) and p.place_id = s.address_place_id and s.isaddress and p.importance is not null) as addressimportance, ";
483+
$sSQL .= "null as extra_place ";
484+
$sSQL .= "from location_property_aux where place_id in ($sPlaceIDs) ";
485+
$sSQL .= "and 30 between $this->iMinAddressRank and $this->iMaxAddressRank ";
486+
$sSQL .= "group by place_id";
487+
if (!$this->bDeDupe) $sSQL .= ", place_id";
488+
$sSQL .= ", get_address_by_language(place_id, -1, $sLanguagePrefArraySQL) ";
489+
}
484490
}
485491

486492
$sSQL .= " order by importance desc";
@@ -1431,7 +1437,7 @@ function lookup()
14311437
$aPlaceIDs = $this->oDB->getCol($sSQL);
14321438

14331439
// If nothing found try the aux fallback table
1434-
if (!sizeof($aPlaceIDs))
1440+
if (CONST_Use_Aux_Location_data && !sizeof($aPlaceIDs))
14351441
{
14361442
$sSQL = "select place_id from location_property_aux where parent_place_id in (".$sPlaceIDs.") and housenumber = '".pg_escape_string($aSearch['sHouseNumber'])."'";
14371443
if (sizeof($this->aExcludePlaceIDs))
@@ -1444,7 +1450,7 @@ function lookup()
14441450
}
14451451
//if nothing was found in placex or location_property_aux, then search in Tiger data for this housenumber(location_property_tiger)
14461452
$searchedHousenumber = intval($aSearch['sHouseNumber']);
1447-
if (!sizeof($aPlaceIDs))
1453+
if (CONST_Use_US_Tiger_Data && !sizeof($aPlaceIDs))
14481454
{
14491455
//new query for lines, not housenumbers anymore
14501456
if($searchedHousenumber%2 == 0){
@@ -1616,9 +1622,12 @@ function lookup()
16161622
$sSQL .= "and (placex.rank_address between $this->iMinAddressRank and $this->iMaxAddressRank ";
16171623
if (14 >= $this->iMinAddressRank && 14 <= $this->iMaxAddressRank) $sSQL .= " OR (extratags->'place') = 'city'";
16181624
if ($this->aAddressRankList) $sSQL .= " OR placex.rank_address in (".join(',',$this->aAddressRankList).")";
1619-
$sSQL .= ") UNION select place_id from location_property_tiger where place_id in (".join(',',array_keys($aResultPlaceIDs)).") ";
1620-
$sSQL .= "and (30 between $this->iMinAddressRank and $this->iMaxAddressRank ";
1621-
if ($this->aAddressRankList) $sSQL .= " OR 30 in (".join(',',$this->aAddressRankList).")";
1625+
if (CONST_Use_US_Tiger_Data)
1626+
{
1627+
$sSQL .= ") UNION select place_id from location_property_tiger where place_id in (".join(',',array_keys($aResultPlaceIDs)).") ";
1628+
$sSQL .= "and (30 between $this->iMinAddressRank and $this->iMaxAddressRank ";
1629+
if ($this->aAddressRankList) $sSQL .= " OR 30 in (".join(',',$this->aAddressRankList).")";
1630+
}
16221631
$sSQL .= ")";
16231632
if (CONST_Debug) var_dump($sSQL);
16241633
$aFilteredPlaceIDs = $this->oDB->getCol($sSQL);

‎lib/PlaceLookup.php

+2-2
Original file line numberDiff line numberDiff line change
@@ -122,7 +122,7 @@ function lookup()
122122

123123
$sLanguagePrefArraySQL = "ARRAY[".join(',',array_map("getDBQuoted", $this->aLangPrefOrder))."]";
124124

125-
if ($this->sType == 'tiger')
125+
if (CONST_Use_US_Tiger_Data && $this->sType == 'tiger')
126126
{
127127
$sSQL = "select place_id,partition, 'T' as osm_type, place_id as osm_id, 'place' as class, 'house' as type, null as admin_level, housenumber, null as street, null as isin, postcode,";
128128
$sSQL .= " 'us' as country_code, parent_place_id, null as linked_place_id, 30 as rank_address, 30 as rank_search,";
@@ -166,7 +166,7 @@ function lookup()
166166

167167
if ($this->bAddressDetails)
168168
{
169-
if($this->sType == 'tiger') // to get addressdetails for tiger data, the housenumber is needed
169+
if(CONST_Use_US_Tiger_Data && $this->sType == 'tiger') // to get addressdetails for tiger data, the housenumber is needed
170170
$aAddress = $this->getAddressNames($aPlace['housenumber']);
171171
else
172172
$aAddress = $this->getAddressNames();

‎lib/ReverseGeocode.php

+1-1
Original file line numberDiff line numberDiff line change
@@ -158,7 +158,7 @@ function lookup()
158158
}
159159

160160
// Only street found? If it's in the US we can check TIGER data for nearest housenumber
161-
if ($bIsInUnitedStates && $iMaxRank_orig >= 28 && $iPlaceID && ($aPlace['rank_search'] == 26 || $aPlace['rank_search'] == 27 ))
161+
if (CONST_Use_US_Tiger_Data && $bIsInUnitedStates && $iMaxRank_orig >= 28 && $iPlaceID && ($aPlace['rank_search'] == 26 || $aPlace['rank_search'] == 27 ))
162162
{
163163
$fSearchDiam = 0.001;
164164
$sSQL = 'SELECT place_id,parent_place_id,30 as rank_search, ST_line_locate_point(linegeo,'.$sPointSQL.') as fraction';

‎settings/settings.php

+10
Original file line numberDiff line numberDiff line change
@@ -13,6 +13,16 @@
1313
@define('CONST_Limit_Reindexing', true);
1414
// Set to false to avoid importing extra postcodes for the US.
1515
@define('CONST_Use_Extra_US_Postcodes', true);
16+
// Set to true after importing Tiger house number data for the US.
17+
// Note: The tables must already exist or queries will throw errors.
18+
// After changing this setting run ./utils/setup --create-functions
19+
// again.
20+
@define('CONST_Use_US_Tiger_Data', false);
21+
// Set to true after importing other external house number data.
22+
// Note: the aux tables must already exist or queries will throw errors.
23+
// After changing this setting run ./utils/setup --create-functions
24+
// again.
25+
@define('CONST_Use_Aux_Location_data', false);
1626

1727
// Proxy settings
1828
@define('CONST_HTTP_Proxy', false);

‎sql/aux_tables.sql

+6
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,6 @@
1+
CREATE TABLE location_property_aux () INHERITS (location_property);
2+
CREATE INDEX idx_location_property_aux_place_id ON location_property_aux USING BTREE (place_id);
3+
CREATE INDEX idx_location_property_aux_parent_place_id ON location_property_aux USING BTREE (parent_place_id);
4+
CREATE INDEX idx_location_property_aux_housenumber_parent_place_id ON location_property_aux USING BTREE (parent_place_id, housenumber);
5+
GRANT SELECT ON location_property_aux TO "{www-user}";
6+

‎sql/functions.sql

+8-2
Original file line numberDiff line numberDiff line change
@@ -1024,7 +1024,7 @@ BEGIN
10241024

10251025
--DEBUG: RAISE WARNING 'placex_insert:END: % % % %',NEW.osm_type,NEW.osm_id,NEW.class,NEW.type;
10261026

1027-
RETURN NEW; -- @DIFFUPDATES@ The following is not needed until doing diff updates, and slows the main index process down
1027+
RETURN NEW; -- %DIFFUPDATES% The following is not needed until doing diff updates, and slows the main index process down
10281028

10291029
IF NEW.rank_address > 0 THEN
10301030
IF (ST_GeometryType(NEW.geometry) in ('ST_Polygon','ST_MultiPolygon') AND ST_IsValid(NEW.geometry)) THEN
@@ -1650,6 +1650,7 @@ BEGIN
16501650
END IF;
16511651
END IF;
16521652

1653+
-- %NOTIGERDATA% IF 0 THEN
16531654
-- for the USA we have an additional address table. Merge in zip codes from there too
16541655
IF NEW.rank_search = 26 AND NEW.calculated_country_code = 'us' THEN
16551656
FOR location IN SELECT distinct postcode from location_property_tiger where parent_place_id = NEW.place_id LOOP
@@ -1662,6 +1663,7 @@ BEGIN
16621663
nameaddress_vector := array_merge(nameaddress_vector, ARRAY[address_street_word_id]);
16631664
END LOOP;
16641665
END IF;
1666+
-- %NOTIGERDATA% END IF;
16651667

16661668
-- RAISE WARNING 'ISIN: %', isin_tokens;
16671669

@@ -2257,18 +2259,22 @@ DECLARE
22572259
hadcountry BOOLEAN;
22582260
BEGIN
22592261
--first query tiger data
2262+
-- %NOTIGERDATA% IF 0 THEN
22602263
select parent_place_id,'us', 30, postcode, null, 'place', 'house' from location_property_tiger
22612264
WHERE place_id = in_place_id AND in_housenumber>=startnumber AND in_housenumber <= endnumber
22622265
INTO for_place_id,searchcountrycode, searchrankaddress, searchpostcode, searchhousename, searchclass, searchtype;
22632266
IF for_place_id IS NOT NULL THEN
22642267
searchhousenumber = in_housenumber::text;
22652268
END IF;
2266-
2269+
-- %NOTIGERDATA% END IF;
2270+
2271+
-- %NOAUXDATA% IF 0 THEN
22672272
IF for_place_id IS NULL THEN
22682273
select parent_place_id,'us', housenumber, 30, postcode, null, 'place', 'house' from location_property_aux
22692274
WHERE place_id = in_place_id
22702275
INTO for_place_id,searchcountrycode, searchhousenumber, searchrankaddress, searchpostcode, searchhousename, searchclass, searchtype;
22712276
END IF;
2277+
-- %NOAUXDATA% END IF;
22722278

22732279
IF for_place_id IS NULL THEN
22742280
select parent_place_id, calculated_country_code, housenumber, rank_search, postcode, name, class, type from placex

‎sql/tables.sql

-9
Original file line numberDiff line numberDiff line change
@@ -75,15 +75,6 @@ CREATE TABLE location_property (
7575
);
7676
SELECT AddGeometryColumn('location_property', 'centroid', 4326, 'POINT', 2);
7777

78-
CREATE TABLE location_property_aux () INHERITS (location_property);
79-
CREATE INDEX idx_location_property_aux_place_id ON location_property_aux USING BTREE (place_id);
80-
CREATE INDEX idx_location_property_aux_parent_place_id ON location_property_aux USING BTREE (parent_place_id);
81-
CREATE INDEX idx_location_property_aux_housenumber_parent_place_id ON location_property_aux USING BTREE (parent_place_id, housenumber);
82-
GRANT SELECT ON location_property_aux TO "{www-user}";
83-
84-
CREATE TABLE location_property_tiger (linegeo GEOMETRY, place_id BIGINT, partition INTEGER, parent_place_id BIGINT, startnumber INTEGER, endnumber INTEGER, interpolationtype TEXT, postcode TEXT);
85-
GRANT SELECT ON location_property_tiger TO "{www-user}";
86-
8778
drop table IF EXISTS search_name;
8879
CREATE TABLE search_name (
8980
place_id BIGINT,

‎utils/setup.php

+20-3
Original file line numberDiff line numberDiff line change
@@ -219,9 +219,26 @@
219219
if (!file_exists(CONST_InstallPath.'/module/nominatim.so')) fail("nominatim module not built");
220220
$sTemplate = file_get_contents(CONST_BasePath.'/sql/functions.sql');
221221
$sTemplate = str_replace('{modulepath}', CONST_InstallPath.'/module', $sTemplate);
222-
if ($aCMDResult['enable-diff-updates']) $sTemplate = str_replace('RETURN NEW; -- @DIFFUPDATES@', '--', $sTemplate);
223-
if ($aCMDResult['enable-debug-statements']) $sTemplate = str_replace('--DEBUG:', '', $sTemplate);
224-
if (CONST_Limit_Reindexing) $sTemplate = str_replace('--LIMIT INDEXING:', '', $sTemplate);
222+
if ($aCMDResult['enable-diff-updates'])
223+
{
224+
$sTemplate = str_replace('RETURN NEW; -- %DIFFUPDATES%', '--', $sTemplate);
225+
}
226+
if ($aCMDResult['enable-debug-statements'])
227+
{
228+
$sTemplate = str_replace('--DEBUG:', '', $sTemplate);
229+
}
230+
if (CONST_Limit_Reindexing)
231+
{
232+
$sTemplate = str_replace('--LIMIT INDEXING:', '', $sTemplate);
233+
}
234+
if (!CONST_Use_US_Tiger_Data)
235+
{
236+
$sTemplate = str_replace('-- %NOTIGERDATA% ', '', $sTemplate);
237+
}
238+
if (!CONST_Use_Aux_Location_data)
239+
{
240+
$sTemplate = str_replace('-- %NOAUXDATA% ', '', $sTemplate);
241+
}
225242
pgsqlRunScript($sTemplate);
226243
}
227244

‎website/details.php

+11-4
Original file line numberDiff line numberDiff line change
@@ -56,10 +56,17 @@
5656

5757
$iPlaceID = (int)$_GET['place_id'];
5858

59-
$iParentPlaceID = $oDB->getOne('select parent_place_id from location_property_tiger where place_id = '.$iPlaceID);
60-
if ($iParentPlaceID) $iPlaceID = $iParentPlaceID;
61-
$iParentPlaceID = $oDB->getOne('select parent_place_id from location_property_aux where place_id = '.$iPlaceID);
62-
if ($iParentPlaceID) $iPlaceID = $iParentPlaceID;
59+
if (CONST_Use_US_Tiger_Data)
60+
{
61+
$iParentPlaceID = $oDB->getOne('select parent_place_id from location_property_tiger where place_id = '.$iPlaceID);
62+
if ($iParentPlaceID) $iPlaceID = $iParentPlaceID;
63+
}
64+
65+
if (CONST_Use_Aux_Location_data)
66+
{
67+
$iParentPlaceID = $oDB->getOne('select parent_place_id from location_property_aux where place_id = '.$iPlaceID);
68+
if ($iParentPlaceID) $iPlaceID = $iParentPlaceID;
69+
}
6370

6471
$hLog = logStart($oDB, 'details', $_SERVER['QUERY_STRING'], $aLangPrefOrder);
6572

‎website/hierarchy.php

+5-5
Original file line numberDiff line numberDiff line change
@@ -47,13 +47,13 @@
4747

4848
$iPlaceID = (int)$_GET['place_id'];
4949

50-
$sAuxHouseNumber = false;
51-
$iParentPlaceID = $oDB->getOne('select parent_place_id from location_property_tiger where place_id = '.$iPlaceID);
52-
if ($iParentPlaceID)
50+
if (CONST_Use_US_Tiger_Data)
5351
{
54-
$iPlaceID = $iParentPlaceID;
52+
$iParentPlaceID = $oDB->getOne('select parent_place_id from location_property_tiger where place_id = '.$iPlaceID);
53+
if ($iParentPlaceID) $iPlaceID = $iParentPlaceID;
5554
}
56-
else
55+
56+
if (CONST_Use_Aux_Location_data)
5757
{
5858
$iParentPlaceID = $oDB->getOne('select parent_place_id from location_property_aux where place_id = '.$iPlaceID);
5959
if ($iParentPlaceID) $iPlaceID = $iParentPlaceID;

0 commit comments

Comments
 (0)