From 5ce42519dbae4ebda33420a55b6880ca6262383a Mon Sep 17 00:00:00 2001 From: ebocher Date: Fri, 8 Dec 2023 10:42:42 +0100 Subject: [PATCH 1/3] Workaround for endless query. See https://github.com/orbisgis/geoclimate/issues/876 --- .../geoclimate/osm/WorkflowOSM.groovy | 2 +- .../osm/InputDataFormattingTest.groovy | 4 +- .../geoclimate/osmtools/OSMTools.groovy | 1 + .../geoclimate/osmtools/Transform.groovy | 136 +++++++++++------- 4 files changed, 89 insertions(+), 54 deletions(-) diff --git a/osm/src/main/groovy/org/orbisgis/geoclimate/osm/WorkflowOSM.groovy b/osm/src/main/groovy/org/orbisgis/geoclimate/osm/WorkflowOSM.groovy index 0ed21dbfb9..8bc8f459ef 100644 --- a/osm/src/main/groovy/org/orbisgis/geoclimate/osm/WorkflowOSM.groovy +++ b/osm/src/main/groovy/org/orbisgis/geoclimate/osm/WorkflowOSM.groovy @@ -445,7 +445,7 @@ Map osm_processing(JdbcDataSource h2gis_datasource, def processing_parameters, d def extract = OSMTools.Loader.extract(query) if (extract) { - Geometry geomArea = h2gis_datasource.firstRow("select st_extent(the_geom) as the_geom from ${zones.zone}".toString()).the_geom + Geometry geomArea = h2gis_datasource.getExtent(zones.zone) geomArea.setSRID(srid) Map gisLayersResults = OSM.InputDataLoading.createGISLayers(h2gis_datasource, extract, geomArea, srid) if (gisLayersResults) { diff --git a/osm/src/test/groovy/org/orbisgis/geoclimate/osm/InputDataFormattingTest.groovy b/osm/src/test/groovy/org/orbisgis/geoclimate/osm/InputDataFormattingTest.groovy index ebc446360d..ad6662d114 100644 --- a/osm/src/test/groovy/org/orbisgis/geoclimate/osm/InputDataFormattingTest.groovy +++ b/osm/src/test/groovy/org/orbisgis/geoclimate/osm/InputDataFormattingTest.groovy @@ -312,9 +312,9 @@ class InputDataFormattingTest { zoneToExtract = "Göteborgs Stad" zoneToExtract = "Riantec" - zoneToExtract =[45.185546,5.751944,45.204296,5.784216] + zoneToExtract =[50, 8.6, 50.2, 8.8] - zoneToExtract="Sassenage" + //zoneToExtract="Sassenage" Map extractData = OSM.InputDataLoading.extractAndCreateGISLayers(h2GIS, zoneToExtract) diff --git a/osmtools/src/main/groovy/org/orbisgis/geoclimate/osmtools/OSMTools.groovy b/osmtools/src/main/groovy/org/orbisgis/geoclimate/osmtools/OSMTools.groovy index 5bcd0c55ce..62b18ac876 100644 --- a/osmtools/src/main/groovy/org/orbisgis/geoclimate/osmtools/OSMTools.groovy +++ b/osmtools/src/main/groovy/org/orbisgis/geoclimate/osmtools/OSMTools.groovy @@ -19,6 +19,7 @@ */ package org.orbisgis.geoclimate.osmtools + import org.orbisgis.geoclimate.osmtools.utils.TransformUtils import org.orbisgis.geoclimate.osmtools.utils.Utilities import org.orbisgis.geoclimate.utils.AbstractScript diff --git a/osmtools/src/main/groovy/org/orbisgis/geoclimate/osmtools/Transform.groovy b/osmtools/src/main/groovy/org/orbisgis/geoclimate/osmtools/Transform.groovy index 8af52cc50d..340b9f1ca3 100644 --- a/osmtools/src/main/groovy/org/orbisgis/geoclimate/osmtools/Transform.groovy +++ b/osmtools/src/main/groovy/org/orbisgis/geoclimate/osmtools/Transform.groovy @@ -319,8 +319,9 @@ String extractWaysAsPolygons(JdbcDataSource datasource, String osmTablesPrefix, CREATE INDEX ON $waysPolygonTmp(id_way); """.toString() - String query = """ DROP TABLE IF EXISTS $outputTableName; - CREATE TABLE $outputTableName AS + def allPolygonsTables = postfix "all_polygons_table" + String query = """ DROP TABLE IF EXISTS $allPolygonsTables; + CREATE TABLE $allPolygonsTables AS SELECT 'w'||a.id_way AS id,""" if (valid_geom) { query += """ case when st_isvalid(a.the_geom) then a.the_geom else st_makevalid(a.the_geom) end as the_geom ${OSMTools.TransformUtils.createTagList(datasource, columnsSelector, columnsToKeep)} @@ -335,27 +336,34 @@ String extractWaysAsPolygons(JdbcDataSource datasource, String osmTablesPrefix, query += " AND b.TAG_KEY IN ('${columnsToKeep.join("','")}') " } + //TODO : Due to some H2 limitations with the execution plan we create the whole table and then we apply a spatial filter + //https://github.com/orbisgis/geoclimate/issues/876 + query += " GROUP BY a.id_way;" + datasource.execute(query) if (geometry) { + def query_out ="""DROP TABLE IF EXISTS $outputTableName; + CREATE TABLE $outputTableName AS SELECT * FROM $allPolygonsTables as a where """ int geom_srid = geometry.getSRID() if (geom_srid == -1) { - query += " and a.the_geom && st_setsrid('$geometry'::geometry, $epsgCode) and st_intersects(a.the_geom, st_setsrid('$geometry'::geometry, $epsgCode)) " + query_out += " a.the_geom && st_setsrid('$geometry'::geometry, $epsgCode) and st_intersects(a.the_geom, st_setsrid('$geometry'::geometry, $epsgCode)) " } else if (geom_srid == epsgCode) { - query += " and a.the_geom && st_setsrid('$geometry'::geometry, $epsgCode) and st_intersects(a.the_geom,st_setsrid('$geometry'::geometry, $epsgCode)) " + query_out += " a.the_geom && st_setsrid('$geometry'::geometry, $epsgCode) and st_intersects(a.the_geom,st_setsrid('$geometry'::geometry, $epsgCode)) " } else { - query += " and a.the_geom && st_transform(st_setsrid('$geometry'::geometry, $geom_srid), $epsgCode) and st_intersects(a.the_geom,st_transform(st_setsrid('$geometry'::geometry, $geom_srid), $epsgCode)) " + query_out += " a.the_geom && st_transform(st_setsrid('$geometry'::geometry, $geom_srid), $epsgCode) and st_intersects(a.the_geom,st_transform(st_setsrid('$geometry'::geometry, $geom_srid), $epsgCode)) " } - datasource.createSpatialIndex(waysPolygonTmp, "the_geom") + datasource.createSpatialIndex(allPolygonsTables, "the_geom") + datasource """ + $query_out; + DROP TABLE IF EXISTS $waysPolygonTmp, $idWaysPolygons, $allPolygonsTables;""".toString() + return outputTableName } - query += " GROUP BY a.id_way;" - - - datasource """ - $query - DROP TABLE IF EXISTS $waysPolygonTmp; - DROP TABLE IF EXISTS $idWaysPolygons; + else{ + datasource """ + ALTER TABLE $allPolygonsTables RENAME TO $outputTableName; + DROP TABLE IF EXISTS $waysPolygonTmp, $idWaysPolygons; """.toString() - - return outputTableName + return outputTableName + } } /** @@ -560,9 +568,10 @@ def extractRelationsAsPolygons(JdbcDataSource datasource, String osmTablesPrefix CREATE INDEX ON $relationsMpHoles(id_relation); """.toString() + String allRelationPolygons = postfix("all_relation_polygons") def query = """ - DROP TABLE IF EXISTS $outputTableName; - CREATE TABLE $outputTableName AS + DROP TABLE IF EXISTS $allRelationPolygons; + CREATE TABLE $allRelationPolygons AS SELECT 'r'||a.id_relation AS id,""" if (valid_geom) { query += """ case when st_isvalid(a.the_geom) then a.the_geom else st_makevalid(a.the_geom) end as the_geom ${OSMTools.TransformUtils.createTagList(datasource, columnsSelector, columnsToKeep)} @@ -577,25 +586,32 @@ def extractRelationsAsPolygons(JdbcDataSource datasource, String osmTablesPrefix if(columnsToKeep) { query += " AND b.TAG_KEY IN ('${columnsToKeep.join("','")}') " } + query += " GROUP BY a.the_geom, a.id_relation;" + datasource.execute(query.toString()) if (geometry) { + def query_out ="""DROP TABLE IF EXISTS $outputTableName; + CREATE TABLE $outputTableName as SELECT * FROM $allRelationPolygons as a where""" int geom_srid = geometry.getSRID() if (geom_srid == -1) { - query += " and a.the_geom && st_setsrid('$geometry'::geometry, $epsgCode) and st_intersects(a.the_geom, st_setsrid('$geometry'::geometry, $epsgCode)) " + query_out += " a.the_geom && st_setsrid('$geometry'::geometry, $epsgCode) and st_intersects(a.the_geom, st_setsrid('$geometry'::geometry, $epsgCode)) " } else if (geom_srid == epsgCode) { - query += " and a.the_geom && st_setsrid('$geometry'::geometry, $epsgCode) and st_intersects(a.the_geom,st_setsrid('$geometry'::geometry, $epsgCode)) " + query_out += " a.the_geom && st_setsrid('$geometry'::geometry, $epsgCode) and st_intersects(a.the_geom,st_setsrid('$geometry'::geometry, $epsgCode)) " } else { - query += " and a.the_geom && st_transform(st_setsrid('$geometry'::geometry, $geom_srid), $epsgCode) and st_intersects(a.the_geom,st_transform(st_setsrid('$geometry'::geometry, $geom_srid), $epsgCode)) " + query_out += " a.the_geom && st_transform(st_setsrid('$geometry'::geometry, $geom_srid), $epsgCode) and st_intersects(a.the_geom,st_transform(st_setsrid('$geometry'::geometry, $geom_srid), $epsgCode)) " } - datasource.createSpatialIndex(relationsMpHoles, "the_geom") + datasource.createSpatialIndex(allRelationPolygons, "the_geom") + datasource.execute(query_out.toString()) + datasource.dropTable(relationsPolygonsOuter, relationsPolygonsInner, relationsPolygonsOuterExploded, + relationsPolygonsInnerExploded,relationsMpHoles,relationFilteredKeys, allRelationPolygons) + return outputTableName + }else{ + datasource.execute("""ALTER TABLE $allRelationPolygons RENAME TO $outputTableName""".toString()) + datasource.dropTable(relationsPolygonsOuter, relationsPolygonsInner, relationsPolygonsOuterExploded, + relationsPolygonsInnerExploded,relationsMpHoles,relationFilteredKeys) + return outputTableName } - query += " GROUP BY a.the_geom, a.id_relation;" - - datasource query.toString() - datasource.dropTable(relationsPolygonsOuter, relationsPolygonsInner, relationsPolygonsOuterExploded, - relationsPolygonsInnerExploded,relationsMpHoles,relationFilteredKeys) - return outputTableName } /** @@ -725,9 +741,11 @@ String extractWaysAsLines(JdbcDataSource datasource, String osmTablesPrefix, int CREATE INDEX ON $waysLinesTmp(ID_WAY); """.toString() + def allLinesTables = postfix "all_lines_table" + def query = """ - DROP TABLE IF EXISTS $outputTableName; - CREATE TABLE $outputTableName AS + DROP TABLE IF EXISTS $allLinesTables; + CREATE TABLE $allLinesTables AS SELECT 'w'||a.id_way AS id, a.the_geom ${OSMTools.TransformUtils.createTagList(datasource, columnsSelector,columnsToKeep)} FROM $waysLinesTmp AS a, ${osmTablesPrefix}_way_tag b WHERE a.id_way=b.id_way and st_isempty(a.the_geom)=false """ @@ -736,25 +754,33 @@ String extractWaysAsLines(JdbcDataSource datasource, String osmTablesPrefix, int query += " AND b.TAG_KEY IN ('${columnsToKeep.join("','")}') " } + query += " GROUP BY a.id_way;" + datasource.execute(query.toString()) if (geometry) { + def query_out = """DROP TABLE IF EXISTS $outputTableName; + CREATE TABLE $outputTableName as select * from $allLinesTables as a where """ int geom_srid = geometry.getSRID() if (geom_srid == -1) { - query += " and a.the_geom && st_setsrid('$geometry'::geometry, $epsgCode) and st_intersects(a.the_geom, st_setsrid('$geometry'::geometry, $epsgCode)) " + query_out += " a.the_geom && st_setsrid('$geometry'::geometry, $epsgCode) and st_intersects(a.the_geom, st_setsrid('$geometry'::geometry, $epsgCode)) " } else if (geom_srid == epsgCode) { - query += " and a.the_geom && st_setsrid('$geometry'::geometry, $epsgCode) and st_intersects(a.the_geom,st_setsrid('$geometry'::geometry, $epsgCode)) " + query_out += " a.the_geom && st_setsrid('$geometry'::geometry, $epsgCode) and st_intersects(a.the_geom,st_setsrid('$geometry'::geometry, $epsgCode)) " } else { - query += " and a.the_geom && st_transform(st_setsrid('$geometry'::geometry, $geom_srid), $epsgCode) and st_intersects(a.the_geom,st_transform(st_setsrid('$geometry'::geometry, $geom_srid), $epsgCode)) " + query_out += " a.the_geom && st_transform(st_setsrid('$geometry'::geometry, $geom_srid), $epsgCode) and st_intersects(a.the_geom,st_transform(st_setsrid('$geometry'::geometry, $geom_srid), $epsgCode)) " } - datasource.createSpatialIndex(waysLinesTmp, "the_geom") - } - - query += " GROUP BY a.id_way;" - - datasource """ - $query + datasource.createSpatialIndex(allLinesTables, "the_geom") + datasource """ + $query_out; + DROP TABLE IF EXISTS $waysLinesTmp, $idWaysTable, $allLinesTables; + """.toString() + return outputTableName + }else{ + datasource """ + ALTER TABLE $allLinesTables RENAME TO $outputTableName; DROP TABLE IF EXISTS $waysLinesTmp, $idWaysTable; """.toString() - return outputTableName + return outputTableName + } + } /** @@ -892,30 +918,38 @@ String extractRelationsAsLines(JdbcDataSource datasource, String osmTablesPrefix def columnsSelector = OSMTools.TransformUtils.getColumnSelector(osmTableTag, tags, columnsToKeep) + def allRelationLines = postfix("all_relation_lines") def query = """ - DROP TABLE IF EXISTS $outputTableName; - CREATE TABLE $outputTableName AS + DROP TABLE IF EXISTS $allRelationLines; + CREATE TABLE $allRelationLines AS SELECT 'r'||a.id_relation AS id, a.the_geom ${OSMTools.TransformUtils.createTagList(datasource, columnsSelector, columnsToKeep)} FROM $relationsLinesTmp AS a, ${osmTablesPrefix}_relation_tag b WHERE a.id_relation=b.id_relation and st_isempty(a.the_geom)=false """ if(columnsToKeep) { query += " AND b.TAG_KEY IN ('${columnsToKeep.join("','")}') " } - + query += " GROUP BY a.id_relation;" + datasource.execute(query) if (geometry) { + def query_out =""" DROP TABLE IF EXISTS $outputTableName; + CREATE TABLE $outputTableName as SELECT * FROM $allRelationLines as a where """ int geom_srid = geometry.getSRID() if (geom_srid == -1) { - query += " and a.the_geom && st_setsrid('$geometry'::geometry, $epsgCode) and st_intersects(a.the_geom, st_setsrid('$geometry'::geometry, $epsgCode)) " + query_out += " a.the_geom && st_setsrid('$geometry'::geometry, $epsgCode) and st_intersects(a.the_geom, st_setsrid('$geometry'::geometry, $epsgCode)) " } else if (geom_srid == epsgCode) { - query += " and a.the_geom && st_setsrid('$geometry'::geometry, $epsgCode) and st_intersects(a.the_geom,st_setsrid('$geometry'::geometry, $epsgCode)) " + query_out += " a.the_geom && st_setsrid('$geometry'::geometry, $epsgCode) and st_intersects(a.the_geom,st_setsrid('$geometry'::geometry, $epsgCode)) " } else { - query += " and a.the_geom && st_transform(st_setsrid('$geometry'::geometry, $geom_srid), $epsgCode) and st_intersects(a.the_geom,st_transform(st_setsrid('$geometry'::geometry, $geom_srid), $epsgCode)) " + query_out += " a.the_geom && st_transform(st_setsrid('$geometry'::geometry, $geom_srid), $epsgCode) and st_intersects(a.the_geom,st_transform(st_setsrid('$geometry'::geometry, $geom_srid), $epsgCode)) " } - datasource.createSpatialIndex(relationsLinesTmp, "the_geom") - } - query += " GROUP BY a.id_relation;" - datasource.execute(""" $query + datasource.createSpatialIndex(allRelationLines, "the_geom") + datasource.execute(""" $query_out ; + DROP TABLE IF EXISTS $relationsLinesTmp, $relationsFilteredKeys, $allRelationLines; + """.toString()) + return outputTableName + }else{ + datasource.execute(""" ALTER TABLE $allRelationLines RENAME TO $outputTableName; DROP TABLE IF EXISTS $relationsLinesTmp, $relationsFilteredKeys; """.toString()) - return outputTableName + return outputTableName + } } From 7ab4392fcfc3e4a62a6fb322cbe0e2543fc8d50a Mon Sep 17 00:00:00 2001 From: ebocher Date: Fri, 8 Dec 2023 11:10:00 +0100 Subject: [PATCH 2/3] Workaround for endless query. See https://github.com/orbisgis/geoclimate/issues/876 --- .../org/orbisgis/geoclimate/osm/WorflowOSMTest.groovy | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/osm/src/test/groovy/org/orbisgis/geoclimate/osm/WorflowOSMTest.groovy b/osm/src/test/groovy/org/orbisgis/geoclimate/osm/WorflowOSMTest.groovy index f50b71e03e..5224077ed8 100644 --- a/osm/src/test/groovy/org/orbisgis/geoclimate/osm/WorflowOSMTest.groovy +++ b/osm/src/test/groovy/org/orbisgis/geoclimate/osm/WorflowOSMTest.groovy @@ -650,9 +650,10 @@ class WorflowOSMTest extends WorkflowAbstractTest { dirFile.delete() dirFile.mkdir() def location = "Nice" - //def nominatim = OSMTools.Utilities.getNominatimData("Nantes") - location="Sassenage" - //location = nominatim.bbox + //def nominatim = org.orbisgis.geoclimate.osmtools.OSMTools.Utilities.getNominatimData("Redon") + // location = nominatim.bbox + + location=[50, 8.6, 50.2, 8.8] def osm_parmeters = [ "description" : "Example of configuration file to run the OSM workflow and store the result in a folder", From 21655ab2cc7720db59a7288094d62a41915c14cf Mon Sep 17 00:00:00 2001 From: ebocher Date: Fri, 8 Dec 2023 11:47:18 +0100 Subject: [PATCH 3/3] Fix query interpolation --- .../groovy/org/orbisgis/geoclimate/osmtools/Transform.groovy | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/osmtools/src/main/groovy/org/orbisgis/geoclimate/osmtools/Transform.groovy b/osmtools/src/main/groovy/org/orbisgis/geoclimate/osmtools/Transform.groovy index 340b9f1ca3..123c3952b9 100644 --- a/osmtools/src/main/groovy/org/orbisgis/geoclimate/osmtools/Transform.groovy +++ b/osmtools/src/main/groovy/org/orbisgis/geoclimate/osmtools/Transform.groovy @@ -929,7 +929,7 @@ String extractRelationsAsLines(JdbcDataSource datasource, String osmTablesPrefix query += " AND b.TAG_KEY IN ('${columnsToKeep.join("','")}') " } query += " GROUP BY a.id_relation;" - datasource.execute(query) + datasource.execute(query.toString()) if (geometry) { def query_out =""" DROP TABLE IF EXISTS $outputTableName; CREATE TABLE $outputTableName as SELECT * FROM $allRelationLines as a where """