Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Workaround for endless query. #889

Merged
merged 5 commits into from
Dec 8, 2023
Merged

Workaround for endless query. #889

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
5ce4251
Workaround for endless query.
ebocher Dec 8, 2023
fc4ef28
Merge branch 'master' into update_november
ebocher Dec 8, 2023
7ab4392
Workaround for endless query.
ebocher Dec 8, 2023
21655ab
Fix query interpolation
ebocher Dec 8, 2023
fd63b7a
Merge remote-tracking branch 'ebocher/update_november' into update_no…
ebocher Dec 8, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion osm/src/main/groovy/org/orbisgis/geoclimate/osm/WorkflowOSM.groovy
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -445,7 +445,7 @@ Map osm_processing(JdbcDataSource h2gis_datasource, def processing_parameters, d

def extract = OSMTools.Loader.extract(query)
if (extract) {
Geometry geomArea = h2gis_datasource.firstRow("select st_extent(the_geom) as the_geom from ${zones.zone}".toString()).the_geom
Geometry geomArea = h2gis_datasource.getExtent(zones.zone)
geomArea.setSRID(srid)
Map gisLayersResults = OSM.InputDataLoading.createGISLayers(h2gis_datasource, extract, geomArea, srid)
if (gisLayersResults) {
Expand Down
4 changes: 3 additions & 1 deletion osm/src/test/groovy/org/orbisgis/geoclimate/osm/InputDataFormattingTest.groovy
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -312,7 +312,9 @@ class InputDataFormattingTest {
zoneToExtract = "Göteborgs Stad"

zoneToExtract = "Riantec"
zoneToExtract =[45.185546,5.751944,45.204296,5.784216]
zoneToExtract =[50, 8.6, 50.2, 8.8]

//zoneToExtract="Sassenage"

zoneToExtract=[50, 8.6, 50.2, 8.8]

Expand Down
9 changes: 5 additions & 4 deletions osm/src/test/groovy/org/orbisgis/geoclimate/osm/WorflowOSMTest.groovy
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -650,9 +650,10 @@ class WorflowOSMTest extends WorkflowAbstractTest {
dirFile.delete()
dirFile.mkdir()
def location = "Nice"
//def nominatim = OSMTools.Utilities.getNominatimData("Nantes")
//location=[50, 8.6, 50.2, 8.8]
//location = nominatim.bbox
//def nominatim = org.orbisgis.geoclimate.osmtools.OSMTools.Utilities.getNominatimData("Redon")
// location = nominatim.bbox

location=[50, 8.6, 50.2, 8.8]

def osm_parmeters = [
"description" : "Example of configuration file to run the OSM workflow and store the result in a folder",
Expand All @@ -674,7 +675,7 @@ class WorflowOSMTest extends WorkflowAbstractTest {
["distance" : 0,
"rsu_indicators" : [

"indicatorUse": ["LCZ", "UTRF"] //, "UTRF", "TEB"]
"indicatorUse": ["LCZ"] //, "UTRF", "TEB"]

]/*,"grid_indicators": [
"x_size": 200,
Expand Down
1 change: 1 addition & 0 deletions osmtools/src/main/groovy/org/orbisgis/geoclimate/osmtools/OSMTools.groovy
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,7 @@
*/
package org.orbisgis.geoclimate.osmtools


import org.orbisgis.geoclimate.osmtools.utils.TransformUtils
import org.orbisgis.geoclimate.osmtools.utils.Utilities
import org.orbisgis.geoclimate.utils.AbstractScript
Expand Down
136 changes: 85 additions & 51 deletions osmtools/src/main/groovy/org/orbisgis/geoclimate/osmtools/Transform.groovy
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -319,8 +319,9 @@ String extractWaysAsPolygons(JdbcDataSource datasource, String osmTablesPrefix,
CREATE INDEX ON $waysPolygonTmp(id_way);
""".toString()

String query = """ DROP TABLE IF EXISTS $outputTableName;
CREATE TABLE $outputTableName AS
def allPolygonsTables = postfix "all_polygons_table"
String query = """ DROP TABLE IF EXISTS $allPolygonsTables;
CREATE TABLE $allPolygonsTables AS
SELECT 'w'||a.id_way AS id,"""
if (valid_geom) {
query += """ case when st_isvalid(a.the_geom) then a.the_geom else st_makevalid(a.the_geom) end as the_geom ${OSMTools.TransformUtils.createTagList(datasource, columnsSelector, columnsToKeep)}
Expand All @@ -335,27 +336,34 @@ String extractWaysAsPolygons(JdbcDataSource datasource, String osmTablesPrefix,
query += " AND b.TAG_KEY IN ('${columnsToKeep.join("','")}') "
}

//TODO : Due to some H2 limitations with the execution plan we create the whole table and then we apply a spatial filter
//https://github.com/orbisgis/geoclimate/issues/876
query += " GROUP BY a.id_way;"
datasource.execute(query)
if (geometry) {
def query_out ="""DROP TABLE IF EXISTS $outputTableName;
CREATE TABLE $outputTableName AS SELECT * FROM $allPolygonsTables as a where """
int geom_srid = geometry.getSRID()
if (geom_srid == -1) {
query += " and a.the_geom && st_setsrid('$geometry'::geometry, $epsgCode) and st_intersects(a.the_geom, st_setsrid('$geometry'::geometry, $epsgCode)) "
query_out += " a.the_geom && st_setsrid('$geometry'::geometry, $epsgCode) and st_intersects(a.the_geom, st_setsrid('$geometry'::geometry, $epsgCode)) "
} else if (geom_srid == epsgCode) {
query += " and a.the_geom && st_setsrid('$geometry'::geometry, $epsgCode) and st_intersects(a.the_geom,st_setsrid('$geometry'::geometry, $epsgCode)) "
query_out += " a.the_geom && st_setsrid('$geometry'::geometry, $epsgCode) and st_intersects(a.the_geom,st_setsrid('$geometry'::geometry, $epsgCode)) "
} else {
query += " and a.the_geom && st_transform(st_setsrid('$geometry'::geometry, $geom_srid), $epsgCode) and st_intersects(a.the_geom,st_transform(st_setsrid('$geometry'::geometry, $geom_srid), $epsgCode)) "
query_out += " a.the_geom && st_transform(st_setsrid('$geometry'::geometry, $geom_srid), $epsgCode) and st_intersects(a.the_geom,st_transform(st_setsrid('$geometry'::geometry, $geom_srid), $epsgCode)) "
}
datasource.createSpatialIndex(waysPolygonTmp, "the_geom")
datasource.createSpatialIndex(allPolygonsTables, "the_geom")
datasource """
$query_out;
DROP TABLE IF EXISTS $waysPolygonTmp, $idWaysPolygons, $allPolygonsTables;""".toString()
return outputTableName
}
query += " GROUP BY a.id_way;"


datasource """
$query
DROP TABLE IF EXISTS $waysPolygonTmp;
DROP TABLE IF EXISTS $idWaysPolygons;
else{
datasource """
ALTER TABLE $allPolygonsTables RENAME TO $outputTableName;
DROP TABLE IF EXISTS $waysPolygonTmp, $idWaysPolygons;
""".toString()

return outputTableName
return outputTableName
}
}

/**
Expand Down Expand Up @@ -560,9 +568,10 @@ def extractRelationsAsPolygons(JdbcDataSource datasource, String osmTablesPrefix
CREATE INDEX ON $relationsMpHoles(id_relation);
""".toString()

String allRelationPolygons = postfix("all_relation_polygons")
def query = """
DROP TABLE IF EXISTS $outputTableName;
CREATE TABLE $outputTableName AS
DROP TABLE IF EXISTS $allRelationPolygons;
CREATE TABLE $allRelationPolygons AS
SELECT 'r'||a.id_relation AS id,"""
if (valid_geom) {
query += """ case when st_isvalid(a.the_geom) then a.the_geom else st_makevalid(a.the_geom) end as the_geom ${OSMTools.TransformUtils.createTagList(datasource, columnsSelector, columnsToKeep)}
Expand All @@ -577,25 +586,32 @@ def extractRelationsAsPolygons(JdbcDataSource datasource, String osmTablesPrefix
if(columnsToKeep) {
query += " AND b.TAG_KEY IN ('${columnsToKeep.join("','")}') "
}
query += " GROUP BY a.the_geom, a.id_relation;"
datasource.execute(query.toString())

if (geometry) {
def query_out ="""DROP TABLE IF EXISTS $outputTableName;
CREATE TABLE $outputTableName as SELECT * FROM $allRelationPolygons as a where"""
int geom_srid = geometry.getSRID()
if (geom_srid == -1) {
query += " and a.the_geom && st_setsrid('$geometry'::geometry, $epsgCode) and st_intersects(a.the_geom, st_setsrid('$geometry'::geometry, $epsgCode)) "
query_out += " a.the_geom && st_setsrid('$geometry'::geometry, $epsgCode) and st_intersects(a.the_geom, st_setsrid('$geometry'::geometry, $epsgCode)) "
} else if (geom_srid == epsgCode) {
query += " and a.the_geom && st_setsrid('$geometry'::geometry, $epsgCode) and st_intersects(a.the_geom,st_setsrid('$geometry'::geometry, $epsgCode)) "
query_out += " a.the_geom && st_setsrid('$geometry'::geometry, $epsgCode) and st_intersects(a.the_geom,st_setsrid('$geometry'::geometry, $epsgCode)) "
} else {
query += " and a.the_geom && st_transform(st_setsrid('$geometry'::geometry, $geom_srid), $epsgCode) and st_intersects(a.the_geom,st_transform(st_setsrid('$geometry'::geometry, $geom_srid), $epsgCode)) "
query_out += " a.the_geom && st_transform(st_setsrid('$geometry'::geometry, $geom_srid), $epsgCode) and st_intersects(a.the_geom,st_transform(st_setsrid('$geometry'::geometry, $geom_srid), $epsgCode)) "
}
datasource.createSpatialIndex(relationsMpHoles, "the_geom")
datasource.createSpatialIndex(allRelationPolygons, "the_geom")
datasource.execute(query_out.toString())
datasource.dropTable(relationsPolygonsOuter, relationsPolygonsInner, relationsPolygonsOuterExploded,
relationsPolygonsInnerExploded,relationsMpHoles,relationFilteredKeys, allRelationPolygons)
return outputTableName
}else{
datasource.execute("""ALTER TABLE $allRelationPolygons RENAME TO $outputTableName""".toString())
datasource.dropTable(relationsPolygonsOuter, relationsPolygonsInner, relationsPolygonsOuterExploded,
relationsPolygonsInnerExploded,relationsMpHoles,relationFilteredKeys)
return outputTableName
}

query += " GROUP BY a.the_geom, a.id_relation;"

datasource query.toString()
datasource.dropTable(relationsPolygonsOuter, relationsPolygonsInner, relationsPolygonsOuterExploded,
relationsPolygonsInnerExploded,relationsMpHoles,relationFilteredKeys)
return outputTableName
}

/**
Expand Down Expand Up @@ -725,9 +741,11 @@ String extractWaysAsLines(JdbcDataSource datasource, String osmTablesPrefix, int
CREATE INDEX ON $waysLinesTmp(ID_WAY);
""".toString()

def allLinesTables = postfix "all_lines_table"

def query = """
DROP TABLE IF EXISTS $outputTableName;
CREATE TABLE $outputTableName AS
DROP TABLE IF EXISTS $allLinesTables;
CREATE TABLE $allLinesTables AS
SELECT 'w'||a.id_way AS id, a.the_geom ${OSMTools.TransformUtils.createTagList(datasource, columnsSelector,columnsToKeep)}
FROM $waysLinesTmp AS a, ${osmTablesPrefix}_way_tag b
WHERE a.id_way=b.id_way and st_isempty(a.the_geom)=false """
Expand All @@ -736,25 +754,33 @@ String extractWaysAsLines(JdbcDataSource datasource, String osmTablesPrefix, int
query += " AND b.TAG_KEY IN ('${columnsToKeep.join("','")}') "
}

query += " GROUP BY a.id_way;"
datasource.execute(query.toString())
if (geometry) {
def query_out = """DROP TABLE IF EXISTS $outputTableName;
CREATE TABLE $outputTableName as select * from $allLinesTables as a where """
int geom_srid = geometry.getSRID()
if (geom_srid == -1) {
query += " and a.the_geom && st_setsrid('$geometry'::geometry, $epsgCode) and st_intersects(a.the_geom, st_setsrid('$geometry'::geometry, $epsgCode)) "
query_out += " a.the_geom && st_setsrid('$geometry'::geometry, $epsgCode) and st_intersects(a.the_geom, st_setsrid('$geometry'::geometry, $epsgCode)) "
} else if (geom_srid == epsgCode) {
query += " and a.the_geom && st_setsrid('$geometry'::geometry, $epsgCode) and st_intersects(a.the_geom,st_setsrid('$geometry'::geometry, $epsgCode)) "
query_out += " a.the_geom && st_setsrid('$geometry'::geometry, $epsgCode) and st_intersects(a.the_geom,st_setsrid('$geometry'::geometry, $epsgCode)) "
} else {
query += " and a.the_geom && st_transform(st_setsrid('$geometry'::geometry, $geom_srid), $epsgCode) and st_intersects(a.the_geom,st_transform(st_setsrid('$geometry'::geometry, $geom_srid), $epsgCode)) "
query_out += " a.the_geom && st_transform(st_setsrid('$geometry'::geometry, $geom_srid), $epsgCode) and st_intersects(a.the_geom,st_transform(st_setsrid('$geometry'::geometry, $geom_srid), $epsgCode)) "
}
datasource.createSpatialIndex(waysLinesTmp, "the_geom")
}

query += " GROUP BY a.id_way;"

datasource """
$query
datasource.createSpatialIndex(allLinesTables, "the_geom")
datasource """
$query_out;
DROP TABLE IF EXISTS $waysLinesTmp, $idWaysTable, $allLinesTables;
""".toString()
return outputTableName
}else{
datasource """
ALTER TABLE $allLinesTables RENAME TO $outputTableName;
DROP TABLE IF EXISTS $waysLinesTmp, $idWaysTable;
""".toString()
return outputTableName
return outputTableName
}

}

/**
Expand Down Expand Up @@ -892,30 +918,38 @@ String extractRelationsAsLines(JdbcDataSource datasource, String osmTablesPrefix

def columnsSelector = OSMTools.TransformUtils.getColumnSelector(osmTableTag, tags, columnsToKeep)

def allRelationLines = postfix("all_relation_lines")
def query = """
DROP TABLE IF EXISTS $outputTableName;
CREATE TABLE $outputTableName AS
DROP TABLE IF EXISTS $allRelationLines;
CREATE TABLE $allRelationLines AS
SELECT 'r'||a.id_relation AS id, a.the_geom ${OSMTools.TransformUtils.createTagList(datasource, columnsSelector, columnsToKeep)}
FROM $relationsLinesTmp AS a, ${osmTablesPrefix}_relation_tag b
WHERE a.id_relation=b.id_relation and st_isempty(a.the_geom)=false """
if(columnsToKeep) {
query += " AND b.TAG_KEY IN ('${columnsToKeep.join("','")}') "
}

query += " GROUP BY a.id_relation;"
datasource.execute(query.toString())
if (geometry) {
def query_out =""" DROP TABLE IF EXISTS $outputTableName;
CREATE TABLE $outputTableName as SELECT * FROM $allRelationLines as a where """
int geom_srid = geometry.getSRID()
if (geom_srid == -1) {
query += " and a.the_geom && st_setsrid('$geometry'::geometry, $epsgCode) and st_intersects(a.the_geom, st_setsrid('$geometry'::geometry, $epsgCode)) "
query_out += " a.the_geom && st_setsrid('$geometry'::geometry, $epsgCode) and st_intersects(a.the_geom, st_setsrid('$geometry'::geometry, $epsgCode)) "
} else if (geom_srid == epsgCode) {
query += " and a.the_geom && st_setsrid('$geometry'::geometry, $epsgCode) and st_intersects(a.the_geom,st_setsrid('$geometry'::geometry, $epsgCode)) "
query_out += " a.the_geom && st_setsrid('$geometry'::geometry, $epsgCode) and st_intersects(a.the_geom,st_setsrid('$geometry'::geometry, $epsgCode)) "
} else {
query += " and a.the_geom && st_transform(st_setsrid('$geometry'::geometry, $geom_srid), $epsgCode) and st_intersects(a.the_geom,st_transform(st_setsrid('$geometry'::geometry, $geom_srid), $epsgCode)) "
query_out += " a.the_geom && st_transform(st_setsrid('$geometry'::geometry, $geom_srid), $epsgCode) and st_intersects(a.the_geom,st_transform(st_setsrid('$geometry'::geometry, $geom_srid), $epsgCode)) "
}
datasource.createSpatialIndex(relationsLinesTmp, "the_geom")
}
query += " GROUP BY a.id_relation;"
datasource.execute(""" $query
datasource.createSpatialIndex(allRelationLines, "the_geom")
datasource.execute(""" $query_out ;
DROP TABLE IF EXISTS $relationsLinesTmp, $relationsFilteredKeys, $allRelationLines;
""".toString())
return outputTableName
}else{
datasource.execute(""" ALTER TABLE $allRelationLines RENAME TO $outputTableName;
DROP TABLE IF EXISTS $relationsLinesTmp, $relationsFilteredKeys;
""".toString())
return outputTableName
return outputTableName
}
}
Loading