Skip to content

Commit

Permalink
Merge pull request #889 from ebocher/update_november
Browse files Browse the repository at this point in the history 
Workaround for endless query.
  • Loading branch information
@ebocher
ebocher authored Dec 8, 2023
2 parents 3974115 + fd63b7a commit 3eafd5a
Show file tree
Hide file tree
Showing 5 changed files with 95 additions and 57 deletions.
2 changes: 1 addition & 1 deletion osm/src/main/groovy/org/orbisgis/geoclimate/osm/WorkflowOSM.groovy
View file
Original file line number Diff line number Diff line change
Expand Up @@ -445,7 +445,7 @@ Map osm_processing(JdbcDataSource h2gis_datasource, def processing_parameters, d

def extract = OSMTools.Loader.extract(query)
if (extract) {
Geometry geomArea = h2gis_datasource.firstRow("select st_extent(the_geom) as the_geom from ${zones.zone}".toString()).the_geom
Geometry geomArea = h2gis_datasource.getExtent(zones.zone)
geomArea.setSRID(srid)
Map gisLayersResults = OSM.InputDataLoading.createGISLayers(h2gis_datasource, extract, geomArea, srid)
if (gisLayersResults) {
Expand Down
4 changes: 3 additions & 1 deletion osm/src/test/groovy/org/orbisgis/geoclimate/osm/InputDataFormattingTest.groovy
View file
Original file line number Diff line number Diff line change
Expand Up @@ -312,7 +312,9 @@ class InputDataFormattingTest {
zoneToExtract = "Göteborgs Stad"

zoneToExtract = "Riantec"
zoneToExtract =[45.185546,5.751944,45.204296,5.784216]
zoneToExtract =[50, 8.6, 50.2, 8.8]

//zoneToExtract="Sassenage"

zoneToExtract=[50, 8.6, 50.2, 8.8]

Expand Down
9 changes: 5 additions & 4 deletions osm/src/test/groovy/org/orbisgis/geoclimate/osm/WorflowOSMTest.groovy
View file
Original file line number Diff line number Diff line change
Expand Up @@ -650,9 +650,10 @@ class WorflowOSMTest extends WorkflowAbstractTest {
dirFile.delete()
dirFile.mkdir()
def location = "Nice"
//def nominatim = OSMTools.Utilities.getNominatimData("Nantes")
//location=[50, 8.6, 50.2, 8.8]
//location = nominatim.bbox
//def nominatim = org.orbisgis.geoclimate.osmtools.OSMTools.Utilities.getNominatimData("Redon")
// location = nominatim.bbox

location=[50, 8.6, 50.2, 8.8]

def osm_parmeters = [
"description" : "Example of configuration file to run the OSM workflow and store the result in a folder",
Expand All @@ -674,7 +675,7 @@ class WorflowOSMTest extends WorkflowAbstractTest {
["distance" : 0,
"rsu_indicators" : [

"indicatorUse": ["LCZ", "UTRF"] //, "UTRF", "TEB"]
"indicatorUse": ["LCZ"] //, "UTRF", "TEB"]

]/*,"grid_indicators": [
"x_size": 200,
Expand Down
1 change: 1 addition & 0 deletions osmtools/src/main/groovy/org/orbisgis/geoclimate/osmtools/OSMTools.groovy
View file
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,7 @@
*/
package org.orbisgis.geoclimate.osmtools


import org.orbisgis.geoclimate.osmtools.utils.TransformUtils
import org.orbisgis.geoclimate.osmtools.utils.Utilities
import org.orbisgis.geoclimate.utils.AbstractScript
Expand Down
136 changes: 85 additions & 51 deletions osmtools/src/main/groovy/org/orbisgis/geoclimate/osmtools/Transform.groovy
View file
Original file line number Diff line number Diff line change
Expand Up @@ -319,8 +319,9 @@ String extractWaysAsPolygons(JdbcDataSource datasource, String osmTablesPrefix,
CREATE INDEX ON $waysPolygonTmp(id_way);
""".toString()

String query = """ DROP TABLE IF EXISTS $outputTableName;
CREATE TABLE $outputTableName AS
def allPolygonsTables = postfix "all_polygons_table"
String query = """ DROP TABLE IF EXISTS $allPolygonsTables;
CREATE TABLE $allPolygonsTables AS
SELECT 'w'||a.id_way AS id,"""
if (valid_geom) {
query += """ case when st_isvalid(a.the_geom) then a.the_geom else st_makevalid(a.the_geom) end as the_geom ${OSMTools.TransformUtils.createTagList(datasource, columnsSelector, columnsToKeep)}
Expand All @@ -335,27 +336,34 @@ String extractWaysAsPolygons(JdbcDataSource datasource, String osmTablesPrefix,
query += " AND b.TAG_KEY IN ('${columnsToKeep.join("','")}') "
}

//TODO : Due to some H2 limitations with the execution plan we create the whole table and then we apply a spatial filter
//https://github.com/orbisgis/geoclimate/issues/876
query += " GROUP BY a.id_way;"
datasource.execute(query)
if (geometry) {
def query_out ="""DROP TABLE IF EXISTS $outputTableName;
CREATE TABLE $outputTableName AS SELECT * FROM $allPolygonsTables as a where """
int geom_srid = geometry.getSRID()
if (geom_srid == -1) {
query += " and a.the_geom && st_setsrid('$geometry'::geometry, $epsgCode) and st_intersects(a.the_geom, st_setsrid('$geometry'::geometry, $epsgCode)) "
query_out += " a.the_geom && st_setsrid('$geometry'::geometry, $epsgCode) and st_intersects(a.the_geom, st_setsrid('$geometry'::geometry, $epsgCode)) "
} else if (geom_srid == epsgCode) {
query += " and a.the_geom && st_setsrid('$geometry'::geometry, $epsgCode) and st_intersects(a.the_geom,st_setsrid('$geometry'::geometry, $epsgCode)) "
query_out += " a.the_geom && st_setsrid('$geometry'::geometry, $epsgCode) and st_intersects(a.the_geom,st_setsrid('$geometry'::geometry, $epsgCode)) "
} else {
query += " and a.the_geom && st_transform(st_setsrid('$geometry'::geometry, $geom_srid), $epsgCode) and st_intersects(a.the_geom,st_transform(st_setsrid('$geometry'::geometry, $geom_srid), $epsgCode)) "
query_out += " a.the_geom && st_transform(st_setsrid('$geometry'::geometry, $geom_srid), $epsgCode) and st_intersects(a.the_geom,st_transform(st_setsrid('$geometry'::geometry, $geom_srid), $epsgCode)) "
}
datasource.createSpatialIndex(waysPolygonTmp, "the_geom")
datasource.createSpatialIndex(allPolygonsTables, "the_geom")
datasource """
$query_out;
DROP TABLE IF EXISTS $waysPolygonTmp, $idWaysPolygons, $allPolygonsTables;""".toString()
return outputTableName
}
query += " GROUP BY a.id_way;"


datasource """
$query
DROP TABLE IF EXISTS $waysPolygonTmp;
DROP TABLE IF EXISTS $idWaysPolygons;
else{
datasource """
ALTER TABLE $allPolygonsTables RENAME TO $outputTableName;
DROP TABLE IF EXISTS $waysPolygonTmp, $idWaysPolygons;
""".toString()

return outputTableName
return outputTableName
}
}

/**
Expand Down Expand Up @@ -560,9 +568,10 @@ def extractRelationsAsPolygons(JdbcDataSource datasource, String osmTablesPrefix
CREATE INDEX ON $relationsMpHoles(id_relation);
""".toString()

String allRelationPolygons = postfix("all_relation_polygons")
def query = """
DROP TABLE IF EXISTS $outputTableName;
CREATE TABLE $outputTableName AS
DROP TABLE IF EXISTS $allRelationPolygons;
CREATE TABLE $allRelationPolygons AS
SELECT 'r'||a.id_relation AS id,"""
if (valid_geom) {
query += """ case when st_isvalid(a.the_geom) then a.the_geom else st_makevalid(a.the_geom) end as the_geom ${OSMTools.TransformUtils.createTagList(datasource, columnsSelector, columnsToKeep)}
Expand All @@ -577,25 +586,32 @@ def extractRelationsAsPolygons(JdbcDataSource datasource, String osmTablesPrefix
if(columnsToKeep) {
query += " AND b.TAG_KEY IN ('${columnsToKeep.join("','")}') "
}
query += " GROUP BY a.the_geom, a.id_relation;"
datasource.execute(query.toString())

if (geometry) {
def query_out ="""DROP TABLE IF EXISTS $outputTableName;
CREATE TABLE $outputTableName as SELECT * FROM $allRelationPolygons as a where"""
int geom_srid = geometry.getSRID()
if (geom_srid == -1) {
query += " and a.the_geom && st_setsrid('$geometry'::geometry, $epsgCode) and st_intersects(a.the_geom, st_setsrid('$geometry'::geometry, $epsgCode)) "
query_out += " a.the_geom && st_setsrid('$geometry'::geometry, $epsgCode) and st_intersects(a.the_geom, st_setsrid('$geometry'::geometry, $epsgCode)) "
} else if (geom_srid == epsgCode) {
query += " and a.the_geom && st_setsrid('$geometry'::geometry, $epsgCode) and st_intersects(a.the_geom,st_setsrid('$geometry'::geometry, $epsgCode)) "
query_out += " a.the_geom && st_setsrid('$geometry'::geometry, $epsgCode) and st_intersects(a.the_geom,st_setsrid('$geometry'::geometry, $epsgCode)) "
} else {
query += " and a.the_geom && st_transform(st_setsrid('$geometry'::geometry, $geom_srid), $epsgCode) and st_intersects(a.the_geom,st_transform(st_setsrid('$geometry'::geometry, $geom_srid), $epsgCode)) "
query_out += " a.the_geom && st_transform(st_setsrid('$geometry'::geometry, $geom_srid), $epsgCode) and st_intersects(a.the_geom,st_transform(st_setsrid('$geometry'::geometry, $geom_srid), $epsgCode)) "
}
datasource.createSpatialIndex(relationsMpHoles, "the_geom")
datasource.createSpatialIndex(allRelationPolygons, "the_geom")
datasource.execute(query_out.toString())
datasource.dropTable(relationsPolygonsOuter, relationsPolygonsInner, relationsPolygonsOuterExploded,
relationsPolygonsInnerExploded,relationsMpHoles,relationFilteredKeys, allRelationPolygons)
return outputTableName
}else{
datasource.execute("""ALTER TABLE $allRelationPolygons RENAME TO $outputTableName""".toString())
datasource.dropTable(relationsPolygonsOuter, relationsPolygonsInner, relationsPolygonsOuterExploded,
relationsPolygonsInnerExploded,relationsMpHoles,relationFilteredKeys)
return outputTableName
}

query += " GROUP BY a.the_geom, a.id_relation;"

datasource query.toString()
datasource.dropTable(relationsPolygonsOuter, relationsPolygonsInner, relationsPolygonsOuterExploded,
relationsPolygonsInnerExploded,relationsMpHoles,relationFilteredKeys)
return outputTableName
}

/**
Expand Down Expand Up @@ -725,9 +741,11 @@ String extractWaysAsLines(JdbcDataSource datasource, String osmTablesPrefix, int
CREATE INDEX ON $waysLinesTmp(ID_WAY);
""".toString()

def allLinesTables = postfix "all_lines_table"

def query = """
DROP TABLE IF EXISTS $outputTableName;
CREATE TABLE $outputTableName AS
DROP TABLE IF EXISTS $allLinesTables;
CREATE TABLE $allLinesTables AS
SELECT 'w'||a.id_way AS id, a.the_geom ${OSMTools.TransformUtils.createTagList(datasource, columnsSelector,columnsToKeep)}
FROM $waysLinesTmp AS a, ${osmTablesPrefix}_way_tag b
WHERE a.id_way=b.id_way and st_isempty(a.the_geom)=false """
Expand All @@ -736,25 +754,33 @@ String extractWaysAsLines(JdbcDataSource datasource, String osmTablesPrefix, int
query += " AND b.TAG_KEY IN ('${columnsToKeep.join("','")}') "
}

query += " GROUP BY a.id_way;"
datasource.execute(query.toString())
if (geometry) {
def query_out = """DROP TABLE IF EXISTS $outputTableName;
CREATE TABLE $outputTableName as select * from $allLinesTables as a where """
int geom_srid = geometry.getSRID()
if (geom_srid == -1) {
query += " and a.the_geom && st_setsrid('$geometry'::geometry, $epsgCode) and st_intersects(a.the_geom, st_setsrid('$geometry'::geometry, $epsgCode)) "
query_out += " a.the_geom && st_setsrid('$geometry'::geometry, $epsgCode) and st_intersects(a.the_geom, st_setsrid('$geometry'::geometry, $epsgCode)) "
} else if (geom_srid == epsgCode) {
query += " and a.the_geom && st_setsrid('$geometry'::geometry, $epsgCode) and st_intersects(a.the_geom,st_setsrid('$geometry'::geometry, $epsgCode)) "
query_out += " a.the_geom && st_setsrid('$geometry'::geometry, $epsgCode) and st_intersects(a.the_geom,st_setsrid('$geometry'::geometry, $epsgCode)) "
} else {
query += " and a.the_geom && st_transform(st_setsrid('$geometry'::geometry, $geom_srid), $epsgCode) and st_intersects(a.the_geom,st_transform(st_setsrid('$geometry'::geometry, $geom_srid), $epsgCode)) "
query_out += " a.the_geom && st_transform(st_setsrid('$geometry'::geometry, $geom_srid), $epsgCode) and st_intersects(a.the_geom,st_transform(st_setsrid('$geometry'::geometry, $geom_srid), $epsgCode)) "
}
datasource.createSpatialIndex(waysLinesTmp, "the_geom")
}

query += " GROUP BY a.id_way;"

datasource """
$query
datasource.createSpatialIndex(allLinesTables, "the_geom")
datasource """
$query_out;
DROP TABLE IF EXISTS $waysLinesTmp, $idWaysTable, $allLinesTables;
""".toString()
return outputTableName
}else{
datasource """
ALTER TABLE $allLinesTables RENAME TO $outputTableName;
DROP TABLE IF EXISTS $waysLinesTmp, $idWaysTable;
""".toString()
return outputTableName
return outputTableName
}

}

/**
Expand Down Expand Up @@ -892,30 +918,38 @@ String extractRelationsAsLines(JdbcDataSource datasource, String osmTablesPrefix

def columnsSelector = OSMTools.TransformUtils.getColumnSelector(osmTableTag, tags, columnsToKeep)

def allRelationLines = postfix("all_relation_lines")
def query = """
DROP TABLE IF EXISTS $outputTableName;
CREATE TABLE $outputTableName AS
DROP TABLE IF EXISTS $allRelationLines;
CREATE TABLE $allRelationLines AS
SELECT 'r'||a.id_relation AS id, a.the_geom ${OSMTools.TransformUtils.createTagList(datasource, columnsSelector, columnsToKeep)}
FROM $relationsLinesTmp AS a, ${osmTablesPrefix}_relation_tag b
WHERE a.id_relation=b.id_relation and st_isempty(a.the_geom)=false """
if(columnsToKeep) {
query += " AND b.TAG_KEY IN ('${columnsToKeep.join("','")}') "
}

query += " GROUP BY a.id_relation;"
datasource.execute(query.toString())
if (geometry) {
def query_out =""" DROP TABLE IF EXISTS $outputTableName;
CREATE TABLE $outputTableName as SELECT * FROM $allRelationLines as a where """
int geom_srid = geometry.getSRID()
if (geom_srid == -1) {
query += " and a.the_geom && st_setsrid('$geometry'::geometry, $epsgCode) and st_intersects(a.the_geom, st_setsrid('$geometry'::geometry, $epsgCode)) "
query_out += " a.the_geom && st_setsrid('$geometry'::geometry, $epsgCode) and st_intersects(a.the_geom, st_setsrid('$geometry'::geometry, $epsgCode)) "
} else if (geom_srid == epsgCode) {
query += " and a.the_geom && st_setsrid('$geometry'::geometry, $epsgCode) and st_intersects(a.the_geom,st_setsrid('$geometry'::geometry, $epsgCode)) "
query_out += " a.the_geom && st_setsrid('$geometry'::geometry, $epsgCode) and st_intersects(a.the_geom,st_setsrid('$geometry'::geometry, $epsgCode)) "
} else {
query += " and a.the_geom && st_transform(st_setsrid('$geometry'::geometry, $geom_srid), $epsgCode) and st_intersects(a.the_geom,st_transform(st_setsrid('$geometry'::geometry, $geom_srid), $epsgCode)) "
query_out += " a.the_geom && st_transform(st_setsrid('$geometry'::geometry, $geom_srid), $epsgCode) and st_intersects(a.the_geom,st_transform(st_setsrid('$geometry'::geometry, $geom_srid), $epsgCode)) "
}
datasource.createSpatialIndex(relationsLinesTmp, "the_geom")
}
query += " GROUP BY a.id_relation;"
datasource.execute(""" $query
datasource.createSpatialIndex(allRelationLines, "the_geom")
datasource.execute(""" $query_out ;
DROP TABLE IF EXISTS $relationsLinesTmp, $relationsFilteredKeys, $allRelationLines;
""".toString())
return outputTableName
}else{
datasource.execute(""" ALTER TABLE $allRelationLines RENAME TO $outputTableName;
DROP TABLE IF EXISTS $relationsLinesTmp, $relationsFilteredKeys;
""".toString())
return outputTableName
return outputTableName
}
}

0 comments on commit 3eafd5a

Please sign in to comment.