Skip to content

Commit

Permalink
Merge pull request #871 from ebocher/couple_of_fixes
Browse files Browse the repository at this point in the history 
Many improvements on OSM module
  • Loading branch information
@ebocher
ebocher authored Nov 8, 2023
2 parents 5b19691 + b46ddd8 commit 185f6b4
Show file tree
Hide file tree
Showing 4 changed files with 76 additions and 72 deletions.
41 changes: 18 additions & 23 deletions osm/src/main/groovy/org/orbisgis/geoclimate/osm/InputDataFormatting.groovy
View file
Original file line number Diff line number Diff line change
Expand Up @@ -44,7 +44,7 @@ import java.util.regex.Pattern
* @return outputTableName The name of the final buildings table
* @return outputEstimatedTableName The name of the table containing the state of estimation for each building
*/
Map formatBuildingLayer(JdbcDataSource datasource, String building, String zone = "", String urban_areas = "", int h_lev_min = 3, String jsonFilename = null) {
Map formatBuildingLayer(JdbcDataSource datasource, String building, String zone = "", String urban_areas = "", int h_lev_min = 3, String jsonFilename = "") {
if (!h_lev_min) {
h_lev_min = 3
}
Expand Down Expand Up @@ -1056,7 +1056,6 @@ String formatSeaLandMask(JdbcDataSource datasource, String coastline, String zon
def coastLinesIntersectsPoints = "coatline_intersect_points_zone${UUID.randomUUID().toString().replaceAll("-", "_")}"
def coastLinesPoints = "coatline_points_zone${UUID.randomUUID().toString().replaceAll("-", "_")}"
def sea_land_mask = "sea_land_mask${UUID.randomUUID().toString().replaceAll("-", "_")}"
def sea_land_mask_in_zone = "sea_land_mask_in_zone${UUID.randomUUID().toString().replaceAll("-", "_")}"
def water_to_be_filtered = "water_to_be_filtered${UUID.randomUUID().toString().replaceAll("-", "_")}"
def water_filtered_exploded = "water_filtered_exploded${UUID.randomUUID().toString().replaceAll("-", "_")}"
def sea_land_triangles = "sea_land_triangles${UUID.randomUUID().toString().replaceAll("-", "_")}"
Expand All @@ -1066,10 +1065,10 @@ String formatSeaLandMask(JdbcDataSource datasource, String coastline, String zon
datasource.createSpatialIndex(coastline, "the_geom")
datasource.execute """DROP TABLE IF EXISTS $coastLinesIntersects,
$islands_mark, $mergingDataTable, $coastLinesIntersectsPoints, $coastLinesPoints,$sea_land_mask,
$sea_land_mask_in_zone,$water_filtered_exploded,$water_to_be_filtered, $sea_land_triangles, $sea_id_triangles, $water_id_triangles;
$water_filtered_exploded,$water_to_be_filtered, $sea_land_triangles, $sea_id_triangles, $water_id_triangles;
CREATE TABLE $coastLinesIntersects AS SELECT ST_intersection(a.the_geom, b.the_geom) as the_geom
from $coastline AS a, $zone AS b WHERE
a.the_geom && b.the_geom AND st_intersects(a.the_geom, b.the_geom);
a.the_geom && b.the_geom AND st_intersects(a.the_geom, b.the_geom) and "natural"= 'coastline';
""".toString()

if (water) {
Expand All @@ -1088,17 +1087,12 @@ String formatSeaLandMask(JdbcDataSource datasource, String coastline, String zon
UNION ALL
SELECT st_tomultiline(the_geom)
from $water ;
CREATE TABLE $sea_land_mask (THE_GEOM GEOMETRY,ID serial, TYPE VARCHAR, ZINDEX INTEGER) AS SELECT THE_GEOM, EXPLOD_ID, 'land', 0 AS ZINDEX FROM
st_explode('(SELECT st_polygonize(st_union(ST_NODE(st_accum(the_geom)))) AS the_geom FROM $mergingDataTable)'); """.toString()
st_explode('(SELECT st_polygonize(st_union(ST_NODE(st_accum(the_geom)))) AS the_geom FROM $mergingDataTable)')
as foo where ST_DIMENSION(the_geom) = 2 AND st_area(the_geom) >0; """.toString()

datasource.execute """
CREATE SPATIAL INDEX IF NOT EXISTS ${sea_land_mask}_the_geom_idx ON $sea_land_mask (THE_GEOM);
CREATE TABLE $sea_land_mask_in_zone as select the_geom, id, type, ZINDEX
from st_explode('(SELECT st_intersection(a.THE_GEOM, b.the_geom) as the_geom, a.id, a.type,a.ZINDEX
FROM $sea_land_mask as a, $zone as b WHERE a.the_geom && b.the_geom AND st_intersects(a.the_geom, b.the_geom))')
where ST_DIMENSION(the_geom) = 2 AND st_area(the_geom) >0;
CREATE SPATIAL INDEX IF NOT EXISTS ${islands_mark}_the_geom_idx ON $islands_mark (THE_GEOM);
Expand All @@ -1116,19 +1110,23 @@ String formatSeaLandMask(JdbcDataSource datasource, String coastline, String zon
CREATE TABLE $sea_land_triangles AS
SELECT * FROM
st_explode('(SELECT CASE WHEN ST_AREA(THE_GEOM) > 100000 THEN ST_Tessellate(the_geom) ELSE THE_GEOM END AS THE_GEOM,
ID, TYPE, ZINDEX FROM $sea_land_mask_in_zone)');
ID, TYPE, ZINDEX FROM $sea_land_mask)');
CREATE SPATIAL INDEX IF NOT EXISTS ${sea_land_triangles}_the_geom_idx ON $sea_land_triangles (THE_GEOM);
DROP TABLE IF EXISTS $sea_id_triangles;
CREATE TABLE $sea_id_triangles AS SELECT DISTINCT a.id FROM $sea_land_triangles a,
$coastLinesIntersectsPoints b WHERE a.THE_GEOM && b.THE_GEOM AND
st_contains(a.THE_GEOM, b.THE_GEOM);
CREATE INDEX ON $sea_id_triangles (id);
--Update sea triangles
st_intersects(a.THE_GEOM, b.THE_GEOM);
CREATE INDEX ON $sea_id_triangles (id);""".toString()

//Set the triangles to sea
datasource.execute """
UPDATE ${sea_land_triangles} SET TYPE='sea' WHERE ID IN(SELECT ID FROM $sea_id_triangles);
""".toString()

//Set the triangles to water
datasource.execute """
DROP TABLE IF EXISTS $water_id_triangles;
CREATE TABLE $water_id_triangles AS SELECT a.ID
FROM ${sea_land_triangles} a, $water b WHERE a.THE_GEOM && b.THE_GEOM AND
Expand Down Expand Up @@ -1160,13 +1158,10 @@ String formatSeaLandMask(JdbcDataSource datasource, String coastline, String zon
from $zone ;
CREATE TABLE $sea_land_mask (THE_GEOM GEOMETRY,ID serial, TYPE VARCHAR, ZINDEX INTEGER) AS SELECT THE_GEOM, EXPLOD_ID, 'land', 0 AS ZINDEX FROM
st_explode('(SELECT st_polygonize(st_union(ST_NODE(st_accum(the_geom)))) AS the_geom FROM $mergingDataTable)');
st_explode('(SELECT st_polygonize(st_union(ST_NODE(st_accum(the_geom)))) AS the_geom FROM $mergingDataTable)') as foo where ST_DIMENSION(the_geom) = 2 AND st_area(the_geom) >0;
CREATE SPATIAL INDEX IF NOT EXISTS ${sea_land_mask}_the_geom_idx ON $sea_land_mask (THE_GEOM);
CREATE TABLE $sea_land_mask_in_zone as select the_geom, id, type, ZINDEX from st_explode('(SELECT st_intersection(a.THE_GEOM, b.the_geom) as the_geom, a.id, a.type,a.ZINDEX
FROM $sea_land_mask as a, $zone as b where a.the_geom && b.the_geom AND st_intersects(a.the_geom, b.the_geom))') where ST_DIMENSION(the_geom) = 2 AND st_area(the_geom) >0;
CREATE SPATIAL INDEX IF NOT EXISTS ${islands_mark}_the_geom_idx ON $islands_mark (THE_GEOM);
CREATE TABLE $coastLinesPoints as SELECT ST_LocateAlong(the_geom, 0.5, -0.01) AS the_geom FROM
Expand All @@ -1186,7 +1181,7 @@ String formatSeaLandMask(JdbcDataSource datasource, String coastline, String zon
CREATE TABLE $sea_land_triangles AS
SELECT * FROM
st_explode('(SELECT CASE WHEN ST_AREA(THE_GEOM) > 100000 THEN ST_Tessellate(the_geom) ELSE THE_GEOM END AS THE_GEOM,
ID, TYPE, ZINDEX FROM $sea_land_mask_in_zone)');
ID, TYPE, ZINDEX FROM $sea_land_mask)');
CREATE SPATIAL INDEX IF NOT EXISTS ${sea_land_triangles}_the_geom_idx ON $sea_land_triangles (THE_GEOM);
Expand All @@ -1209,7 +1204,7 @@ String formatSeaLandMask(JdbcDataSource datasource, String coastline, String zon

datasource.execute("""DROP TABLE IF EXISTS $coastLinesIntersects,
$islands_mark, $mergingDataTable, $coastLinesIntersectsPoints, $coastLinesPoints,$sea_land_mask,
$sea_land_mask_in_zone,$water_filtered_exploded,$water_to_be_filtered, $sea_land_triangles, $sea_id_triangles, $water_id_triangles
$water_filtered_exploded,$water_to_be_filtered, $sea_land_triangles, $sea_id_triangles, $water_id_triangles
""".toString())
debug 'The sea/land mask has been computed'
return outputTableName
Expand Down
38 changes: 29 additions & 9 deletions osm/src/main/resources/org/orbisgis/geoclimate/osm/buildingParams.json
View file
Original file line number Diff line number Diff line change
@@ -1,10 +1,6 @@
{
"tags": {
"building": [],
"railway": [
"station",
"train_station"
]
"building": []
},
"columns": [
"height",
Expand Down Expand Up @@ -183,7 +179,7 @@
"! church"
]
},
"sport:entertainment_arts_culture": {
"sport:sport": {
"building": [
"swimming_pool",
"fitness_centre",
Expand Down Expand Up @@ -212,7 +208,7 @@
"track"
]
},
"sports_centre:entertainment_arts_culture": {
"sports_centre:sport": {
"building": [
"sports_centre",
"sports_hall"
Expand Down Expand Up @@ -344,7 +340,7 @@
"kiosk",
"money_transfer",
"market",
"market_place",
"marketplace",
"pharmacy",
"post_office",
"retail",
Expand Down Expand Up @@ -519,6 +515,16 @@
"! no"
]
},
"shelter": {
"amenity": [
"shelter"
]
},
"shelter:transport": {
"shelter_type": [
"public_transport"
]
},
"transport": {
"building": [
"train_station",
Expand All @@ -535,7 +541,8 @@
"railway": [
"station",
"train_station",
"building"
"roundhouse"

],
"public_transport": [
"train_station",
Expand Down Expand Up @@ -585,6 +592,19 @@
"entertainment_arts_culture": {
"leisure": [
"! no"
], "amenity": [
"cinema","arts_centre", "brothel", "casino",
"community_centre", "conference_centre",
"events_venue", "exhibition_centre", "gambling",
"music_venue", "nightclub", "planetarium",
"social_centre","studio", "theatre"
],
"building": [
"cinema","arts_centre", "brothel", "casino",
"community_centre", "conference_centre",
"events_venue", "exhibition_centre", "gambling",
"music_venue", "nightclub", "planetarium",
"social_centre","studio", "theatre"
]
},
"sustenance:commercial": {
Expand Down
55 changes: 21 additions & 34 deletions osm/src/test/groovy/org/orbisgis/geoclimate/osm/InputDataFormattingTest.groovy
View file
Original file line number Diff line number Diff line change
Expand Up @@ -216,8 +216,8 @@ class InputDataFormattingTest {

//Sea/Land mask
String inputSeaLandTableName = OSM.InputDataFormatting.formatSeaLandMask(h2GIS, extractData.coastline, zoneEnvelopeTableName)
assertEquals(4, h2GIS.getTable(inputSeaLandTableName).getRowCount())
assertTrue h2GIS.firstRow("select count(*) as count from ${inputSeaLandTableName} where type='land'").count == 3
assertEquals(2, h2GIS.getTable(inputSeaLandTableName).getRowCount())
assertTrue h2GIS.firstRow("select count(*) as count from ${inputSeaLandTableName} where type='land'").count == 1
h2GIS.getTable(inputSeaLandTableName).save(new File(folder, "osm_sea_land.geojson").getAbsolutePath(), true)
}

Expand Down Expand Up @@ -309,11 +309,10 @@ class InputDataFormattingTest {
//aeroway Toulouse https://www.openstreetmap.org/way/739797641#map=14/43.6316/1.3590
zoneToExtract = [43.610539, 1.334152, 43.648808, 1.392689]

zoneToExtract = "Göteborgs Stad"
zoneToExtract = "Göteborgs Stad"

zoneToExtract = "Barcelona Eixample"

//zoneToExtract = [69.24666, 28.3359, 69.32999, 28.41923]
zoneToExtract = "Riantec"
zoneToExtract =[45.575525,5.913734,45.578859,5.919549]

Map extractData = OSM.InputDataLoading.extractAndCreateGISLayers(h2GIS, zoneToExtract)

Expand All @@ -332,37 +331,21 @@ class InputDataFormattingTest {
h2GIS.getTable(extractData.zone_envelope).save("${file.absolutePath + File.separator}osm_zone_envelope_${formatedPlaceName}.geojson", true)

//Urban Areas
/*format = OSM.InputDataFormatting.formatUrbanAreas()
format.execute([
datasource : h2GIS,
inputTableName : extractData.results.urbanAreasTableName,
inputZoneEnvelopeTableName: extractData.results.zoneEnvelopeTableName,
epsg : epsg])
def urbanAreasTableName = format.results.outputTableName;
h2GIS.getTable(format.results.outputTableName).save("./target/osm_urban_areas_${formatedPlaceName}.geojson", true)
def inputUrbanAreas = OSM.InputDataFormatting.formatUrbanAreas(h2GIS,
extractData.urban_areas,extractData.zone)
h2GIS.save(inputUrbanAreas,"${file.absolutePath + File.separator}osm_urban_areas_${formatedPlaceName}.geojson", true)

//Buildings
format = OSM.InputDataFormatting.formatBuildingLayer()
format.execute([
datasource : h2GIS,
inputTableName : extractData.results.buildingTableName,
inputZoneEnvelopeTableName: extractData.results.zoneEnvelopeTableName,
epsg : epsg,
urbanAreasTableName : urbanAreasTableName])
h2GIS.getTable(format.results.outputTableName).save("./target/osm_building_${formatedPlaceName}.geojson", true)
assertTrue h2GIS.firstRow("select count(*) as count from ${format.results.outputTableName} where NB_LEV is null").count == 0
assertTrue h2GIS.firstRow("select count(*) as count from ${format.results.outputTableName} where NB_LEV<0").count == 0
assertTrue h2GIS.firstRow("select count(*) as count from ${format.results.outputTableName} where HEIGHT_WALL is null").count == 0
assertTrue h2GIS.firstRow("select count(*) as count from ${format.results.outputTableName} where HEIGHT_WALL<0").count == 0
assertTrue h2GIS.firstRow("select count(*) as count from ${format.results.outputTableName} where HEIGHT_ROOF is null").count == 0
assertTrue h2GIS.firstRow("select count(*) as count from ${format.results.outputTableName} where HEIGHT_ROOF<0").count == 0
h2GIS.save(extractData.building,"${file.absolutePath + File.separator}building_${formatedPlaceName}.geojson", true)
def inputBuildings = OSM.InputDataFormatting.formatBuildingLayer(h2GIS,
extractData.building,extractData.zone,inputUrbanAreas)
h2GIS.save(inputBuildings.building,"${file.absolutePath + File.separator}osm_building_${formatedPlaceName}.geojson", true)


//Roads

*/
def inputRoadTableName = OSM.InputDataFormatting.formatRoadLayer( h2GIS,extractData.road, extractData.zone_envelope)
h2GIS.getTable(inputRoadTableName).save("${file.absolutePath + File.separator}osm_road_${formatedPlaceName}.geojson", true)
h2GIS.save(inputRoadTableName,"${file.absolutePath + File.separator}osm_road_${formatedPlaceName}.geojson", true)

//Rails
/*format = OSM.InputDataFormatting.formatRailsLayer()
Expand All @@ -377,22 +360,26 @@ class InputDataFormattingTest {
//Vegetation
def inputVegetationTableName = OSM.InputDataFormatting.formatVegetationLayer(
h2GIS,extractData.vegetation,extractData.zone_envelope)
h2GIS.getTable(inputVegetationTableName).save("${file.absolutePath + File.separator}osm_vegetation_${formatedPlaceName}.geojson", true)
h2GIS.save(inputVegetationTableName,"${file.absolutePath + File.separator}osm_vegetation_${formatedPlaceName}.geojson", true)


//Hydrography
def inputWaterTableName = OSM.InputDataFormatting.formatWaterLayer(h2GIS, extractData.water, extractData.zone_envelope)
h2GIS.getTable(inputWaterTableName).save("${file.absolutePath + File.separator}osm_water_${formatedPlaceName}.geojson", true)
h2GIS.save(inputWaterTableName,"${file.absolutePath + File.separator}osm_water_${formatedPlaceName}.geojson", true)

//Impervious
String imperviousTable = OSM.InputDataFormatting.formatImperviousLayer(h2GIS, extractData.impervious,
extractData.zone_envelope)
h2GIS.getTable(imperviousTable).save("${file.absolutePath + File.separator}osm_impervious_${formatedPlaceName}.geojson", true)
h2GIS.save(imperviousTable,"${file.absolutePath + File.separator}osm_impervious_${formatedPlaceName}.geojson", true)

//Save coastlines to debug
h2GIS.save(extractData.coastline,"${file.absolutePath + File.separator}osm_coastlines_${formatedPlaceName}.geojson", true)


//Sea/Land mask
def inputSeaLandTableName = OSM.InputDataFormatting.formatSeaLandMask(h2GIS, extractData.coastline,
extractData.zone_envelope, inputWaterTableName)
h2GIS.getTable(inputSeaLandTableName).save("${file.absolutePath + File.separator}osm_sea_land_${formatedPlaceName}.geojson", true)
h2GIS.save(inputSeaLandTableName,"${file.absolutePath + File.separator}osm_sea_land_${formatedPlaceName}.geojson", true)

} else {
assertTrue(false)
Expand Down
14 changes: 8 additions & 6 deletions osm/src/test/groovy/org/orbisgis/geoclimate/osm/WorflowOSMTest.groovy
View file
Original file line number Diff line number Diff line change
Expand Up @@ -29,7 +29,6 @@ import org.junit.jupiter.api.io.TempDir
import org.orbisgis.data.H2GIS
import org.orbisgis.data.POSTGIS
import org.orbisgis.geoclimate.Geoindicators
import org.orbisgis.geoclimate.osmtools.OSMTools

import static org.junit.jupiter.api.Assertions.*

Expand Down Expand Up @@ -651,13 +650,14 @@ class WorflowOSMTest extends WorkflowAbstractTest {
File dirFile = new File(directory)
dirFile.delete()
dirFile.mkdir()
def location = "Nice"

def location = "Redon"
//def nominatim = OSMTools.Utilities.getNominatimData("Nantes")

location=[47.2, -1.6, 47.4, -1.4]
//location = nominatim.bbox

def nominatim = OSMTools.Utilities.getNominatimData("Redon")

location = nominatim.bbox
location = [33.8, 7.6, 34, 7.8]
def osm_parmeters = [
"description" : "Example of configuration file to run the OSM workflow and store the result in a folder",
"geoclimatedb": [
Expand All @@ -677,7 +677,9 @@ class WorflowOSMTest extends WorkflowAbstractTest {
"parameters" :
["distance" : 0,
"rsu_indicators" : [

"indicatorUse": ["LCZ", "TEB"] //, "UTRF", "TEB"]

]/*,"grid_indicators": [
"x_size": 200,
"y_size": 200,
Expand All @@ -688,7 +690,7 @@ class WorflowOSMTest extends WorkflowAbstractTest {
"ROAD_FRACTION", "IMPERVIOUS_FRACTION",
"LCZ_PRIMARY",
//"BUILDING_HEIGHT_WEIGHTED", //"BUILDING_SURFACE_DENSITY", "SEA_LAND_FRACTION",
"ASPECT_RATIO","SVF",
"ASPECT_RATIO",//"SVF",
"HEIGHT_OF_ROUGHNESS_ELEMENTS", "TERRAIN_ROUGHNESS_CLASS"]
], "worldpop_indicators": true,
"road_traffic" : true,
Expand Down

0 comments on commit 185f6b4

Please sign in to comment.