diff --git a/bdtopo/src/main/groovy/org/orbisgis/geoclimate/bdtopo/BDTopoV2Workflow.groovy b/bdtopo/src/main/groovy/org/orbisgis/geoclimate/bdtopo/BDTopoV2Workflow.groovy index 61ddb3a48e..caaeffafd6 100644 --- a/bdtopo/src/main/groovy/org/orbisgis/geoclimate/bdtopo/BDTopoV2Workflow.groovy +++ b/bdtopo/src/main/groovy/org/orbisgis/geoclimate/bdtopo/BDTopoV2Workflow.groovy @@ -126,7 +126,11 @@ Integer loadDataFromPostGIS(Object input_database_properties, Object code, Objec def outputTableNameRoad = "ROUTE" if (inputTables.route) { //Extract route - def inputTableName = "(SELECT ID, st_setsrid(the_geom, $commune_srid) as the_geom, NATURE, LARGEUR, POS_SOL, FRANCHISST, SENS FROM ${inputTables.route} WHERE st_setsrid(the_geom, $commune_srid) && 'SRID=$commune_srid;$geomToExtract'::GEOMETRY AND ST_INTERSECTS(st_setsrid(the_geom, $commune_srid), 'SRID=$commune_srid;$geomToExtract'::GEOMETRY))" + def inputTableName = """(SELECT ID, st_setsrid(the_geom, $commune_srid) as the_geom, NATURE, LARGEUR, POS_SOL, + FRANCHISST, SENS, IMPORTANCE, CL_ADMIN FROM ${inputTables.route} WHERE + st_setsrid(the_geom, $commune_srid) && 'SRID=$commune_srid;$geomToExtract'::GEOMETRY + AND ST_INTERSECTS(st_setsrid(the_geom, $commune_srid), 'SRID=$commune_srid;$geomToExtract'::GEOMETRY) + AND NATURE NOT IN ('Bac auto', 'Bac piéton', 'Escalier'))""".toString() logger.debug "Loading in the H2GIS database $outputTableNameRoad" IOMethods.exportToDataBase(sourceConnection, inputTableName, h2gis_datasource.getConnection(), outputTableNameRoad, -1, 1000) } else { @@ -341,7 +345,12 @@ def filterLinkedShapeFiles(def location, float distance, LinkedHashMap inputTabl //Extract route outputTableName = "ROUTE" logger.debug "Loading in the H2GIS database $outputTableName" - h2gis_datasource.execute("DROP TABLE IF EXISTS $outputTableName ; CREATE TABLE $outputTableName as SELECT ID, $formatting_geom, NATURE, LARGEUR, POS_SOL, FRANCHISST, SENS FROM ${inputTables.route} WHERE the_geom && 'SRID=$sourceSRID;$geomToExtract'::GEOMETRY AND ST_INTERSECTS(the_geom, 'SRID=$sourceSRID;$geomToExtract'::GEOMETRY)".toString()) + h2gis_datasource.execute("""DROP TABLE IF EXISTS $outputTableName ; + CREATE TABLE $outputTableName as SELECT ID, $formatting_geom, NATURE, LARGEUR, POS_SOL, FRANCHISST, SENS, + IMPORTANCE, CL_ADMIN FROM ${inputTables.route} + WHERE the_geom && 'SRID=$sourceSRID;$geomToExtract'::GEOMETRY + AND ST_INTERSECTS(the_geom, 'SRID=$sourceSRID;$geomToExtract'::GEOMETRY) + AND NATURE NOT IN ('Bac auto', 'Bac piéton', 'Escalier')""".toString()) } else { logger.error "The route table must be provided" return diff --git a/bdtopo/src/main/groovy/org/orbisgis/geoclimate/bdtopo/BDTopoV3Workflow.groovy b/bdtopo/src/main/groovy/org/orbisgis/geoclimate/bdtopo/BDTopoV3Workflow.groovy index db1157a506..0a01553d20 100644 --- a/bdtopo/src/main/groovy/org/orbisgis/geoclimate/bdtopo/BDTopoV3Workflow.groovy +++ b/bdtopo/src/main/groovy/org/orbisgis/geoclimate/bdtopo/BDTopoV3Workflow.groovy @@ -53,13 +53,27 @@ def filterLinkedShapeFiles(def location, float distance, LinkedHashMap inputTabl //The zone is a osm bounding box represented by ymin,xmin , ymax,xmax, if (location in Collection) { debug "Loading in the H2GIS database $outputTableName" - h2gis_datasource.execute("""DROP TABLE IF EXISTS $outputTableName ; CREATE TABLE $outputTableName as SELECT + def communeColumns = h2gis_datasource.getColumnNames(inputTables.commune) + if(communeColumns.contains("INSEE_COM")) { + h2gis_datasource.execute("""DROP TABLE IF EXISTS $outputTableName ; CREATE TABLE $outputTableName as SELECT ST_INTERSECTION(the_geom, ST_MakeEnvelope(${location[1]},${location[0]},${location[3]},${location[2]}, $sourceSRID)) as the_geom, INSEE_COM AS CODE_INSEE from ${inputTables.commune} where the_geom && ST_MakeEnvelope(${location[1]},${location[0]},${location[3]},${location[2]}, $sourceSRID) """.toString()) + } + else { + error "Cannot find a column insee_com or code_insee to filter the commune" + return + } } else if (location instanceof String) { logger.debug "Loading in the H2GIS database $outputTableName" + def communeColumns = h2gis_datasource.getColumnNames(inputTables.commune) + if(communeColumns.contains("INSEE_COM")){ h2gis_datasource.execute("""DROP TABLE IF EXISTS $outputTableName ; CREATE TABLE $outputTableName as SELECT $formatting_geom, INSEE_COM AS CODE_INSEE FROM ${inputTables.commune} WHERE INSEE_COM='$location' or lower(nom)='${location.toLowerCase()}'""".toString()) + } + else { + error "Cannot find a column insee_com or code_insee to filter the commune" + return + } } else { @@ -87,8 +101,12 @@ def filterLinkedShapeFiles(def location, float distance, LinkedHashMap inputTabl outputTableName = "troncon_de_route" logger.debug "Loading in the H2GIS database $outputTableName" h2gis_datasource.execute("""DROP TABLE IF EXISTS $outputTableName ; - CREATE TABLE $outputTableName as SELECT ID, $formatting_geom, NATURE, LARGEUR, POS_SOL, SENS FROM ${inputTables.troncon_de_route} - WHERE the_geom && 'SRID=$sourceSRID;$geomToExtract'::GEOMETRY AND ST_INTERSECTS(the_geom, 'SRID=$sourceSRID;$geomToExtract'::GEOMETRY)""".toString()) + CREATE TABLE $outputTableName as SELECT ID, $formatting_geom, NATURE, LARGEUR, POS_SOL, SENS, + IMPORTANCE, CL_ADMIN, NAT_RESTR FROM ${inputTables.troncon_de_route} + WHERE the_geom && 'SRID=$sourceSRID;$geomToExtract'::GEOMETRY + AND ST_INTERSECTS(the_geom, 'SRID=$sourceSRID;$geomToExtract'::GEOMETRY) + AND NATURE NOT IN ('Bac ou liaison maritime', 'Escalier') + """.toString()) } else { logger.error "The troncon_de_route table must be provided" return @@ -241,18 +259,30 @@ Integer loadDataFromPostGIS(Object input_database_properties, Object code, Objec //Check if code is a string or a bbox //The zone is a osm bounding box represented by ymin,xmin , ymax,xmax, if (code in Collection) { - //def tmp_insee = code.join("_") - String inputTableName = """(SELECT - ST_INTERSECTION(st_setsrid(the_geom, $commune_srid), ST_MakeEnvelope(${code[1]},${code[0]},${code[3]},${code[2]}, $commune_srid)) as the_geom, CODE_INSEE from $commune_location where + def communeColumns = h2gis_datasource.getColumnNames(commune_location) + if(communeColumns.contains("INSEE_COM")) { + String inputTableName = """(SELECT + ST_INTERSECTION(st_setsrid(the_geom, $commune_srid), ST_MakeEnvelope(${code[1]},${code[0]},${code[3]},${code[2]}, $commune_srid)) as the_geom, INSEE_COM as CODE_INSEE from $commune_location where st_setsrid(the_geom, $commune_srid) && ST_MakeEnvelope(${code[1]},${code[0]},${code[3]},${code[2]}, $commune_srid) and st_intersects(st_setsrid(the_geom, $commune_srid), ST_MakeEnvelope(${code[1]},${code[0]},${code[3]},${code[2]}, $commune_srid)))""".toString() - logger.debug "Loading in the H2GIS database $outputTableName" - IOMethods.exportToDataBase(sourceConnection, inputTableName, h2gis_datasource.getConnection(), outputTableName, -1, 10) + logger.debug "Loading in the H2GIS database $outputTableName" + IOMethods.exportToDataBase(sourceConnection, inputTableName, h2gis_datasource.getConnection(), outputTableName, -1, 100) + }else { + error "Cannot find a column insee_com or code_insee to filter the commune" + return + } } else if (code instanceof String) { - String inputTableName = "(SELECT st_setsrid(the_geom, $commune_srid) as the_geom, CODE_INSEE FROM $commune_location WHERE CODE_INSEE='$code' or lower(nom)='${code.toLowerCase()}')" - logger.debug "Loading in the H2GIS database $outputTableName" - IOMethods.exportToDataBase(sourceConnection, inputTableName, h2gis_datasource.getConnection(), outputTableName, -1, 1000) + def communeColumns = h2gis_datasource.getColumnNames(commune_location) + if(communeColumns.contains("insee_com")){ + String inputTableName = "(SELECT st_setsrid(the_geom, $commune_srid) as the_geom, INSEE_COM as CODE_INSEE FROM $commune_location WHERE INSEE_COM='$code' or lower(nom)='${code.toLowerCase()}')" + logger.debug "Loading in the H2GIS database $outputTableName" + IOMethods.exportToDataBase(sourceConnection, inputTableName, h2gis_datasource.getConnection(), outputTableName, -1, 1000) + } + else { + error "Cannot find a column insee_com to filter the commune" + return + } } def count = h2gis_datasource."$outputTableName".rowCount if (count > 0) { @@ -268,7 +298,11 @@ Integer loadDataFromPostGIS(Object input_database_properties, Object code, Objec def outputTableNameRoad = "troncon_de_route" if (inputTables.troncon_de_route) { //Extract route - def inputTableName = "(SELECT ID, st_setsrid(the_geom, $commune_srid) as the_geom, NATURE, LARGEUR, POS_SOL, FRANCHISST, SENS FROM ${inputTables.troncon_de_route} WHERE st_setsrid(the_geom, $commune_srid) && 'SRID=$commune_srid;$geomToExtract'::GEOMETRY AND ST_INTERSECTS(st_setsrid(the_geom, $commune_srid), 'SRID=$commune_srid;$geomToExtract'::GEOMETRY))" + def inputTableName = """(SELECT ID, st_setsrid(the_geom, $commune_srid) as the_geom, NATURE, LARGEUR, POS_SOL, + FRANCHISST, SENS, IMPORTANCE, CL_ADMIN, NAT_RESTR FROM ${inputTables.troncon_de_route} + WHERE st_setsrid(the_geom, $commune_srid) && 'SRID=$commune_srid;$geomToExtract'::GEOMETRY + AND ST_INTERSECTS(st_setsrid(the_geom, $commune_srid), 'SRID=$commune_srid;$geomToExtract'::GEOMETRY) + AND NATURE NOT IN ('Bac ou liaison maritime', 'Escalier'))""".toString() logger.debug "Loading in the H2GIS database $outputTableNameRoad" IOMethods.exportToDataBase(sourceConnection, inputTableName, h2gis_datasource.getConnection(), outputTableNameRoad, -1, 1000) } else { diff --git a/bdtopo/src/main/groovy/org/orbisgis/geoclimate/bdtopo/InputDataFormatting.groovy b/bdtopo/src/main/groovy/org/orbisgis/geoclimate/bdtopo/InputDataFormatting.groovy index cc88081f76..5fb41c4fcb 100644 --- a/bdtopo/src/main/groovy/org/orbisgis/geoclimate/bdtopo/InputDataFormatting.groovy +++ b/bdtopo/src/main/groovy/org/orbisgis/geoclimate/bdtopo/InputDataFormatting.groovy @@ -56,7 +56,7 @@ String formatBuildingLayer(JdbcDataSource datasource, String building, String zo def queryMapper = "SELECT " if (zone) { datasource.createSpatialIndex(building, "the_geom") - datasource.createSpatialIndex(zone,"the_geom") + datasource.createSpatialIndex(zone, "the_geom") queryMapper += " a.* FROM $building as a, $zone as b WHERE a.the_geom && b.the_geom and st_intersects(a.the_geom " + ",b.the_geom) " } else { @@ -97,7 +97,7 @@ String formatBuildingLayer(JdbcDataSource datasource, String building, String zo "Annexe" : ["annex": "building"], "Industriel, agricole ou commercial": ["commercial": "commercial"], "Bâtiment" : ["building": "building"], - "Industrie lourde" :["heavy_industry":"industrial"] + "Industrie lourde" : ["heavy_industry": "industrial"] ] def building_type_level = ["building" : 1, @@ -122,7 +122,7 @@ String formatBuildingLayer(JdbcDataSource datasource, String building, String zo "sport" : 0, "sports_centre" : 0, "grandstand" : 0, - "transport" : 0, + "transport" : 0, "train_station" : 0, "toll_booth" : 0, "terminal" : 0, @@ -171,7 +171,7 @@ String formatBuildingLayer(JdbcDataSource datasource, String building, String zo def srid = geom.getSRID() for (int i = 0; i < geom.getNumGeometries(); i++) { Geometry subGeom = geom.getGeometryN(i) - if (subGeom instanceof Polygon && subGeom.getArea()>1) { + if (subGeom instanceof Polygon && subGeom.getArea() > 1) { stmt.addBatch """ INSERT INTO ${outputTableName} values( ST_GEOMFROMTEXT('${subGeom}',$srid), @@ -193,10 +193,10 @@ String formatBuildingLayer(JdbcDataSource datasource, String building, String zo } //Let's use the urban areas table to improve building qualification if (urban_areas) { - datasource.createSpatialIndex(outputTableName,"the_geom") - datasource.createIndex(outputTableName,"id_build") - datasource.createIndex(outputTableName,"type") - datasource.createSpatialIndex(urban_areas,"the_geom") + datasource.createSpatialIndex(outputTableName, "the_geom") + datasource.createIndex(outputTableName, "id_build") + datasource.createIndex(outputTableName, "type") + datasource.createSpatialIndex(urban_areas, "the_geom") def buildinType = postfix("BUILDING_TYPE") datasource.execute """create table $buildinType as SELECT @@ -279,6 +279,7 @@ String formatRoadLayer(JdbcDataSource datasource, String road, String zone = "") //Define the mapping between the values in BDTopo and those used in the abstract model def road_types = ["Autoroute" : "motorway", + "Type autoroutier" : "motorway", 'Quasi-autoroute' : 'trunk', 'Bretelle' : 'highway_link', 'Route à 2 chaussées': 'primary', @@ -293,21 +294,10 @@ String formatRoadLayer(JdbcDataSource datasource, String road, String zone = "") 'Gué ou radier' : null, 'Pont' : 'bridge', 'Tunnel' : 'tunnel', - 'NC' : null - ] - - def road_surfaces = - ["Autoroute" : "asphalt", - 'Quasi-autoroute' : 'asphalt', - 'Bretelle' : 'asphalt', - 'Route à 2 chaussées': 'asphalt', - 'Route à 1 chaussée' : 'asphalt', - 'Route empierrée' : 'paved', - 'Chemin' : 'ground', - 'Sentier' : 'ground', - 'Pont' : 'asphalt', - 'Tunnel' : 'asphalt', - 'NC': null + 'NC' : null, + 'Rond-point' : 'roundabout', + 'Nationale' : 'primary', + 'Départementale' : 'secondary' ] def road_types_width = @@ -320,10 +310,10 @@ String formatRoadLayer(JdbcDataSource datasource, String road, String zone = "") "residential" : 8, "unclassified": 3, "track" : 2, - "path" : 1, - "footway" : 1, - "cycleway" : 1, - "steps" : 1, + "path" : 2, + "footway" : 2, + "cycleway" : 2, + "steps" : 2, "highway_link": 8, "roundabout" : 4, "ferry" : 0, @@ -336,7 +326,7 @@ String formatRoadLayer(JdbcDataSource datasource, String road, String zone = "") columnNames.remove("THE_GEOM") queryMapper += columnNames.join(",") if (zone) { - datasource.createSpatialIndex(road,"the_geom") + datasource.createSpatialIndex(road, "the_geom") queryMapper += ", st_intersection(a.the_geom, b.the_geom) as the_geom " + "FROM " + "$road AS a, $zone AS b " + @@ -348,48 +338,128 @@ String formatRoadLayer(JdbcDataSource datasource, String road, String zone = "") int rowcount = 1 datasource.withBatch(100) { stmt -> datasource.eachRow(queryMapper) { row -> - def road_type = row.TYPE - def road_surface = row.SURFACE - if (road_type) { - road_type = road_types.get(road_type) - road_surface=road_surfaces.get(road_surface) - } else { - road_type = "unclassified" + def qualified_road_maxspeed = 50 + def qualified_road_type = 'unclassified' + def qualified_road_surface = 'asphalt' + def qualified_sidewalk = 'no' + def qualified_road_width = 3 + def qualified_crossing = null + def qualified_road_zindex = row.ZINDEX + def road_rank = row.RANK + //def road_admin_scale = row.ADMIN_SCALE + switch (row.TYPE) { + case 'Autoroute': + qualified_road_maxspeed = 130 + qualified_road_type = 'motorway' + qualified_road_width = road_types_width.get(qualified_road_type) + break + case 'Type autoroutier': + qualified_road_maxspeed = 130 + qualified_road_type = 'motorway' + qualified_road_width = road_types_width.get(qualified_road_type) + break + case 'Quasi-autoroute': + qualified_road_maxspeed = 130 + qualified_road_type = 'trunk' + qualified_road_width = road_types_width.get(qualified_road_type) + break + case 'Bretelle': + qualified_road_maxspeed = 50 + qualified_road_type = 'highway_link' + qualified_road_width = road_types_width.get(qualified_road_type) + break + case 'Route à 2 chaussées': + qualified_road_maxspeed = 80 + qualified_road_type = 'primary' + qualified_road_width = road_types_width.get(qualified_road_type) + if (road_rank == 6) { + qualified_road_maxspeed = 0 + } else if (road_rank == 5) { + qualified_road_maxspeed = 30 + } + break + case 'Route à 1 chaussée': + if (road_rank == 6) { + qualified_road_maxspeed = 0 + } else if (road_rank == 5) { + qualified_road_maxspeed = 30 + } else if (road_rank <= 4) { + qualified_road_maxspeed = 80 + } + qualified_road_type = 'unclassified' + qualified_road_width = road_types_width.get(qualified_road_type) + break + case 'Route empierrée': + qualified_road_surface = 'paved' + qualified_road_maxspeed = 10 + qualified_road_type = 'track' + qualified_road_width = road_types_width.get(qualified_road_type) + break + case 'Chemin': + qualified_road_surface = 'ground' + qualified_road_maxspeed = 0 + qualified_road_type = 'track' + qualified_road_width = road_types_width.get(qualified_road_type) + break + case 'Sentier': + qualified_road_surface = 'ground' + qualified_road_maxspeed = 0 + qualified_road_type = 'path' + qualified_road_width = road_types_width.get(qualified_road_type) + break + case 'Pont': + qualified_road_maxspeed = 0 + qualified_road_type = 'bridge' + qualified_road_width = road_types_width.get(qualified_road_type) + break + case 'NC': + qualified_road_maxspeed = 0 + qualified_road_width = road_types_width.get(qualified_road_type) + break + case 'Rond-point': + qualified_road_maxspeed = 30 + qualified_road_type = 'roundabout' + qualified_road_width = road_types_width.get(qualified_road_type) + break + case 'Piste cyclable': + qualified_road_maxspeed = 0 + qualified_road_type = 'cycleway' + qualified_road_width = road_types_width.get(qualified_road_type) + break + default: + break } + + //Use the original width def width = row.WIDTH - if (width == null || width <= 0) { - width = road_types_width.get(road_type) - } - def road_zindex = row.ZINDEX - if (!road_zindex) { - road_zindex = 0 + if (width || width > 0) { + qualified_road_width = width } def road_crossing = row.CROSSING - if (road_crossing) { - road_crossing = road_types.get(road_crossing) - if(!road_zindex && road_crossing){ - road_zindex=1 + if (road_crossing == 'Gué ou radier') { + qualified_road_zindex = 0 + } else if (road_crossing == 'Pont') { + qualified_crossing = 'bridge' + if (!qualified_road_zindex) { + qualified_road_zindex = 1 } } - def road_sens = row.SENS - if (road_sens) { - if (road_sens == "Double") { - road_sens = 3 - } else if (road_sens == "Direct") { - road_sens = 1 - } else if (road_sens == "Inverse") { - road_sens = 2 - } else { - road_sens = -1 - } + + def road_sens = row.DIRECTION + + if (road_sens == "Double") { + road_sens = 3 + } else if (road_sens == "Direct") { + road_sens = 1 + } else if (road_sens == "Inverse") { + road_sens = 2 + } else { + road_sens = -1 } def ID_SOURCE = row.ID_SOURCE - def road_sidewalk = row.SIDEWALK - //Not yet managed - def road_maxspeed = null - if (road_zindex >= 0 && road_type && road_type!= 'path') { + if (qualified_road_zindex >= 0 && qualified_road_type != 'path') { Geometry geom = row.the_geom def epsg = geom.getSRID() for (int i = 0; i < geom.getNumGeometries(); i++) { @@ -398,14 +468,14 @@ String formatRoadLayer(JdbcDataSource datasource, String road, String zone = "") '${geom.getGeometryN(i)}',$epsg), ${rowcount++}, '${ID_SOURCE}', - ${width}, - '${road_type}', - '${road_crossing}', - '${road_surface}', - '${road_sidewalk}', - ${road_maxspeed}, + ${qualified_road_width}, + '${qualified_road_type}', + '${qualified_crossing}', + '${qualified_road_surface}', + '${qualified_sidewalk}', + ${qualified_road_maxspeed}, ${road_sens}, - ${road_zindex}) + ${qualified_road_zindex}) """.toString() } } @@ -434,7 +504,7 @@ String formatHydroLayer(JdbcDataSource datasource, String water, String zone = " if (datasource.hasTable(water)) { String query if (zone) { - datasource.createSpatialIndex(water,"the_geom") + datasource.createSpatialIndex(water, "the_geom") query = "select st_intersection(a.the_geom, b.the_geom) as the_geom " + ", a.ZINDEX, a.ID_SOURCE" + " FROM " + @@ -454,7 +524,7 @@ String formatHydroLayer(JdbcDataSource datasource, String water, String zone = " def epsg = geom.getSRID() for (int i = 0; i < geom.getNumGeometries(); i++) { Geometry subGeom = geom.getGeometryN(i) - if (subGeom instanceof Polygon && subGeom.getArea()>1) { + if (subGeom instanceof Polygon && subGeom.getArea() > 1) { stmt.addBatch "insert into $outputTableName values(ST_GEOMFROMTEXT('${subGeom}',$epsg), ${rowcount++}, '${row.ID_SOURCE}', '${water_type}', ${water_zindex})".toString() } } @@ -487,7 +557,7 @@ String formatRailsLayer(JdbcDataSource datasource, String rail, String zone = "" columnNames.remove("THE_GEOM") queryMapper += columnNames.join(",") if (zone) { - datasource.createSpatialIndex(rail,"the_geom") + datasource.createSpatialIndex(rail, "the_geom") queryMapper += ", st_intersection(a.the_geom, b.the_geom) as the_geom " + "FROM " + "$rail AS a, $zone AS b " + @@ -500,18 +570,18 @@ String formatRailsLayer(JdbcDataSource datasource, String rail, String zone = "" def rail_types = ['LGV' : 'highspeed', 'Principale' : 'rail', - 'Voie ferrée principale' : 'rail', + 'Voie ferrée principale' : 'rail', 'Voie de service' : 'service_track', 'Voie non exploitée' : 'disused', 'Transport urbain' : 'tram', 'Funiculaire ou crémaillère': 'funicular', 'Metro' : 'subway', - 'Métro' : 'subway', + 'Métro' : 'subway', 'Tramway' : 'tram', 'Pont' : 'bridge', 'Tunnel' : 'tunnel', - 'Sans objet' : null, - 'NC': null] + 'Sans objet' : null, + 'NC' : null] int rowcount = 1 datasource.withBatch(100) { stmt -> datasource.eachRow(queryMapper) { row -> @@ -521,9 +591,9 @@ String formatRailsLayer(JdbcDataSource datasource, String rail, String zone = "" } else { rail_type = "unclassified" } - def rail_usage="" - if(rail_type in ["highspeed", "rail", "tram", "bridge"]){ - rail_usage="main" + def rail_usage = "" + if (rail_type in ["highspeed", "rail", "tram", "bridge"]) { + rail_usage = "main" } def rail_zindex = row.ZINDEX if (!rail_zindex) { @@ -532,13 +602,13 @@ String formatRailsLayer(JdbcDataSource datasource, String rail, String zone = "" //1.435 default value for standard gauge //1 constant for balasting - def rail_width = !row.WIDTH?1.435+1:row.WIDTH+1 + def rail_width = !row.WIDTH ? 1.435 + 1 : row.WIDTH + 1 def rail_crossing = row.CROSSING if (rail_crossing) { rail_crossing = rail_types.get(rail_crossing) - if(!rail_zindex && rail_crossing){ - rail_zindex=1 + if (!rail_zindex && rail_crossing) { + rail_zindex = 1 } } if (rail_zindex >= 0 && rail_type) { @@ -587,7 +657,7 @@ String formatVegetationLayer(JdbcDataSource datasource, String vegetation, Strin columnNames.remove("THE_GEOM") queryMapper += columnNames.join(",") if (zone) { - datasource.createSpatialIndex(vegetation,"the_geom") + datasource.createSpatialIndex(vegetation, "the_geom") queryMapper += ", st_intersection(a.the_geom, b.the_geom) as the_geom " + "FROM " + "$vegetation AS a, $zone AS b " + @@ -614,7 +684,7 @@ String formatVegetationLayer(JdbcDataSource datasource, String vegetation, Strin 'Rizière' : 'rice_field', 'Piste en herbe' : 'grass', 'Terrain de football' : 'grass', - 'Terrain de rugby': 'grass'] + 'Terrain de rugby' : 'grass'] def vegetation_classes = [ 'tree' : 'high', @@ -632,7 +702,7 @@ String formatVegetationLayer(JdbcDataSource datasource, String vegetation, Strin 'sugar_cane' : 'low', 'unclassified' : 'low', 'hops' : 'low', - 'rice_field' :'low', + 'rice_field' : 'low', 'grass' : 'low' ] @@ -654,7 +724,7 @@ String formatVegetationLayer(JdbcDataSource datasource, String vegetation, Strin def epsg = geom.getSRID() for (int i = 0; i < geom.getNumGeometries(); i++) { Geometry subGeom = geom.getGeometryN(i) - if (subGeom instanceof Polygon && subGeom.getArea()>1) { + if (subGeom instanceof Polygon && subGeom.getArea() > 1) { stmt.addBatch """ INSERT INTO $outputTableName VALUES( ST_GEOMFROMTEXT('${subGeom}',$epsg), @@ -748,7 +818,7 @@ String formatImperviousLayer(H2GIS datasource, String impervious) { def weight_values = [ "government": 5, "entertainment_arts_culture": 10, "education": 10, "military": 20, "industrial": 20, "commercial": 20, "healthcare": 10, "transport": 15, "building": 10, - "sport" : 10, "cemetery": 10, "religious":10] + "sport" : 10, "cemetery": 10, "religious": 10] //We must remove all overlapping geometries and then choose the attribute TYPE to set according some priorities def polygonizedTable = postfix("impervious_polygonized") @@ -757,7 +827,7 @@ String formatImperviousLayer(H2GIS datasource, String impervious) { SELECT * from ST_EXPLODE('(select st_polygonize(st_union(st_accum(ST_ToMultiLine( the_geom)))) as the_geom from $impervious)') """.toString()) - datasource.createSpatialIndex(impervious,"the_geom") + datasource.createSpatialIndex(impervious, "the_geom") datasource.execute(""" CREATE SPATIAL INDEX ON $polygonizedTable(THE_GEOM); CREATE INDEX ON $polygonizedTable(EXPLOD_ID);""".toString()) diff --git a/bdtopo/src/main/groovy/org/orbisgis/geoclimate/bdtopo/InputDataLoading.groovy b/bdtopo/src/main/groovy/org/orbisgis/geoclimate/bdtopo/InputDataLoading.groovy index 0140132a8f..29dcfc12c1 100644 --- a/bdtopo/src/main/groovy/org/orbisgis/geoclimate/bdtopo/InputDataLoading.groovy +++ b/bdtopo/src/main/groovy/org/orbisgis/geoclimate/bdtopo/InputDataLoading.groovy @@ -134,7 +134,10 @@ def loadV2( String route = tablesExist.get("route") if (!route) { route = "route" - datasource.execute("DROP TABLE IF EXISTS $route; CREATE TABLE $route (THE_GEOM geometry(linestring, $srid), ID varchar, LARGEUR DOUBLE PRECISION, NATURE varchar, POS_SOL integer, FRANCHISST varchar, SENS varchar);".toString()) + datasource.execute("""DROP TABLE IF EXISTS $route; + CREATE TABLE $route (THE_GEOM geometry(linestring, $srid), ID varchar, + LARGEUR DOUBLE PRECISION, NATURE varchar, POS_SOL integer, FRANCHISST varchar, SENS varchar, + IMPORTANCE VARCHAR, CL_ADMIN VARCHAR);""".toString()) } String troncon_voie_ferree = tablesExist.get("troncon_voie_ferree") if (!troncon_voie_ferree) { @@ -235,8 +238,13 @@ def loadV2( //4. Prepare the Road table (from the layer "ROUTE") that are in the study area (ZONE_BUFFER) datasource.execute(""" DROP TABLE IF EXISTS INPUT_ROAD; - CREATE TABLE INPUT_ROAD (THE_GEOM geometry, ID_SOURCE varchar(24), WIDTH DOUBLE PRECISION, TYPE varchar, SURFACE varchar, SIDEWALK varchar, ZINDEX integer, CROSSING varchar, SENS varchar) - AS SELECT ST_FORCE2D(ST_MAKEVALID(a.THE_GEOM)) as the_geom, a.ID, a.LARGEUR, a.NATURE, '', '', a.POS_SOL, a.FRANCHISST, a.SENS FROM $route a, ZONE_EXTENDED b WHERE a.the_geom && b.the_geom AND ST_INTERSECTS(a.the_geom, b.the_geom) and a.POS_SOL>=0; + CREATE TABLE INPUT_ROAD (THE_GEOM geometry, ID_SOURCE varchar(24), WIDTH DOUBLE PRECISION, TYPE varchar, ZINDEX integer, CROSSING varchar, + DIRECTION varchar, MAXSPEED INTEGER, RANK INTEGER, ADMIN_SCALE VARCHAR) + AS SELECT ST_FORCE2D(ST_MAKEVALID(a.THE_GEOM)) as the_geom, a.ID, a.LARGEUR, a.NATURE, + a.POS_SOL, a.FRANCHISST, a.SENS ,NULL, + CASE WHEN a.IMPORTANCE IN ('1', '2', '3', '4', '5') THEN CAST (a.IMPORTANCE AS INTEGER) ELSE NULL END , + a.CL_ADMIN + FROM $route a, ZONE_EXTENDED b WHERE a.the_geom && b.the_geom AND ST_INTERSECTS(a.the_geom, b.the_geom) and a.POS_SOL>=0; """.toString()) //5. Prepare the Rail table (from the layer "TRONCON_VOIE_FERREE") that are in the study area (ZONE) @@ -429,7 +437,8 @@ Map loadV3(JdbcDataSource datasource, troncon_de_route = "troncon_de_route" datasource.execute("""DROP TABLE IF EXISTS $troncon_de_route; CREATE TABLE $troncon_de_route (THE_GEOM geometry(linestring, $srid), ID varchar, - LARGEUR DOUBLE PRECISION, NATURE varchar, POS_SOL integer, FRANCHISST varchar, SENS varchar);""".toString()) + LARGEUR DOUBLE PRECISION, NATURE varchar, POS_SOL integer, FRANCHISST varchar, SENS varchar, + IMPORTANCE VARCHAR, CL_ADMIN VARCHAR, NAT_RESTR VARCHAR);""".toString()) } String troncon_de_voie_ferree = tablesExist.get("troncon_de_voie_ferree") @@ -561,15 +570,29 @@ Map loadV3(JdbcDataSource datasource, //4. Prepare the Road table (from the layer "troncon_de_route") that are in the study area (ZONE_BUFFER) datasource.execute(""" DROP TABLE IF EXISTS INPUT_ROAD; - CREATE TABLE INPUT_ROAD (THE_GEOM geometry, ID_SOURCE varchar(24), WIDTH DOUBLE PRECISION, TYPE varchar, SURFACE varchar, SIDEWALK varchar, ZINDEX integer, CROSSING varchar, SENS varchar) - AS SELECT ST_FORCE2D(ST_MAKEVALID(a.THE_GEOM)) as the_geom, a.ID, a.LARGEUR, a.NATURE, '', '', a.POS_SOL, CASE WHEN a.POS_SOL=1 THEN 'bridge' else '' end , a.SENS FROM $troncon_de_route a, ZONE_EXTENDED b WHERE a.the_geom && b.the_geom AND ST_INTERSECTS(a.the_geom, b.the_geom) and a.POS_SOL>=0; + CREATE TABLE INPUT_ROAD (THE_GEOM geometry, ID_SOURCE varchar(24), WIDTH DOUBLE PRECISION, + TYPE varchar, ZINDEX integer, CROSSING varchar, DIRECTION varchar, + RANK INTEGER, ADMIN_SCALE VARCHAR) + AS SELECT ST_FORCE2D(a.THE_GEOM) as the_geom, a.ID, a.LARGEUR, CASE WHEN a.NAT_RESTR = 'Piste cyclable' then a.NAT_RESTR else a.NATURE end, + CASE WHEN a.POS_SOL='Gué ou radier' THEN 0 ELSE CAST(a.POS_SOL AS INT ) END AS POS_SOL, + CASE WHEN a.POS_SOL='1' OR a.POS_SOL='Gué ou radier' THEN a.POS_SOL else null end , + CASE WHEN a.SENS='Double sens' THEN 'Double' + WHEN a.SENS='Sens direct' THEN 'Direct' + WHEN a.SENS='Sens inverse' THEN 'Inverse' + ELSE null END AS SENS, + CASE WHEN a.IMPORTANCE IN ('1', '2', '3', '4', '5', '6') THEN CAST (a.IMPORTANCE AS INTEGER) ELSE NULL END , + a.CL_ADMIN + FROM $troncon_de_route a, + ZONE_EXTENDED b WHERE a.the_geom && b.the_geom AND ST_INTERSECTS(a.the_geom, b.the_geom) + and a.POS_SOL not in ('-4' , '-3' ,'-2' ,'-1'); """.toString()) //5. Prepare the Rail table (from the layer "troncon_de_voie_ferree") that are in the study area (ZONE) datasource.execute(""" DROP TABLE IF EXISTS INPUT_RAIL; CREATE TABLE INPUT_RAIL (THE_GEOM geometry, ID_SOURCE varchar(24), TYPE varchar, ZINDEX integer, CROSSING varchar, WIDTH DOUBLE PRECISION) - AS SELECT ST_FORCE2D(ST_MAKEVALID(a.THE_GEOM)) as the_geom, a.ID, a.NATURE, a.POS_SOL, CASE WHEN a.POS_SOL=1 THEN 'bridge' else '' end, + AS SELECT ST_FORCE2D(a.THE_GEOM) as the_geom, a.ID, a.NATURE, + a.POS_SOL, CASE WHEN a.POS_SOL=1 THEN 'Pont' else '' end, CASE WHEN a.NB_VOIES = 0 THEN 1.435 ELSE 1.435 * a.NB_VOIES END FROM $troncon_de_voie_ferree a, $zoneTable b WHERE a.the_geom && b.the_geom AND ST_INTERSECTS(a.the_geom, b.the_geom) and a.POS_SOL>=0; """.toString()) diff --git a/bdtopo/src/test/groovy/org/orbisgis/geoclimate/bdtopo/WorkflowAbstractTest.groovy b/bdtopo/src/test/groovy/org/orbisgis/geoclimate/bdtopo/WorkflowAbstractTest.groovy index eef5835bad..2f1f0d45d5 100644 --- a/bdtopo/src/test/groovy/org/orbisgis/geoclimate/bdtopo/WorkflowAbstractTest.groovy +++ b/bdtopo/src/test/groovy/org/orbisgis/geoclimate/bdtopo/WorkflowAbstractTest.groovy @@ -19,7 +19,6 @@ */ package org.orbisgis.geoclimate.bdtopo - import org.junit.jupiter.api.Test import org.junit.jupiter.api.io.CleanupMode import org.junit.jupiter.api.io.TempDir @@ -60,6 +59,19 @@ abstract class WorkflowAbstractTest { */ abstract ArrayList getFileNames() + + Map getResultFiles(String dataFolder){ + if(dataFolder){ + def files = [:] + new File(dataFolder).eachFileRecurse groovy.io.FileType.FILES, { file -> + if (file.name.toLowerCase().endsWith(".geojson")) { + files.put((file.name.take(file.name.lastIndexOf('.'))), file.getAbsolutePath()) + } + } + return files + } + } + /** * Return the path of the data sample * @return @@ -351,4 +363,53 @@ abstract class WorkflowAbstractTest { ] assertNull(BDTopo.workflow(bdTopoParameters, getVersion())) } + + @Test + void testFormatData() { + String dataFolder = getDataFolderPath() + def bdTopoParameters = [ + "description" : "Full workflow configuration file", + "geoclimatedb": [ + "folder": folder.absolutePath, + "name" : "testFormatedData;AUTO_SERVER=TRUE", + "delete": true + ], + "input" : [ + "folder" : dataFolder, + "locations": [getInseeCode()]], + "output" : [ + "folder": ["path": folder.absolutePath]], + "parameters" : + ["distance": 0] + ] + + Map process = BDTopo.workflow(bdTopoParameters, getVersion()) + assertNotNull(process) + + Map resultFiles = getResultFiles(folder.absolutePath) + H2GIS h2GIS = H2GIS.open(folder.getAbsolutePath()+File.separator+"bdtopo_${getVersion()}_format") + resultFiles.each { + h2GIS.load( it.value, it.key,true) + } + //Check the data + //Building + int count = h2GIS.getRowCount("building") + List building_cols = [ "ID_BUILD", "ID_SOURCE", "HEIGHT_WALL", "HEIGHT_ROOF", "NB_LEV", "TYPE", "MAIN_USE", "ROOF_SHAPE","ZINDEX", "THE_GEOM"] + assertTrue h2GIS.getColumnNames("building").intersect(building_cols).size()==building_cols.size() + assertEquals(0, h2GIS.firstRow("SELECT COUNT(*) as count FROM building where HEIGHT_WALL = 0 OR HEIGHT_ROOF = 0 OR NB_LEV = 0").count) + assertEquals(count, h2GIS.firstRow("SELECT COUNT(*) as count FROM building where TYPE IS NOT NULL OR MAIN_USE IS NOT NULL").count) + assertEquals(count, h2GIS.firstRow("SELECT COUNT(*) as count FROM building where ZINDEX BETWEEN -4 AND 4").count) + + //Road + count = h2GIS.getRowCount("road") + List road_cols = [ "ID_ROAD", "ID_SOURCE", "WIDTH","TYPE", "SURFACE", "SIDEWALK", "CROSSING","MAXSPEED", "DIRECTION", "ZINDEX", "THE_GEOM"] + assertTrue h2GIS.getColumnNames("road").intersect(road_cols).size()==road_cols.size() + assertEquals(0, h2GIS.firstRow("SELECT COUNT(*) as count FROM road where WIDTH = 0 ").count) + assertEquals(count, h2GIS.firstRow("SELECT COUNT(*) as count FROM road where TYPE IS NOT NULL OR SIDEWALK is not null").count) + assertEquals(count, h2GIS.firstRow("SELECT COUNT(*) as count FROM road where MAXSPEED !=0 OR MAXSPEED>= -1").count) + assertEquals(count, h2GIS.firstRow("SELECT COUNT(*) as count FROM road where DIRECTION !=0 OR DIRECTION>= -1").count) + assertEquals(count, h2GIS.firstRow("SELECT COUNT(*) as count FROM road where ZINDEX BETWEEN -4 AND 4").count) + + } + } \ No newline at end of file diff --git a/bdtopo/src/test/groovy/org/orbisgis/geoclimate/bdtopo/WorkflowDebugTest.groovy b/bdtopo/src/test/groovy/org/orbisgis/geoclimate/bdtopo/WorkflowDebugTest.groovy index db1f931fce..c692c1453f 100644 --- a/bdtopo/src/test/groovy/org/orbisgis/geoclimate/bdtopo/WorkflowDebugTest.groovy +++ b/bdtopo/src/test/groovy/org/orbisgis/geoclimate/bdtopo/WorkflowDebugTest.groovy @@ -50,18 +50,18 @@ class WorkflowDebugTest { def user = postgis_b.user def password = postgis_b.password def url = postgis_b.url - def locations = ["35236"] + def locations = ["54395"] def local_database_name = "geoclimate_test_integration;AUTO_SERVER=TRUE" /*================================================================================ * Input database and tables */ def input = [ + "locations": locations, "database": [ "user" : user, "password" : password, "url" : url, - "locations": locations, "tables" : ["commune" : "ign_bdtopo_2018.commune", "bati_indifferencie" : "ign_bdtopo_2018.bati_indifferencie", "bati_industriel" : "ign_bdtopo_2018.bati_industriel", @@ -113,7 +113,7 @@ class WorkflowDebugTest { ], "input" : input, "output" : output, - "parameters" : ["distance" : 1000, + "parameters" : ["distance" : 100, "rsu_indicators": [ "indicatorUse": ["LCZ", "UTRF", "TEB"] ], @@ -137,7 +137,7 @@ class WorkflowDebugTest { @Test void testIntegrationFolderInput() { def input_data = "/media/ebocher/Extreme SSD/bdtopo/bdtopo2/BDTOPO_2-2_TOUSTHEMES_SHP_LAMB93_D035_2018-09-25/BDTOPO/1_DONNEES_LIVRAISON_2018-11-00144/BDT_2-2_SHP_LAMB93_D035-ED182" - def locations = ["Dijon"] + def locations = ["Gimont"] String directory = "/tmp/bdtopo2" File dirFile = new File(directory) dirFile.delete() @@ -169,16 +169,15 @@ class WorkflowDebugTest { ] //BDTopo.v2(bdTopoParameters) - input_data = "/media/ebocher/Extreme SSD/bdtopo/bdtopo3/BDTOPO_3-3_TOUSTHEMES_SHP_LAMB93_D021_2023-03-15" + input_data = "/home/ebocher/Téléchargements/BDTOPO_3-3_TOUSTHEMES_SHP_LAMB93_D032_2023-09-15/BDTOPO" - directory = "/tmp/bdtopo3" + directory = "/tmp/bdtopo3/result" dirFile = new File(directory) dirFile.delete() dirFile.mkdir() bdTopoParameters.input.folder=input_data bdTopoParameters.output.folder.path=directory - BDTopo.v3(bdTopoParameters) } diff --git a/bdtopo/src/test/groovy/org/orbisgis/geoclimate/bdtopo/v2/WorkflowBDTopoV2Test.groovy b/bdtopo/src/test/groovy/org/orbisgis/geoclimate/bdtopo/v2/WorkflowBDTopoV2Test.groovy index 50e5b9d549..4d2656f1bc 100644 --- a/bdtopo/src/test/groovy/org/orbisgis/geoclimate/bdtopo/v2/WorkflowBDTopoV2Test.groovy +++ b/bdtopo/src/test/groovy/org/orbisgis/geoclimate/bdtopo/v2/WorkflowBDTopoV2Test.groovy @@ -33,6 +33,7 @@ class WorkflowBDTopoV2Test extends WorkflowAbstractTest { "SURFACE_ROUTE", "SURFACE_ACTIVITE"] } + @Override int getVersion() { return 2 diff --git a/bdtopo/src/test/groovy/org/orbisgis/geoclimate/bdtopo/v3/WorkflowBDTopoV3Test.groovy b/bdtopo/src/test/groovy/org/orbisgis/geoclimate/bdtopo/v3/WorkflowBDTopoV3Test.groovy index e4b9ddfc79..8f7f82b389 100644 --- a/bdtopo/src/test/groovy/org/orbisgis/geoclimate/bdtopo/v3/WorkflowBDTopoV3Test.groovy +++ b/bdtopo/src/test/groovy/org/orbisgis/geoclimate/bdtopo/v3/WorkflowBDTopoV3Test.groovy @@ -19,11 +19,21 @@ */ package org.orbisgis.geoclimate.bdtopo.v3 + +import org.junit.jupiter.api.* +import org.orbisgis.data.H2GIS import org.orbisgis.geoclimate.bdtopo.WorkflowAbstractTest +import static org.junit.jupiter.api.Assertions.assertEquals +import static org.junit.jupiter.api.Assertions.assertTrue + class WorkflowBDTopoV3Test extends WorkflowAbstractTest { + public WorkflowBDTopoV3Test(){ + new File("/tmp/test_bd").mkdir() + folder = new File("/tmp/test_bd") + } @Override int getVersion() { return 3 diff --git a/geoindicators/src/main/groovy/org/orbisgis/geoclimate/geoindicators/WorkflowGeoIndicators.groovy b/geoindicators/src/main/groovy/org/orbisgis/geoclimate/geoindicators/WorkflowGeoIndicators.groovy index 53f407da35..610feeb4f6 100644 --- a/geoindicators/src/main/groovy/org/orbisgis/geoclimate/geoindicators/WorkflowGeoIndicators.groovy +++ b/geoindicators/src/main/groovy/org/orbisgis/geoclimate/geoindicators/WorkflowGeoIndicators.groovy @@ -1724,7 +1724,6 @@ String rasterizeIndicators(JdbcDataSource datasource, } } - // If any surface fraction calculation is needed, create the priority list containing only needed fractions // and also set which type of statistics is needed if "BUILDING_HEIGHT" is activated def surfaceFractionsProcess diff --git a/osm/src/test/groovy/org/orbisgis/geoclimate/osm/WorflowOSMTest.groovy b/osm/src/test/groovy/org/orbisgis/geoclimate/osm/WorflowOSMTest.groovy index eb9d0c5b8f..6f13898c1d 100644 --- a/osm/src/test/groovy/org/orbisgis/geoclimate/osm/WorflowOSMTest.groovy +++ b/osm/src/test/groovy/org/orbisgis/geoclimate/osm/WorflowOSMTest.groovy @@ -29,7 +29,7 @@ import org.junit.jupiter.api.io.TempDir import org.orbisgis.data.H2GIS import org.orbisgis.data.POSTGIS import org.orbisgis.geoclimate.Geoindicators -import org.orbisgis.geoclimate.utils.LoggerUtils +import org.orbisgis.geoclimate.osmtools.OSMTools import static org.junit.jupiter.api.Assertions.* @@ -654,7 +654,9 @@ class WorflowOSMTest extends WorkflowAbstractTest { def location = "Redon" - //def nominatim = OSMTools.Utilities.getNominatimData("Lorient") + def nominatim = OSMTools.Utilities.getNominatimData("Redon") + + location = nominatim.bbox def osm_parmeters = [ "description" : "Example of configuration file to run the OSM workflow and store the result in a folder", @@ -677,14 +679,16 @@ class WorflowOSMTest extends WorkflowAbstractTest { "rsu_indicators" : [ "indicatorUse": ["LCZ"] //, "UTRF", "TEB"] ],"grid_indicators": [ - "x_size": 100, - "y_size": 100, + "x_size": 200, + "y_size": 200, //"rowCol": true, "indicators": ["BUILDING_FRACTION","BUILDING_HEIGHT", "BUILDING_POP", - "BUILDING_TYPE_FRACTION","WATER_FRACTION","VEGETATION_FRACTION", + //"BUILDING_TYPE_FRACTION", + "WATER_FRACTION","VEGETATION_FRACTION", "ROAD_FRACTION", "IMPERVIOUS_FRACTION", - "BUILDING_HEIGHT_WEIGHTED", "BUILDING_SURFACE_DENSITY", "SEA_LAND_FRACTION", - "ASPECT_RATIO",//"SVF", + "LCZ_PRIMARY", + //"BUILDING_HEIGHT_WEIGHTED", //"BUILDING_SURFACE_DENSITY", "SEA_LAND_FRACTION", + "ASPECT_RATIO","SVF", "HEIGHT_OF_ROUGHNESS_ELEMENTS", "TERRAIN_ROUGHNESS_CLASS"] ], "worldpop_indicators": true, "road_traffic" : true, @@ -694,6 +698,8 @@ class WorflowOSMTest extends WorkflowAbstractTest { ] ] OSM.workflow(osm_parmeters) + + } @Disabled