From a321ae3a39635bebe787694706878039dcf1d989 Mon Sep 17 00:00:00 2001 From: Sebastian Straub Date: Sun, 20 Aug 2017 17:21:10 +0200 Subject: [PATCH] bugfix release: 1.0.1 --- CHANGELOG.md | 21 +- README.md | 8 +- maps4cim-cli/pom.xml | 8 +- maps4cim-core/pom.xml | 18 +- .../java/de/nx42/maps4cim/LoggerConfig.java | 3 +- .../java/de/nx42/maps4cim/ResourceLoader.java | 2 +- .../de/nx42/maps4cim/map/relief/SRTM.java | 27 +- .../nx42/maps4cim/map/relief/Viewfinder.java | 77 + .../map/relief/srtm/TileDownload.java | 421 +- .../map/relief/srtm/TileDownloadUSGS.java | 183 + .../relief/srtm/TileDownloadViewfinder.java | 93 + .../maps4cim/map/texture/osm/OsmHash.java | 18 +- .../map/texture/osm/OverpassBridge.java | 8 +- .../nx42/maps4cim/update/UpdateChecker.java | 6 +- .../java/de/nx42/maps4cim/util/Network.java | 35 +- .../nx42/maps4cim/res/srtm/srtm-mapping.obj | Bin 160153 -> 160157 bytes .../map/relief/srtm/TileDownloadTest.java | 136 +- .../map/relief/srtm/TileDownloadUSGSTest.java | 139 + .../src/test/resources/srtm/Africa.html | 6511 +++++---- .../src/test/resources/srtm/Australia.html | 2131 ++- .../src/test/resources/srtm/Eurasia.html | 11763 ++++++++-------- .../src/test/resources/srtm/Islands.html | 293 +- .../test/resources/srtm/North_America.html | 4835 ++++--- .../test/resources/srtm/South_America.html | 3625 +++-- maps4cim-gui/pom.xml | 30 +- .../src/main/assembly/zip-release.xml | 8 +- .../java/de/nx42/maps4cim/gui/MainWindow.java | 2 +- .../gui/util/TextAreaLogAppender.java | 2 +- .../AddComponentOnJFrameAtRuntime.java | 41 - .../src/test/java/external/FileBrowser.java | 684 - .../test/java/external/LabelRenderTest.java | 78 - .../src/test/java/external/box/BoxPlots.java | 975 -- .../test/java/external/box/DataReader.java | 156 - .../test/java/external/box/DatasetPicker.java | 79 - .../test/java/external/box/Statistics.java | 531 - pom.xml | 24 +- update.xml | 29 +- 37 files changed, 15356 insertions(+), 17644 deletions(-) create mode 100644 maps4cim-core/src/main/java/de/nx42/maps4cim/map/relief/Viewfinder.java create mode 100644 maps4cim-core/src/main/java/de/nx42/maps4cim/map/relief/srtm/TileDownloadUSGS.java create mode 100644 maps4cim-core/src/main/java/de/nx42/maps4cim/map/relief/srtm/TileDownloadViewfinder.java create mode 100644 maps4cim-core/src/test/java/de/nx42/maps4cim/map/relief/srtm/TileDownloadUSGSTest.java delete mode 100644 maps4cim-gui/src/test/java/external/AddComponentOnJFrameAtRuntime.java delete mode 100644 maps4cim-gui/src/test/java/external/FileBrowser.java delete mode 100644 maps4cim-gui/src/test/java/external/LabelRenderTest.java delete mode 100644 maps4cim-gui/src/test/java/external/box/BoxPlots.java delete mode 100644 maps4cim-gui/src/test/java/external/box/DataReader.java delete mode 100644 maps4cim-gui/src/test/java/external/box/DatasetPicker.java delete mode 100644 maps4cim-gui/src/test/java/external/box/Statistics.java diff --git a/CHANGELOG.md b/CHANGELOG.md index 99dfe37..7600682 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,10 @@ # Changelog +### 1.0.1 (2014-08-20) + +* resolved an issue that prevented downloading heightmaps from USGS servers. + ### 1.0.0 (2014-05-15) * New data sources @@ -120,16 +124,23 @@ release schedule: "when it's done" ### 1.0 stable -* done +* Add user agent for overpass API +* display overpass download link if request fails + better warning for huge maps ### 1.1 testing +* add new relief source: +* Select Overpass Server +* gui: map zoom buttons +* find out why names started sucking so hard... maybe need to analyze an empty map again -.- + +### "when it's done" + * xml config: IDREF relation from entity to color * selection inaccurate for large extents -* Select Overpass Server +* alternative SRTM source: (licensing issues, no file index available) * store paths for config open/save, etc. * Notification API -* gui: map zoom buttons, OSM attribution * user input validation * Finish HeightmapWindow * Texture sources @@ -138,6 +149,6 @@ release schedule: "when it's done" - Custom Image: color mapping * Global log-level switch for GUI-console output (verbose, default, off & show stacktrace option would be nice -> warnings/errors are always shown!) -* Make XML schema-aware! -* Cleanup code +* Make XML schema-aware * Write unit tests, where appropriate +* Cleanup code diff --git a/README.md b/README.md index fad0b80..24987ac 100644 --- a/README.md +++ b/README.md @@ -120,6 +120,10 @@ Further information: ## Changelog +*1.0.1* + +* resolved an issue that prevented downloading heightmaps from USGS servers. + *1.0.0* * New data sources: @@ -189,9 +193,9 @@ maps4cim is free software, distributed under the terms of the [screen-game]: https://raw.github.com/Klamann/maps4cim/master/docs/ingame-screen.jpg [Exchange]: http://www.cimexchange.com/files/file/694-maps4cim/ "Download maps4cim from cimexchange.com (english)" [Filebase]: http://www.citiesskylines.de/filebase/index.php/Entry/382-maps4cim/ "Download maps4cim from citiesskylines.de (german)" -[github-release]: https://github.com/Klamann/maps4cim/releases/download/1.0.0/maps4cim-1.0.0.zip "Download maps4cim from github" +[github-release]: https://github.com/Klamann/maps4cim/releases/latest "Download maps4cim from github" [ForumEN]: http://www.cimexchange.com/topic/2204-maps4cim-a-real-world-map-generator-for-cim-2/ "Support Thread in the cimexchange-forum (english)" -[ForumDE]: http://www.citiesskylines.de/index.php/Thread/2578-maps4cim-a-real-world-map-generator-for-CiM-2/?postID=31552 "Support Thread in the citiesinmotion.net-forum (german)" +[ForumDE]: http://www.citiesskylines.de/index.php/Thread/2578-maps4cim-a-real-world-map-generator-for-CiM-2/ "Support Thread in the citiesinmotion.net-forum (german)" [Deveopers.md]: https://github.com/Klamann/maps4cim/blob/master/docs/Deveopers.md "Further information for developers" [CHANGELOG.md]: https://github.com/Klamann/maps4cim/blob/master/CHANGELOG.md "maps4cim's changelog" [docs]: https://github.com/Klamann/maps4cim/tree/master/docs "documentation for maps4cim" diff --git a/maps4cim-cli/pom.xml b/maps4cim-cli/pom.xml index cf29b56..4a47abb 100644 --- a/maps4cim-cli/pom.xml +++ b/maps4cim-cli/pom.xml @@ -5,7 +5,7 @@ de.nx42.maps4cim maps4cim - 1.0.0 + 1.0.1 ../ @@ -51,10 +51,10 @@ maven-assembly-plugin - 2.4 + 3.0.0 - 1.6 - 1.6 + 1.7 + 1.7 true diff --git a/maps4cim-core/pom.xml b/maps4cim-core/pom.xml index 0768ea6..13ddf61 100644 --- a/maps4cim-core/pom.xml +++ b/maps4cim-core/pom.xml @@ -5,7 +5,7 @@ de.nx42.maps4cim maps4cim - 1.0.0 + 1.0.1 ../ @@ -25,12 +25,12 @@ com.google.guava guava - 17.0 + 23.0 org.openstreetmap.osmosis osmosis-xml - 0.43.1 + 0.45 gov.nih.imagej @@ -40,12 +40,18 @@ com.github.jinahya bit-io - 1.1.2 + 1.1.4 net.sf.oval oval - 1.81 + 1.87 + + + org.jsoup + jsoup + 1.8.2 + test @@ -66,7 +72,7 @@ org.apache.maven.plugins maven-jar-plugin - 2.4 + 3.0.2 diff --git a/maps4cim-core/src/main/java/de/nx42/maps4cim/LoggerConfig.java b/maps4cim-core/src/main/java/de/nx42/maps4cim/LoggerConfig.java index 9fe9d04..4b51190 100644 --- a/maps4cim-core/src/main/java/de/nx42/maps4cim/LoggerConfig.java +++ b/maps4cim-core/src/main/java/de/nx42/maps4cim/LoggerConfig.java @@ -29,6 +29,7 @@ import ch.qos.logback.core.rolling.FixedWindowRollingPolicy; import ch.qos.logback.core.rolling.RollingFileAppender; import ch.qos.logback.core.rolling.SizeBasedTriggeringPolicy; +import ch.qos.logback.core.util.FileSize; import ch.qos.logback.core.util.StatusPrinter; import org.slf4j.LoggerFactory; @@ -160,7 +161,7 @@ protected static FileAppender getFileAppender(File logFile) { // rollover after logfixe exceeds 1MB SizeBasedTriggeringPolicy triggeringPolicy = new SizeBasedTriggeringPolicy(); - triggeringPolicy.setMaxFileSize("1MB"); + triggeringPolicy.setMaxFileSize(FileSize.valueOf("1mb")); triggeringPolicy.start(); // layout of the log entries diff --git a/maps4cim-core/src/main/java/de/nx42/maps4cim/ResourceLoader.java b/maps4cim-core/src/main/java/de/nx42/maps4cim/ResourceLoader.java index bdc8865..c9d5408 100644 --- a/maps4cim-core/src/main/java/de/nx42/maps4cim/ResourceLoader.java +++ b/maps4cim-core/src/main/java/de/nx42/maps4cim/ResourceLoader.java @@ -148,7 +148,7 @@ public static ResourceBundle getMessages() { */ public static byte[] readBase64Resource(String path) throws IllegalArgumentException, IOException { String base64 = Resources.toString(Resources.getResource(path), Charsets.UTF_8); - String trimmed = CharMatcher.WHITESPACE.removeFrom(base64); + String trimmed = CharMatcher.whitespace().removeFrom(base64); return BaseEncoding.base64().decode(trimmed); } diff --git a/maps4cim-core/src/main/java/de/nx42/maps4cim/map/relief/SRTM.java b/maps4cim-core/src/main/java/de/nx42/maps4cim/map/relief/SRTM.java index 4b75341..1c04fe1 100644 --- a/maps4cim-core/src/main/java/de/nx42/maps4cim/map/relief/SRTM.java +++ b/maps4cim-core/src/main/java/de/nx42/maps4cim/map/relief/SRTM.java @@ -23,17 +23,17 @@ import java.net.SocketTimeoutException; import java.net.UnknownHostException; +import org.slf4j.LoggerFactory; + import com.google.common.io.ByteArrayDataInput; import com.google.common.io.ByteStreams; import com.google.common.io.Files; -import org.slf4j.LoggerFactory; - import de.nx42.maps4cim.config.Config; import de.nx42.maps4cim.config.relief.SrtmDef; import de.nx42.maps4cim.map.ReliefMap; import de.nx42.maps4cim.map.ex.ReliefProcessingException; -import de.nx42.maps4cim.map.relief.srtm.TileDownload; +import de.nx42.maps4cim.map.relief.srtm.TileDownloadUSGS; import de.nx42.maps4cim.util.Compression; import de.nx42.maps4cim.util.arr2d.Arrays2D; import de.nx42.maps4cim.util.arr2d.GapInterpolator; @@ -253,7 +253,7 @@ protected float getValue(short[][] srtm, int x, int y) { protected short[][] retrieveSRTMdata(Area ar) throws SocketTimeoutException, IOException, UnknownHostException { // get source - TileDownload td = new TileDownload(); + TileDownloadUSGS td = new TileDownloadUSGS(); File[][] files = td.getTiles(ar); // transform to single short array @@ -265,10 +265,7 @@ protected short[][] retrieveSRTMdata(Area ar) throws SocketTimeoutException, IOE // multiple tiles, need to be combined log.debug("combining {} tiles.", files.length * files[0].length); short[][][][] source = unpackSRTMTiles(files); - - // validate only, if within SRTM bounds - boolean validate = ar.getMaxLat() < 61 && ar.getMinLat() >= -60; - return Arrays2D.combine(source, 1, validate); + return Arrays2D.combine(source, 1, false); } } @@ -295,7 +292,7 @@ protected short[][] getNativeSRTM(File input) throws IOException { protected short[][] getNativeSRTM(byte[] input) throws IOException { if(input == null || input.length == 0) { - return getEmptySRTMTile(); + return getWaterSRTMTile(); } ByteArrayDataInput badi = ByteStreams.newDataInput(input); @@ -318,12 +315,20 @@ protected static byte[] readArchiveSRTM(File zipFile) throws IOException { } return Compression.readFirstZipEntry(zipFile); } + + protected short[][] getFlatSRTMTile() { + return getEmptySRTMTile((short) 1); + } + + protected short[][] getWaterSRTMTile() { + return getEmptySRTMTile((short) -40); + } - protected short[][] getEmptySRTMTile() { + protected short[][] getEmptySRTMTile(short value) { short[][] srtm = new short[srtmLength][srtmLength]; for (int y = 0; y < srtmLength; y++) { for (int x = 0; x < srtmLength; x++) { - srtm[y][x] = 1; + srtm[y][x] = value; } } return srtm; diff --git a/maps4cim-core/src/main/java/de/nx42/maps4cim/map/relief/Viewfinder.java b/maps4cim-core/src/main/java/de/nx42/maps4cim/map/relief/Viewfinder.java new file mode 100644 index 0000000..a8ba634 --- /dev/null +++ b/maps4cim-core/src/main/java/de/nx42/maps4cim/map/relief/Viewfinder.java @@ -0,0 +1,77 @@ +package de.nx42.maps4cim.map.relief; + +/** + * source: http://www.viewfinderpanoramas.org/dem3.html + * + * + */ +public class Viewfinder { + + /* + * HGT File Format + * (source: http://www.viewfinderpanoramas.org/dem3.html#hgt) + * + * An HGT file covers an area of 1°x1°. Its south western corner can be + * deduced from its file name: for example, n51e002.hgt covers the area + * between N 51° E 2° and N 52° E 3°, and s14w077.hgt covers S 14° W 77° to + * S 13° W 76°. The file size depends on the resolution. If this is 1", + * there are 3601 rows of 3601 cells each; if it is 3", there are 1201 rows + * of 1201 cells each. The rows are laid out like text on a page, starting + * with the northernmost row, with each row reading from west to east. Each + * cell has two bytes, and the elevation at that cell is + * 256*(1st byte) + (2nd byte). It follows that a 3" HGT file has a file + * length of 2 x 1201 x 1201. SRTM 3" cells are calculated by calculating + * the mean of 1" cells and their eight neighbors. + * It follows that the highest local point is likely to be higher than the + * highest SRTM 3" cell. The difference should vary with the steepness of + * the local relief. + */ + + /* + * Tile package URL retrieval + * + * OK, this is kinda messed up, as there is no proper tile package index + * available and the naming is totally inconsistent. + * This however is not the case for the tiles themselves, these are named + * consistently r"[n|s]\d\d[e|w]\d\d\d.hgt", e.g. "n51e002.hgt" + * + * A possible solution would be parsing the available overview maps + * - http://www.viewfinderpanoramas.org/Coverage%20map%20viewfinderpanoramas_org3.htm + * - http://www.imagico.de/files/ferranti.php + * - http://www.imagico.de/files/ferranti.php?list=dem1list.txt + * now we could deduce the locations from the pixels of the overview maps + * + * Better approach: read the package lists + * - http://www.viewfinderpanoramas.org/dem3list.txt + * - http://www.viewfinderpanoramas.org/dem1list.txt + * or get download links from this site: + * - http://www.imagico.de/map/demsearch.php + * + * These follow a specific pattern: + * id:path[:info], e.g. U22:/dem3/U22.zip or AN2:/ANTDEM3/16-30.zip:-90:-90:-60:-0 + * + * id (tolowercase) can be: + * - "[n|s]\d\d[e|w]\d\d\d", e.g. n47e006: parse directly + * - "s?[a-z]\d\d", e.g. SM18: convert using algorithm below + * - otherwise: use coordinates from comment, see algorithm below + * + * short notation to available tiles: + * west-east in steps of 6°: + * 01 = 180..175°W, 02 = 174..169°W, 30 = 6..1°W, 31 = 0..5°E, 60 = 175..180°E + * north-south in steps of 4°: + * U = 80..83°N, A = 0..3°N, SA = 1..4°S, SN = 53..56°S + * + * comment to coordinates (e.g. "63:-25:67:-13") + * n63w25 to n66w14, last coordinates are exclusive! + * + * downloaded files can contain multiple .hgt files in several subfolders + * as well as unrelated files (readmes, etc.). + * To get all relevant data, list all files and filter with + * r"[n|s]\d\d[e|w]\d\d\d.hgt" + * + * + * + * + */ + +} diff --git a/maps4cim-core/src/main/java/de/nx42/maps4cim/map/relief/srtm/TileDownload.java b/maps4cim-core/src/main/java/de/nx42/maps4cim/map/relief/srtm/TileDownload.java index 3951572..8a78b03 100644 --- a/maps4cim-core/src/main/java/de/nx42/maps4cim/map/relief/srtm/TileDownload.java +++ b/maps4cim-core/src/main/java/de/nx42/maps4cim/map/relief/srtm/TileDownload.java @@ -17,364 +17,187 @@ package de.nx42.maps4cim.map.relief.srtm; import java.io.File; -import java.io.FileNotFoundException; import java.io.IOException; -import java.io.InputStream; -import java.net.MalformedURLException; -import java.net.SocketTimeoutException; -import java.net.URISyntaxException; -import java.net.URL; -import java.net.UnknownHostException; import java.text.ParseException; -import com.google.common.collect.Table; -import com.google.common.io.Resources; - import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import de.nx42.maps4cim.ResourceLoader; import de.nx42.maps4cim.map.Cache; -import de.nx42.maps4cim.util.Network; import de.nx42.maps4cim.util.gis.Area; /** * Download and cache required SRTM tiles */ -public class TileDownload { - - private static final Logger log = LoggerFactory.getLogger(TileDownload.class); +public abstract class TileDownload { + + private static final Logger log = LoggerFactory.getLogger(TileDownload.class); - /* - * Map: http://dds.cr.usgs.gov/srtm/version2_1/Documentation/Continent_def.gif - * Base-URL: http://dds.cr.usgs.gov/srtm/version2_1/SRTM3/ - */ - - /** - * Maps from to . Nonexisting tiles are not contained - * in this data structure. - */ - protected final Table downloadMapping; protected final Cache cache = new Cache(); - public TileDownload() { - this.downloadMapping = loadMapping(); - } - - protected static Table loadMapping() { - InputStream serialized = ResourceLoader.getMappingSRTM(); - try { - Table mapping = ResourceLoader.deserializeObject(serialized); - serialized.close(); - return mapping; - } catch (FileNotFoundException e) { - log.error("The serialized srtm-mapping was not found in the classpath!", e); - } catch (ClassNotFoundException e) { - log.error("The srtm-mapping could not be casted to the guava Table datastructure.", e); - } catch (IOException e) { - log.error("Error while accessing the serialized srtm-mapping.", e); - } - throw new RuntimeException("Could not load SRTM Download-URL mapping!"); - } - public boolean exists(double lat, double lon) { - return downloadMapping.contains(floor(lat), floor(lon)); - } - - public boolean exists(int lat, int lon) { - return downloadMapping.contains(lat, lon); + return exists(floor(lat), floor(lon)); } - public File[][] getTiles(Area ar) throws SocketTimeoutException, IOException, UnknownHostException { - - // fst: lat: bottom to top [-90;90] or S90 to N90 - // snd: lon: left to right [-180;180] or W180 to E180 - // [0][0] is bottom left (south west) [n][n] is top right (north east) + public abstract boolean exists(int lat, int lon); + public File[][] getTiles(Area ar) throws IOException { + CoordinateInt[][] coords = getCoordinates(ar); + File[][] tiles = new File[coords.length][coords[0].length]; + for (int i = 0; i < coords.length; i++) { + for (int j = 0; j < coords[i].length; j++) { + tiles[i][j] = getTile(coords[i][j]); + } + } + return tiles; + } + + static CoordinateInt[][] getCoordinates(Area ar) { int minLat = floor(ar.getMinLat()); int maxLat = (int) Math.ceil(ar.getMaxLat()); int minLon = floor(ar.getMinLon()); int maxLon = (int) Math.ceil(ar.getMaxLon()); - int sizeLat = maxLat - minLat; - int sizeLon = maxLon - minLon; - - // special case: area overlaps the 180° longitude - if(minLon > maxLon) { // e.g. min: 175, max: -178 - log.warn("Your selection overlaps the east/west border at " + - "E180, W180 -> this is currently not supported!"); - } - - File[][] tiles = new File[sizeLat][sizeLon]; + int sizeLon = minLon > maxLon ? ((180-minLon) + (180+maxLon)) : maxLon - minLon; + + CoordinateInt[][] coords = new CoordinateInt[sizeLat][sizeLon]; for (int i = 0; i < sizeLat; i++) { + int gap = 180-minLon; for (int j = 0; j < sizeLon; j++) { - File tile = getTile(minLat + i, minLon + j); - tiles[i][j] = tile; + int lon = minLon+j < 180 ? minLon+j : (-180+j-gap); + coords[i][j] = new CoordinateInt(minLat + i, lon); } } - return tiles; + return coords; } - - public File getTile(double lat, double lon) throws SocketTimeoutException, IOException { - return getTile(floor(lat), floor(lon)); + + public File getTile(CoordinateInt c) throws IOException { + return getTile(c.lat, c.lon); } - protected int floor(double a) { - return (int) StrictMath.floor(a); - } - - public File getTile(int lat, int lon) throws SocketTimeoutException, IOException, UnknownHostException { - - // return null reference, if tile does not exist (only sea level) - if(!exists(lat, lon)) { - log.warn("Tile ({},{}) is not covered in the SRTM dataset, so this part of the map will be flat.", lat, lon); - return null; - } - - if(lat >= 59 || lat <= -59) { - log.warn("Tile ({},{}) is close to the boundaries of the SRTM dataset, the elevations may be incorrect.", lat, lon); - } - - // search cache first, then download file if necessaray - String entry = DownloadURL.getFileName(lat, lon); - if(cache.has(entry)) { - log.debug("SRTM Tile ({},{}) has been loaded from cache.", lat, lon); - return cache.get(entry); - } else { - log.debug("Downloading SRTM Tile for ({},{}). It will be stored in cache for later use.", lat, lon); - return downloadTile(lat, lon); - } - } - - protected File downloadTile(int lat, int lon) throws SocketTimeoutException, UnknownHostException, IOException { - URL src = getDownloadURL(lat, lon); - File temp = cache.allocate(DownloadURL.getFileName(lat, lon)); - - try { - Network.downloadToFile(src, temp, 5, 2); - } catch(FileNotFoundException e) { - // known URL failure, try alternative URLs... - src = downloadMapping.get(lat, lon).getAlternativeUrl(lat, lon); - Network.downloadToFile(src, temp, 5, 2); - } - - File dest = cache.moveToCache(temp, true); - return dest; + public File getTile(double lat, double lon) throws IOException { + return getTile(floor(lat), floor(lon)); } + public abstract File getTile(int lat, int lon) throws IOException; - protected URL getDownloadURL(int lat, int lon) { - DownloadURL dl = downloadMapping.get(lat, lon); - try { - return dl.getUrl(lat, lon); - } catch (MalformedURLException e) { - log.error(String.format("Could not create a valid SRTM download URL " + - "for (%s,%s).", lat, lon), e); - throw new RuntimeException("Creating of SRTM tile download URL failed"); - } - } + protected static CoordinateInt parseCoordinate(String hgtFileName) throws ParseException { - protected static SimpleCoord parseCoordinate(String hgtFileName) throws ParseException { - - /* - * lat: - * - Format: [N|S]dd - * - N -> positive - * - S -> negative - * lon: - * - Format: [E|W]ddd - * - E -> positive - * - W -> negative - */ + /* + * lat: - Format: [N|S]dd - N -> positive - S -> negative lon: - Format: + * [E|W]ddd - E -> positive - W -> negative + */ - String parse = hgtFileName.trim(); - int lat = Integer.MIN_VALUE; - int lon = Integer.MIN_VALUE; + String parse = hgtFileName.trim(); + int lat = Integer.MIN_VALUE; + int lon = Integer.MIN_VALUE; - // parse - for (int i = 0; i < parse.length(); i++) { + // parse + for (int i = 0; i < parse.length(); i++) { char c = parse.charAt(i); - if(c == 'N') { - lat = getNumAfterIndex(parse, ++i); - } else if(c == 'S') { - lat = -(getNumAfterIndex(parse, ++i)); - } else if(c == 'E') { - lon = getNumAfterIndex(parse, ++i); - } else if(c == 'W') { - lon = -(getNumAfterIndex(parse, ++i)); + if (c == 'N') { + lat = getNumAfterIndex(parse, ++i); + } else if (c == 'S') { + lat = -(getNumAfterIndex(parse, ++i)); + } else if (c == 'E') { + lon = getNumAfterIndex(parse, ++i); + } else if (c == 'W') { + lon = -(getNumAfterIndex(parse, ++i)); } } - // check results - if(lat < -90 || lat > 90) { - throw new ParseException(String.format("Latitude must be between " + - "[-90;+90], but is %s", lat), -1); - } else if(lon < -180 || lon > 180) { - throw new ParseException(String.format("Longitude must be between " + - "[-180;+180], but is %s", lon), -1); - } + // check results + if (lat < -90 || lat > 90) { + throw new ParseException(String.format("Latitude must be between " + "[-90;+90], but is %s", lat), -1); + } else if (lon < -180 || lon > 180) { + throw new ParseException(String.format("Longitude must be between " + "[-180;+180], but is %s", lon), -1); + } - // return coordinate - return new SimpleCoord(lat, lon); + // return coordinate + return new CoordinateInt(lat, lon); } /** - * Returns the biggest number that can be parsed in a string after a specified - * index, e.g.: - * "number25 be it" - * for index 6 (which is the position of '2') this method will return 25, - * for index 7 it will be just 5, all other indices will throw - * NumberFormatException (no number found) or some IndexOutOfBounds. - * @param s the string to look for entries in - * @param start the first index of the number to parse + * Returns the biggest number that can be parsed in a string after a + * specified index, e.g.: "number25 be it" for index 6 (which is the + * position of '2') this method will return 25, for index 7 it will be just + * 5, all other indices will throw NumberFormatException (no number found) + * or some IndexOutOfBounds. + * + * @param s + * the string to look for entries in + * @param start + * the first index of the number to parse * @return */ protected static int getNumAfterIndex(String s, int start) throws ParseException { - if(!Character.isDigit(s.charAt(start))) { - throw new ParseException(String.format("No digit recognized at" + - " index %s in %s, just '%s'", start, s, s.charAt(start)), start); - } - - int last = start+1; - search: - while(last < s.length()) { - if(Character.isDigit(s.charAt(last))) { - last++; - } else { - break search; - } - } - return Integer.parseInt(s.substring(start, last)); - } - - // data structures specific for this class - - protected enum DownloadURL { - - Africa("Africa"), - Australia("Australia"), - Eurasia("Eurasia"), - Islands("Islands"), - North_America("North_America"), - South_America("South_America"); - - protected static final String protocol = "http"; - protected static final String host = "dds.cr.usgs.gov"; - protected static final String fileStart = "/srtm/version2_1/SRTM3/"; - protected static final String ext = ".hgt.zip"; - - protected final String folder; - - DownloadURL(String folder) { - this.folder = folder; - } - - public String getFolder() { - return folder; - } - - public URL getUrl(int lat, int lon) throws MalformedURLException { - return new URL(protocol, host, fileStart + folder + "/" + getNonationNSEW(lat, lon) + ext); - } - - public URL getAlternativeUrl(int lat, int lon) throws MalformedURLException { - return new URL(protocol, host, fileStart + folder + "/" + getNonationNSEW(lat, lon) + ext.substring(1)); - } - - public URL getIndexURL() throws MalformedURLException { - return new URL(protocol, host, fileStart + folder); - } - - public File getIndexLocal() { - try { - return new File(Resources.getResource("srtm/" + folder + ".html").toURI()); - } catch (URISyntaxException e) { - log.error("Could not resolve path to local file", e); - return null; - } - } - - /** - * Transforms lat=(+/-)n, lon=(+/-)m to "[N|S]dd[E|W]ddd" - * e.g. lat=47, lon=11 becomes "N47E011" - * and lat=-11, lon=-123 becomes "S11W123" - * @param lat the geographic latitude - * @param lon the geographic longitude - * @return string representation using N,S,E,W notation - */ - public static String getNonationNSEW(int lat, int lon) { - StringBuilder sb = new StringBuilder(16); - - sb.append(lat >= 0 ? 'N' : 'S'); - sb.append(String.format("%02d", Math.abs(lat))); - sb.append(lon >= 0 ? 'E' : 'W'); - sb.append(String.format("%03d", Math.abs(lon))); - - return sb.toString(); + if (!Character.isDigit(s.charAt(start))) { + throw new ParseException(String.format("No digit recognized at " + + "index %s in %s, just '%s'", start, s, s.charAt(start)), start); } - public static String getFileName(int lat, int lon) { - return getNonationNSEW(lat, lon) + ext; + int last = start + 1; + search: while (last < s.length()) { + if (Character.isDigit(s.charAt(last))) { + last++; + } else { + break search; + } } + return Integer.parseInt(s.substring(start, last)); } + static int floor(double a) { + return (int) StrictMath.floor(a); + } + + /** + * Transforms lat=(+/-)n, lon=(+/-)m to "[N|S]dd[E|W]ddd" e.g. lat=47, + * lon=11 becomes "N47E011" and lat=-11, lon=-123 becomes "S11W123" + * + * @param lat the geographic latitude + * @param lon the geographic longitude + * @return string representation using N,S,E,W notation + */ + public static String getNonationNSEW(int lat, int lon) { + StringBuilder sb = new StringBuilder(16); + sb.append(lat >= 0 ? 'N' : 'S'); + sb.append(String.format("%02d", Math.abs(lat))); + sb.append(lon >= 0 ? 'E' : 'W'); + sb.append(String.format("%03d", Math.abs(lon))); + return sb.toString(); + } - - protected static class SimpleCoord { + protected static class CoordinateInt { public final int lat; public final int lon; - public SimpleCoord(int lat, int lon) { + public CoordinateInt(int lat, int lon) { this.lat = lat; this.lon = lon; } + @Override + public int hashCode() { + return lat + 97 * lon; + } + @Override + public boolean equals(Object obj) { + if (this == obj) + return true; + if (obj == null) + return false; + if (getClass() != obj.getClass()) + return false; + CoordinateInt other = (CoordinateInt) obj; + if (lat != other.lat) + return false; + if (lon != other.lon) + return false; + return true; + } + @Override + public String toString() { + return "CoordinateInt [lat=" + lat + ", lon=" + lon + "]"; + } } - - - /* - * The following code depends on JSoup (Maven: org.jsoup). - * Remove the comments to update the tile mapping (usually, this - * is not required) - * DO NOT REMOVE THIS CODE - */ - -// protected static void storeMapping(File f) { -// try { -// // generate mapping -// Table mapping = generateMapping(); -// try { -// // serialize -// ObjectOutputStream oos = new ObjectOutputStream(new FileOutputStream(f)); -// oos.writeObject(mapping); -// oos.flush(); -// oos.close(); -// } catch (Exception e) { -// log.error("Serializing of srtm-mapping failed.", e); -// } -// } catch (Exception e) { -// log.error("Writing of srtm-mapping failed.", e); -// } -// } -// -// protected static Table generateMapping() throws MalformedURLException, IOException, ParseException, SocketTimeoutException { -// -// Table hits = TreeBasedTable.create(); -// for (DownloadURL url : DownloadURL.values()) { -// //String index = url.getIndexURL(); -// File index = url.getIndexLocal(); -// //Document doc = Jsoup.connect(index).userAgent("Mozilla").timeout(8000).get(); -// Document doc = Jsoup.parse(index, null); -// Elements links = doc.select("ul > li > a[href]"); -// for (Element link : links) { -// String hit = link.attr("href"); -// if (hit.endsWith("hgt.zip")) { -// String name = hit.substring(hit.lastIndexOf('/')); -// SimpleCoord coord = parseCoordinate(name); -// hits.put(coord.lat, coord.lon, url); -// } -// } -// } -// return hits; -// } - } diff --git a/maps4cim-core/src/main/java/de/nx42/maps4cim/map/relief/srtm/TileDownloadUSGS.java b/maps4cim-core/src/main/java/de/nx42/maps4cim/map/relief/srtm/TileDownloadUSGS.java new file mode 100644 index 0000000..5a28a0a --- /dev/null +++ b/maps4cim-core/src/main/java/de/nx42/maps4cim/map/relief/srtm/TileDownloadUSGS.java @@ -0,0 +1,183 @@ +/** + * maps4cim - a real world map generator for CiM 2 + * Copyright 2013 Sebastian Straub + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package de.nx42.maps4cim.map.relief.srtm; + +import java.io.File; +import java.io.FileNotFoundException; +import java.io.IOException; +import java.io.InputStream; +import java.net.MalformedURLException; +import java.net.URISyntaxException; +import java.net.URL; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import com.google.common.collect.Table; +import com.google.common.io.Resources; + +import de.nx42.maps4cim.ResourceLoader; +import de.nx42.maps4cim.map.Cache; +import de.nx42.maps4cim.util.Network; + +/** + * Download and cache required SRTM tiles + */ +public class TileDownloadUSGS extends TileDownload { + + private static final Logger log = LoggerFactory.getLogger(TileDownloadUSGS.class); + + /* + * Map: https://dds.cr.usgs.gov/srtm/version2_1/Documentation/Continent_def.gif + * Base-URL: https://dds.cr.usgs.gov/srtm/version2_1/SRTM3/ + */ + + /** + * Maps from to . Nonexisting tiles are not contained + * in this data structure. + */ + protected final Table downloadMapping; + protected final Cache cache = new Cache(); + + public TileDownloadUSGS() { + this.downloadMapping = loadMapping(); + } + + protected static Table loadMapping() { + try(InputStream serialized = ResourceLoader.getMappingSRTM();) { + Table mapping = ResourceLoader.deserializeObject(serialized); + return mapping; + } catch (FileNotFoundException e) { + log.error("The serialized srtm-mapping was not found in the classpath!", e); + } catch (ClassNotFoundException e) { + log.error("The srtm-mapping could not be casted to the guava Table datastructure.", e); + } catch (IOException e) { + log.error("Error while accessing the serialized srtm-mapping.", e); + } + throw new RuntimeException("Could not load SRTM Download-URL mapping!"); + } + + @Override + public boolean exists(int lat, int lon) { + return downloadMapping.contains(lat, lon); + } + + @Override + public File getTile(int lat, int lon) throws IOException { + + // return null reference, if tile does not exist (only sea level) + if(!exists(lat, lon)) { + log.warn("Tile ({},{}) is not covered in the SRTM dataset, so this part of the map will be flat.", lat, lon); + return null; + } + + if(lat >= 59 || lat <= -59) { + log.warn("Tile ({},{}) is close to the boundaries of the SRTM dataset, the elevations may be incorrect.", lat, lon); + } + + // search cache first, then download file if necessaray + String entry = DownloadURL.getFileName(lat, lon); + if(cache.has(entry)) { + log.debug("SRTM Tile ({},{}) has been loaded from cache.", lat, lon); + return cache.get(entry); + } else { + log.debug("Downloading SRTM Tile for ({},{}). It will be stored in cache for later use.", lat, lon); + return downloadTile(lat, lon); + } + } + + protected File downloadTile(int lat, int lon) throws IOException { + URL src = getDownloadURL(lat, lon); + File temp = cache.allocate(DownloadURL.getFileName(lat, lon)); + + try { + Network.downloadToFile(src, temp); + } catch(FileNotFoundException e) { + // known URL failure, try alternative URLs... + src = downloadMapping.get(lat, lon).getAlternativeUrl(lat, lon); + Network.downloadToFile(src, temp); + } + + File dest = cache.moveToCache(temp, true); + return dest; + } + + + protected URL getDownloadURL(int lat, int lon) { + DownloadURL dl = downloadMapping.get(lat, lon); + try { + return dl.getUrl(lat, lon); + } catch (MalformedURLException e) { + log.error(String.format("Could not create a valid SRTM download URL " + + "for (%s,%s).", lat, lon), e); + throw new RuntimeException("Creating of SRTM tile download URL failed"); + } + } + + // data structures specific for this class + + protected enum DownloadURL { + + Africa("Africa"), + Australia("Australia"), + Eurasia("Eurasia"), + Islands("Islands"), + North_America("North_America"), + South_America("South_America"); + + protected static final String protocol = "https"; + protected static final String host = "dds.cr.usgs.gov"; + protected static final String fileStart = "/srtm/version2_1/SRTM3/"; + protected static final String ext = ".hgt.zip"; + + protected final String folder; + + DownloadURL(String folder) { + this.folder = folder; + } + + public String getFolder() { + return folder; + } + + public URL getUrl(int lat, int lon) throws MalformedURLException { + return new URL(protocol, host, fileStart + folder + "/" + getNonationNSEW(lat, lon) + ext); + } + + public URL getAlternativeUrl(int lat, int lon) throws MalformedURLException { + return new URL(protocol, host, fileStart + folder + "/" + getNonationNSEW(lat, lon) + ext.substring(1)); + } + + public URL getIndexURL() throws MalformedURLException { + return new URL(protocol, host, fileStart + folder); + } + + public File getIndexLocal() { + try { + return new File(Resources.getResource("srtm/" + folder + ".html").toURI()); + } catch (URISyntaxException e) { + log.error("Could not resolve path to local file", e); + return null; + } + } + + public static String getFileName(int lat, int lon) { + return getNonationNSEW(lat, lon) + ext; + } + } + +} diff --git a/maps4cim-core/src/main/java/de/nx42/maps4cim/map/relief/srtm/TileDownloadViewfinder.java b/maps4cim-core/src/main/java/de/nx42/maps4cim/map/relief/srtm/TileDownloadViewfinder.java new file mode 100644 index 0000000..0edf769 --- /dev/null +++ b/maps4cim-core/src/main/java/de/nx42/maps4cim/map/relief/srtm/TileDownloadViewfinder.java @@ -0,0 +1,93 @@ +package de.nx42.maps4cim.map.relief.srtm; + +import java.io.File; +import java.io.IOException; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import com.google.common.collect.Table; + +public class TileDownloadViewfinder extends TileDownload { + + private static final Logger log = LoggerFactory.getLogger(TileDownloadViewfinder.class); + + protected static final String siteUrl = "http://www.viewfinderpanoramas.org"; + protected static final String packageIndex1 = siteUrl + "/dem1list.txt"; + protected static final String packageIndex3 = siteUrl + "/dem3list.txt"; + + /** + * Maps from to the download URL relative to the site URL. + * Nonexisting tiles are not contained in this data structure. + */ + //protected final Table downloadMapping; + + /* + * Tile package URL retrieval + * + * OK, this is kinda messed up, as there is no proper tile package index + * available and the naming is rather inconsistent. + * This however is not the case for the tiles themselves, these are named + * consistently r"[n|s]\d\d[e|w]\d\d\d.hgt", e.g. "n51e002.hgt" + * + * A possible solution would be parsing the available overview maps + * - http://www.viewfinderpanoramas.org/Coverage%20map%20viewfinderpanoramas_org3.htm + * - http://www.imagico.de/files/ferranti.php + * - http://www.imagico.de/files/ferranti.php?list=dem1list.txt + * now we could deduce the locations from the pixels of the overview maps + * + * Better approach: read the package lists + * - http://www.viewfinderpanoramas.org/dem3list.txt + * - http://www.viewfinderpanoramas.org/dem1list.txt + * + * These follow a specific pattern: + * id:path[:info], e.g. U22:/dem3/U22.zip or AN2:/ANTDEM3/16-30.zip:-90:-90:-60:-0 + * + * id (tolowercase) can be: + * - "[n|s]\d\d[e|w]\d\d\d", e.g. n47e006: parse directly + * - "s?[a-z]\d\d", e.g. SM18: convert using algorithm below + * - otherwise: use coordinates from comment, see algorithm below + * + * short notation to available tiles: + * west-east in steps of 6°: + * 01 = 180..175°W, 02 = 174..169°W, 30 = 6..1°W, 31 = 0..5°E, 60 = 175..180°E + * north-south in steps of 4°: + * U = 80..83°N, A = 0..3°N, SA = 1..4°S, SN = 53..56°S + * + * comment to coordinates (e.g. "63:-25:67:-13") + * n63w25 to n66w14, last coordinates are exclusive! + * + * downloaded files can contain multiple .hgt files in several subfolders + * as well as unrelated files (readmes, etc.). + * To get all relevant data, list all files and filter with + * r"[n|s]\d\d[e|w]\d\d\d.hgt" + * + */ + + protected static Table loadMapping() { + /* + * - look for index in cache + * - download index if necessary + * - parse 3° index, then substitute 1° index + * + * issues: + * how to separate and combine 1/3° files? + */ + + + return null; + } + + @Override + public boolean exists(int lat, int lon) { + // TODO Auto-generated method stub + return false; + } + + @Override + public File getTile(int lat, int lon) throws IOException { + // TODO Auto-generated method stub + return null; + } + +} diff --git a/maps4cim-core/src/main/java/de/nx42/maps4cim/map/texture/osm/OsmHash.java b/maps4cim-core/src/main/java/de/nx42/maps4cim/map/texture/osm/OsmHash.java index a18c578..9decefb 100644 --- a/maps4cim-core/src/main/java/de/nx42/maps4cim/map/texture/osm/OsmHash.java +++ b/maps4cim-core/src/main/java/de/nx42/maps4cim/map/texture/osm/OsmHash.java @@ -22,15 +22,16 @@ import java.util.Arrays; import java.util.List; +import org.apache.commons.codec.binary.Base64; + +import com.github.jinahya.bio.BitInput; +import com.github.jinahya.bio.BitOutput; +import com.github.jinahya.bio.BufferInput; +import com.github.jinahya.bio.BufferOutput; import com.google.common.hash.HashFunction; import com.google.common.hash.Hasher; import com.google.common.hash.Hashing; -import org.apache.commons.codec.binary.Base64; - -import com.github.jinahya.io.bit.BitInput; -import com.github.jinahya.io.bit.BitOutput; - import de.nx42.maps4cim.config.texture.osm.EntityDef; import de.nx42.maps4cim.map.Cache; import de.nx42.maps4cim.map.ex.TextureProcessingException; @@ -117,7 +118,7 @@ protected static String getQueryHashLocation(Area bounds) throws IOException { int bufSize = (int) Math.ceil(4 * locationPrecision / 8.0); ByteBuffer byteBuf = ByteBuffer.allocate(bufSize); - BitOutput bitOut = BitOutput.newInstance(byteBuf); // direct + BitOutput bitOut = new BitOutput(new BufferOutput(byteBuf)); storeCoordinate(bounds.getMinLat(), bitOut); storeCoordinate(bounds.getMaxLat(), bitOut); @@ -133,7 +134,7 @@ protected static String getQueryHashLocation(Area bounds) throws IOException { protected static Area parseLocationHash(String locationHash) throws IOException { byte[] base64decode = Base64.decodeBase64(locationHash); ByteBuffer byteBuf = ByteBuffer.wrap(base64decode); - final BitInput bitIn = BitInput.newInstance(byteBuf); + final BitInput bitIn = new BitInput(new BufferInput(byteBuf)); double minLat = restoreCoordinate(bitIn); double maxLat = restoreCoordinate(bitIn); @@ -186,11 +187,10 @@ protected String getXmlFileName() { } /** - * Stores the file with a qualified file name and the given hash in the + * Stores the file with a qualified file name in the * user's cache directory, so it can be later retrieved again. * The file will be zipped. * @param xml the osm xml file to cache - * @param hash the hash code that identifies this file * @throws IOException if the file cannot be moved to the cache */ protected void storeInCache(File xml) throws IOException { diff --git a/maps4cim-core/src/main/java/de/nx42/maps4cim/map/texture/osm/OverpassBridge.java b/maps4cim-core/src/main/java/de/nx42/maps4cim/map/texture/osm/OverpassBridge.java index 08a3816..e91fb48 100644 --- a/maps4cim-core/src/main/java/de/nx42/maps4cim/map/texture/osm/OverpassBridge.java +++ b/maps4cim-core/src/main/java/de/nx42/maps4cim/map/texture/osm/OverpassBridge.java @@ -86,9 +86,11 @@ public class OverpassBridge { /** known public Overpass servers */ protected static final String[] servers = new String[] { - "http://overpass-api.de/api/interpreter?data=", // with gzip-support! + "http://overpass-api.de/api/interpreter?data=", // with gzip-support! "http://overpass.osm.rambler.ru/cgi/interpreter?data=", // more powerful, but no gzip & sometimes buggy... - "http://api.openstreetmap.fr/oapi/interpreter?data=" + "http://api.openstreetmap.fr/oapi/interpreter?data=", + "http://overpass.osm.ch/api/interpreter?data=", + "http://overpass.openstreetmap.ie/api/interpreter?data=", }; protected static final String queryBegin = "("; @@ -170,7 +172,7 @@ protected File downloadAndCache(OsmHash hash) throws TextureProcessingException // 5 seconds connection timeout, 90 seconds for the server to execute the query // (so after this time, the download must start, or a timeout occurs) - Network.downloadToFile(query, dest, 5, 90); + Network.downloadToFile(query, dest, 10, 120); // zip result and store in cache if(caching) { diff --git a/maps4cim-core/src/main/java/de/nx42/maps4cim/update/UpdateChecker.java b/maps4cim-core/src/main/java/de/nx42/maps4cim/update/UpdateChecker.java index 5b2ce88..f4ddc9f 100644 --- a/maps4cim-core/src/main/java/de/nx42/maps4cim/update/UpdateChecker.java +++ b/maps4cim-core/src/main/java/de/nx42/maps4cim/update/UpdateChecker.java @@ -34,8 +34,8 @@ public class UpdateChecker { private static final Logger log = LoggerFactory.getLogger(UpdateChecker.class); protected static final String[] updateURLs = new String[] { - "http://nx42.de/projects/maps4cim/update/update.xml", - "http://raw.githubusercontent.com/Klamann/maps4cim/master/update.xml" + "http://raw.githubusercontent.com/Klamann/maps4cim/master/update.xml", + "http://nx42.de/projects/maps4cim/update/update.xml" }; protected static final String xmlFileName = "update.xml"; @@ -60,7 +60,7 @@ protected static Update getUpdateXml() { try { URL url = new URL(s); File xml = Cache.temporaray(xmlFileName); - Network.downloadToFile(url, xml, 5, 5); + Network.downloadToFile(url, xml); try { return Serializer.deserialize(Update.class, xml); diff --git a/maps4cim-core/src/main/java/de/nx42/maps4cim/util/Network.java b/maps4cim-core/src/main/java/de/nx42/maps4cim/util/Network.java index 3fbe91b..8a93bb0 100644 --- a/maps4cim-core/src/main/java/de/nx42/maps4cim/util/Network.java +++ b/maps4cim-core/src/main/java/de/nx42/maps4cim/util/Network.java @@ -24,7 +24,6 @@ import java.net.SocketTimeoutException; import java.net.URL; import java.net.URLConnection; -import java.net.UnknownHostException; import java.util.zip.GZIPInputStream; import com.google.common.io.ByteStreams; @@ -53,24 +52,40 @@ public class Network { * @throws SocketTimeoutException if the connection times out */ public static void downloadToFile(URL src, File dest, double connTimeout, - double readTimeout) throws IOException, SocketTimeoutException, - UnknownHostException { + double readTimeout) throws IOException, SocketTimeoutException { - URLConnection conn = src.openConnection(); - conn.setRequestProperty("Accept-Encoding", "gzip"); + URLConnection conn = openConection(src); conn.setConnectTimeout((int) (connTimeout * 1000)); conn.setReadTimeout((int) (readTimeout * 1000)); - InputStream in = conn.getInputStream(); + storeConnectionAnswer(conn, dest); + } + + public static void downloadToFile(URL src, File dest) throws IOException { + URLConnection conn = openConection(src); + storeConnectionAnswer(conn, dest); + } + + private static URLConnection openConection(URL src) throws IOException { + URLConnection conn = src.openConnection(); + conn.setRequestProperty("Accept-Encoding", "gzip"); + return conn; + } + + private static void storeConnectionAnswer(URLConnection conn, File dest) throws IOException { + // create input stream and upgrade to gzip, if required + InputStream in = conn.getInputStream(); if ("gzip".equals(conn.getContentEncoding())) { in = new GZIPInputStream(in); } + + // create output stream to file and copy data OutputStream out = new FileOutputStream(dest); + ByteStreams.copy(in, out); - ByteStreams.copy(in, out); - - in.close(); - out.close(); + // close all streams + in.close(); + out.close(); } } diff --git a/maps4cim-core/src/main/resources/de/nx42/maps4cim/res/srtm/srtm-mapping.obj b/maps4cim-core/src/main/resources/de/nx42/maps4cim/res/srtm/srtm-mapping.obj index 9efb7a9773dd65abfcb2f6da9bcee3b55f9137b1..b06db7fd91209e906c6fc4a03cbbdb1f557e9050 100644 GIT binary patch delta 33 pcmbRFm~-x9&JA2ljJA`xnRHn~gWZFhEtuLZm>9QPFfk?O0|2w`30?pI delta 28 kcmbRHm~-Z1&JA2ljFywRnRJ_NnA&Za7`NLnF(u{$0Eiq2AOHXW diff --git a/maps4cim-core/src/test/java/de/nx42/maps4cim/map/relief/srtm/TileDownloadTest.java b/maps4cim-core/src/test/java/de/nx42/maps4cim/map/relief/srtm/TileDownloadTest.java index 17f9306..133afde 100644 --- a/maps4cim-core/src/test/java/de/nx42/maps4cim/map/relief/srtm/TileDownloadTest.java +++ b/maps4cim-core/src/test/java/de/nx42/maps4cim/map/relief/srtm/TileDownloadTest.java @@ -1,6 +1,5 @@ package de.nx42.maps4cim.map.relief.srtm; -import static org.junit.Assert.assertArrayEquals; import static org.junit.Assert.assertEquals; import static org.junit.Assert.fail; @@ -8,14 +7,14 @@ import org.junit.Test; -import de.nx42.maps4cim.map.relief.srtm.TileDownload.SimpleCoord; +import de.nx42.maps4cim.map.relief.srtm.TileDownload.CoordinateInt; +import de.nx42.maps4cim.util.gis.Area; public class TileDownloadTest { @Test public void testGetNumAfterIndex() throws Exception { String input = "N47E011.hgt.zip"; - try { int n = TileDownload.getNumAfterIndex(input, 1); int e = TileDownload.getNumAfterIndex(input, 4); @@ -50,7 +49,7 @@ public void testParseCoordinate4() throws Exception { protected static void helpTestParseCoordinate(String parse, int expLat, int expLon) { try { - SimpleCoord sc = TileDownload.parseCoordinate(parse); + CoordinateInt sc = TileDownload.parseCoordinate(parse); assertEquals(expLat, sc.lat); assertEquals(expLon, sc.lon); } catch (ParseException e) { @@ -59,109 +58,30 @@ protected static void helpTestParseCoordinate(String parse, int expLat, int expL } - // The following tests depend on the availablitity of external servers - // and disk write permissions, disabled by default... - -// @Test -// public void testGetTileIntInt() throws Exception { -// TileDownload td = new TileDownload(); -// File f = td.getTile(47, 11); -// Cache c = new Cache(); -// -// assertTrue(c.has(f.getName())); -// assertEquals(new File(Cache.cacheDir, "N47E011.hgt.zip").toString(), f.toString()); -// } - -// @Test -// public void testGetTiles() throws Exception { -// Area ar = new Area(new Coordinate(48.4, 11.7), 8, UnitOfLength.KILOMETER); -// Cache c = new Cache(); -// File[][] expected = new File[][]{ { c.getUnchecked("N48E011.hgt.zip") } }; -// -// runTestGetTiles(ar, expected); -// } - -// @Test -// public void testGetTiles2() throws Exception { -// Area ar = new Area(new Coordinate(48.4, 12.0), 8, UnitOfLength.KILOMETER); -// Cache c = new Cache(); -// File[][] expected = new File[][] { -// { c.getUnchecked("N48E011.hgt.zip"), c.getUnchecked("N48E012.hgt.zip") } -// }; -// -// runTestGetTiles(ar, expected); -// } - -// @Test -// public void testGetTiles3() throws Exception { -// Area ar = new Area(new Coordinate(48.0, 12.0), 8, UnitOfLength.KILOMETER); -// Cache c = new Cache(); -// File[][] expected = new File[][] { -// { c.getUnchecked("N47E011.hgt.zip"), c.getUnchecked("N47E012.hgt.zip") }, -// { c.getUnchecked("N48E011.hgt.zip"), c.getUnchecked("N48E012.hgt.zip") } -// }; -// -// runTestGetTiles(ar, expected); -// } - -// protected static void runTestGetTiles(Area ar, File[][] expected) { -// System.out.println("tile download test for area " + ar.toString()); -// -// try { -// TileDownload td = new TileDownload(); -// File[][] actual = td.getTiles(ar); -// assertArray2dEquals(expected, actual); -// } catch(Exception e) { -// fail(e.getMessage()); -// } -// } - - - public static void assertArray2dEquals(Object[][] expected, Object[][] actual) { - if (expected.length != actual.length || expected[0].length != actual[0].length) { - fail("Arrays are of different size!"); - } - for (int i = 0; i < actual.length; i++) { - assertArrayEquals(expected[i], actual[i]); - } - } - - // the following tests can be used to update the srtm download mapping - -// @Test -// public void testStoreMapping() { -// try { -// -// // store -// File store = new File(TestHelper.getTestFolder(), "mapping.obj"); -// TileDownload.storeMapping(store); -// -// // read -// ObjectInputStream ois = new ObjectInputStream( -// new FileInputStream(store)); -// @SuppressWarnings("unchecked") -// Table mapping = -// (Table) ois.readObject(); -// ois.close(); -// -// // assert -// assertEquals(mapping.hashCode(), mapping.hashCode()); -// assertEquals(mapping.toString(), mapping.toString()); -// assertTrue(mapping.equals(mapping)); -// -// } catch(Exception e) { -// fail(e.getMessage()); -// } -// } -// -// @Test -// public void testGenerateMapping() { -// try { -// TileDownload.generateMapping(); -// } catch (Exception e) { -// e.printStackTrace(); -// fail(e.getMessage()); -// } -// } + @Test + public void testGetCoordinates() throws Exception { + Area ar = new Area(48,11,50,13); + CoordinateInt[][] coords = TileDownload.getCoordinates(ar); + assertEquals(2, coords.length); + assertEquals(2, coords[0].length); + assertEquals(new CoordinateInt(48,11), coords[0][0]); + assertEquals(new CoordinateInt(48,12), coords[0][1]); + assertEquals(new CoordinateInt(49,11), coords[1][0]); + assertEquals(new CoordinateInt(49,12), coords[1][1]); + } + + @Test + public void testGetCoordinates2() throws Exception { + Area ar = new Area(48,175,50,-178); + CoordinateInt[][] coords = TileDownload.getCoordinates(ar); + assertEquals(2, coords.length); + assertEquals(7, coords[0].length); + assertEquals(new CoordinateInt(48,175), coords[0][0]); + assertEquals(new CoordinateInt(48,179), coords[0][4]); + assertEquals(new CoordinateInt(48,-180), coords[0][5]); + assertEquals(new CoordinateInt(48,-179), coords[0][6]); + assertEquals(new CoordinateInt(49,175), coords[1][0]); + assertEquals(new CoordinateInt(49,-179), coords[1][6]); + } } diff --git a/maps4cim-core/src/test/java/de/nx42/maps4cim/map/relief/srtm/TileDownloadUSGSTest.java b/maps4cim-core/src/test/java/de/nx42/maps4cim/map/relief/srtm/TileDownloadUSGSTest.java new file mode 100644 index 0000000..76a1841 --- /dev/null +++ b/maps4cim-core/src/test/java/de/nx42/maps4cim/map/relief/srtm/TileDownloadUSGSTest.java @@ -0,0 +1,139 @@ +package de.nx42.maps4cim.map.relief.srtm; + +import static org.junit.Assert.assertArrayEquals; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; + +import java.io.File; +import java.io.FileOutputStream; +import java.io.IOException; +import java.io.ObjectOutputStream; +import java.text.ParseException; + +import org.jsoup.Jsoup; +import org.jsoup.nodes.Document; +import org.jsoup.nodes.Element; +import org.jsoup.select.Elements; + +import com.google.common.collect.Table; +import com.google.common.collect.TreeBasedTable; + +import de.nx42.maps4cim.map.Cache; +import de.nx42.maps4cim.map.relief.srtm.TileDownload.CoordinateInt; +import de.nx42.maps4cim.map.relief.srtm.TileDownloadUSGS.DownloadURL; +import de.nx42.maps4cim.util.gis.Area; +import de.nx42.maps4cim.util.gis.Coordinate; +import de.nx42.maps4cim.util.gis.UnitOfLength; + +public class TileDownloadUSGSTest { + + // The following tests depend on the availablitity of external servers + // and disk write permissions, disabled by default... + + // @Test + public void testGetTileIntInt() throws Exception { + TileDownload td = new TileDownloadUSGS(); + File f = td.getTile(47, 11); + Cache c = new Cache(); + + assertTrue(c.has(f.getName())); + assertEquals(new File(Cache.cacheDir, "N47E011.hgt.zip").toString(), f.toString()); + } + + // @Test + public void testGetTiles() throws Exception { + Area ar = new Area(new Coordinate(48.4, 11.7), 8, UnitOfLength.KILOMETER); + Cache c = new Cache(); + File[][] expected = new File[][]{ { c.getUnchecked("N48E011.hgt.zip") } }; + + runTestGetTiles(ar, expected); + } + + // @Test + public void testGetTiles2() throws Exception { + Area ar = new Area(new Coordinate(48.4, 12.0), 8, UnitOfLength.KILOMETER); + Cache c = new Cache(); + File[][] expected = new File[][] { + { c.getUnchecked("N48E011.hgt.zip"), c.getUnchecked("N48E012.hgt.zip") } + }; + + runTestGetTiles(ar, expected); + } + + // @Test + public void testGetTiles3() throws Exception { + Area ar = new Area(new Coordinate(48.0, 12.0), 8, UnitOfLength.KILOMETER); + Cache c = new Cache(); + File[][] expected = new File[][] { + { c.getUnchecked("N47E011.hgt.zip"), c.getUnchecked("N47E012.hgt.zip") }, + { c.getUnchecked("N48E011.hgt.zip"), c.getUnchecked("N48E012.hgt.zip") } + }; + + runTestGetTiles(ar, expected); + } + + // helper functions + + protected static void runTestGetTiles(Area ar, File[][] expected) { + System.out.println("tile download test for area " + ar.toString()); + + try { + TileDownload td = new TileDownloadUSGS(); + File[][] actual = td.getTiles(ar); + assertArray2dEquals(expected, actual); + } catch(Exception e) { + fail(e.getMessage()); + } + } + + public static void assertArray2dEquals(Object[][] expected, Object[][] actual) { + if (expected.length != actual.length || expected[0].length != actual[0].length) { + fail("Arrays are of different size!"); + } + for (int i = 0; i < actual.length; i++) { + assertArrayEquals(expected[i], actual[i]); + } + } + + // the following test can be used to update the srtm download mapping + // disabled by default! + + // @Test + public void testStoreMapping() { + File f = new File("target/srtm-mapping.obj"); + // generate mapping + try { + System.out.println("Generating file mapping..."); + Table mapping = generateMapping(); + System.out.println("Writing to file " + f.toString()); + try (ObjectOutputStream oos = new ObjectOutputStream(new FileOutputStream(f))) { + oos.writeObject(mapping); + } + } catch(Exception e) { + fail(e.getMessage()); + } + } + + public static Table generateMapping() throws IOException, ParseException { + Table hits = TreeBasedTable.create(); + for (DownloadURL url : DownloadURL.values()) { + File index = url.getIndexLocal(); + System.out.println("reading " + index.toString()); + // Document doc = + // Jsoup.connect(index).userAgent("Mozilla").timeout(8000).get(); + Document doc = Jsoup.parse(index, null); + Elements links = doc.select("ul > li > a[href]"); + for (Element link : links) { + String hit = link.attr("href"); + if (hit.endsWith("hgt.zip")) { + String name = hit.substring(hit.lastIndexOf('/')); + CoordinateInt coord = TileDownload.parseCoordinate(name); + hits.put(coord.lat, coord.lon, url); + } + } + } + return hits; + } + +} diff --git a/maps4cim-core/src/test/resources/srtm/Africa.html b/maps4cim-core/src/test/resources/srtm/Africa.html index 1b301ed..82f619a 100644 --- a/maps4cim-core/src/test/resources/srtm/Africa.html +++ b/maps4cim-core/src/test/resources/srtm/Africa.html @@ -1,3261 +1,3260 @@ - - - + + + Index of /srtm/version2_1/SRTM3/Africa

Index of /srtm/version2_1/SRTM3/Africa

-