From 778406047438abc2330bc44c774c1bcd88ded872 Mon Sep 17 00:00:00 2001 From: =?utf8?q?Fr=C3=A9d=C3=A9ric=20Perrin?= Date: Sat, 5 Oct 2019 16:39:26 +0100 Subject: [PATCH] Use SRTM 1deg data from NASA servers --- .../function/srtm/DownloadSrtmFunction.java | 57 ++++++++++++++++--- .../function/srtm/LookupSrtmFunction.java | 16 +++--- src/tim/prune/function/srtm/SrtmTile.java | 2 +- src/tim/prune/function/srtm/TileFinder.java | 52 ++--------------- 4 files changed, 63 insertions(+), 64 deletions(-) diff --git a/src/tim/prune/function/srtm/DownloadSrtmFunction.java b/src/tim/prune/function/srtm/DownloadSrtmFunction.java index 0350ce3..f58f504 100644 --- a/src/tim/prune/function/srtm/DownloadSrtmFunction.java +++ b/src/tim/prune/function/srtm/DownloadSrtmFunction.java @@ -4,9 +4,13 @@ import java.io.File; import java.io.FileOutputStream; import java.io.IOException; import java.io.InputStream; +import java.net.CookieHandler; +import java.net.CookieManager; +import java.net.CookiePolicy; import java.net.URL; -import java.net.URLConnection; +import java.net.HttpURLConnection; import java.util.ArrayList; +import java.util.Base64; import javax.swing.JOptionPane; @@ -36,6 +40,9 @@ public class DownloadSrtmFunction extends GenericFunction implements Runnable */ public DownloadSrtmFunction(App inApp) { super(inApp); + + CookieHandler.setDefault(new CookieManager(null, CookiePolicy.ACCEPT_ALL)); + } /** @return name key */ @@ -104,7 +111,8 @@ public class DownloadSrtmFunction extends GenericFunction implements Runnable // Update progress bar if (_progress != null) { - _progress.setMaximum(inTileList.size()); + // advance by 1 for HTTP connection open, then by 1 for every tenth of the download + _progress.setMaximum(inTileList.size() * 11); _progress.setValue(0); } @@ -138,26 +146,59 @@ public class DownloadSrtmFunction extends GenericFunction implements Runnable try { // Set progress - _progress.setValue(t); + _progress.setValue(t * 10); // See if we've already got this tile or not File outputFile = getFileToWrite(urls[t]); if (outputFile != null) { - // System.out.println("Download: Need to download: " + urls[t]); + int redirects = 5; + System.out.println("Download: Need to download: " + urls[t]); + HttpURLConnection conn = (HttpURLConnection) urls[t].openConnection(); + String auth = "Basic " + Base64.getEncoder().encodeToString(urls[t].getUserInfo().getBytes()); + long fileLength = 0L; + + while (redirects > 0) { + redirects--; + + conn.setRequestProperty("User-Agent", "GpsPrune v" + GpsPrune.VERSION_NUMBER); + conn.setRequestProperty("Authorization", auth); + conn.setInstanceFollowRedirects(false); + conn.setUseCaches(false); + + int status = conn.getResponseCode(); + if (status == 200) + { + inStream = conn.getInputStream(); + fileLength = conn.getContentLengthLong(); + break; + } + else if (status == 302) + { + // redirected to SSO server then back to original resource + String newUrl = conn.getHeaderField("Location"); + conn = (HttpURLConnection) (new URL(newUrl)).openConnection(); + } + else + { + throw new IOException("Invalid response from server: " +status+conn.getContent()); + } + } + _progress.setValue(t * 10 + 1); outStream = new FileOutputStream(outputFile); - URLConnection conn = urls[t].openConnection(); - conn.setRequestProperty("User-Agent", "GpsPrune v" + GpsPrune.VERSION_NUMBER); - inStream = conn.getInputStream(); + // Copy all the bytes to the file int c; + long written = 0L; while ((c = inStream.read()) != -1) { outStream.write(c); + written++; + _progress.setValue(t * 10 + 1 + (int) ((10 * written) / fileLength)); } numDownloaded++; } - // else System.out.println("Don't need to download: " + urls[t].getFile()); + else System.out.println("Don't need to download: " + urls[t].getFile()); } catch (IOException ioe) {errorMessage = ioe.getClass().getName() + " - " + ioe.getMessage(); } diff --git a/src/tim/prune/function/srtm/LookupSrtmFunction.java b/src/tim/prune/function/srtm/LookupSrtmFunction.java index 1599744..0ae2e45 100644 --- a/src/tim/prune/function/srtm/LookupSrtmFunction.java +++ b/src/tim/prune/function/srtm/LookupSrtmFunction.java @@ -44,7 +44,8 @@ public class LookupSrtmFunction extends GenericFunction implements Runnable private boolean _running = false; /** Expected size of hgt file in bytes */ - private static final long HGT_SIZE = 2884802L; + private static final int HGT_ROW = 3601; + private static final int HGT_SIZE = 2 * HGT_ROW * HGT_ROW; /** Altitude below which is considered void */ private static final int VOID_VAL = -32768; @@ -182,7 +183,7 @@ public class LookupSrtmFunction extends GenericFunction implements Runnable { // Set progress _progress.setValue(t); - final int ARRLENGTH = 1201 * 1201; + final int ARRLENGTH = HGT_ROW * HGT_ROW; int[] heights = new int[ARRLENGTH]; // Open zipinputstream on url and check size ZipInputStream inStream = getStreamToHgtFile(urls[t]); @@ -218,12 +219,12 @@ public class LookupSrtmFunction extends GenericFunction implements Runnable { if (new SrtmTile(point).equals(tile)) { - double x = (point.getLongitude().getDouble() - tile.getLongitude()) * 1200; - double y = 1201 - (point.getLatitude().getDouble() - tile.getLatitude()) * 1200; - int idx1 = ((int)y)*1201 + (int)x; + double x = (point.getLongitude().getDouble() - tile.getLongitude()) * (HGT_ROW - 1); + double y = HGT_ROW - (point.getLatitude().getDouble() - tile.getLatitude()) * (HGT_ROW - 1); + int idx1 = ((int)y)*HGT_ROW + (int)x; try { - int[] fouralts = {heights[idx1], heights[idx1+1], heights[idx1-1201], heights[idx1-1200]}; + int[] fouralts = {heights[idx1], heights[idx1+1], heights[idx1-HGT_ROW], heights[idx1-HGT_ROW+1]}; int numVoids = (fouralts[0]==VOID_VAL?1:0) + (fouralts[1]==VOID_VAL?1:0) + (fouralts[2]==VOID_VAL?1:0) + (fouralts[3]==VOID_VAL?1:0); // if (numVoids > 0) System.out.println(numVoids + " voids found"); @@ -316,7 +317,8 @@ public class LookupSrtmFunction extends GenericFunction implements Runnable // System.out.println("Lookup: Trying online: " + inUrl.toString()); _hadToDownload = true; // MAYBE: Only download if we're in online mode? - return new ZipInputStream(inUrl.openStream()); + // return new ZipInputStream(inUrl.openStream()); + return null; } /** diff --git a/src/tim/prune/function/srtm/SrtmTile.java b/src/tim/prune/function/srtm/SrtmTile.java index 301bbaf..0dee1b8 100644 --- a/src/tim/prune/function/srtm/SrtmTile.java +++ b/src/tim/prune/function/srtm/SrtmTile.java @@ -68,6 +68,6 @@ public class SrtmTile + (Math.abs(_longitude) < 100?"0":"") + (Math.abs(_longitude) < 10?"0":"") + Math.abs(_longitude) - + ".hgt.zip"; + + ".SRTMGL1.hgt.zip"; } } diff --git a/src/tim/prune/function/srtm/TileFinder.java b/src/tim/prune/function/srtm/TileFinder.java index 60a9479..8683268 100644 --- a/src/tim/prune/function/srtm/TileFinder.java +++ b/src/tim/prune/function/srtm/TileFinder.java @@ -13,11 +13,7 @@ import java.util.ArrayList; public abstract class TileFinder { /** URL prefix for all tiles */ - private static final String URL_PREFIX = "https://dds.cr.usgs.gov/srtm/version2_1/SRTM3/"; - /** Directory names for each continent */ - private static final String[] CONTINENTS = {"", "Eurasia", "North_America", "Australia", - "Islands", "South_America", "Africa"}; - + private static final String URL_PREFIX = "https://fred_earthdata:Xaev0oca@e4ftl01.cr.usgs.gov/MEASURES/SRTMGL1.003/2000.02.11/"; /** * Get the Urls for the given list of tiles @@ -29,54 +25,14 @@ public abstract class TileFinder if (inTiles == null || inTiles.size() < 1) {return null;} URL[] urls = new URL[inTiles.size()]; // Read dat file into array - byte[] lookup = readDatFile(); for (int t=0; t 0) { - try { - urls[t] = new URL(URL_PREFIX + CONTINENTS[dir] + "/" + tile.getTileName()); - } catch (MalformedURLException e) {} // ignore error, url stays null - } - } catch (ArrayIndexOutOfBoundsException e) {} // ignore error, url stays null - } - return urls; - } - - /** - * Read the dat file and get the contents - * @return byte array containing file contents - */ - private static byte[] readDatFile() - { - InputStream in = null; - try - { - // Need absolute path to dat file - in = TileFinder.class.getResourceAsStream("/tim/prune/function/srtm/srtmtiles.dat"); - if (in != null) - { - byte[] buffer = new byte[in.available()]; - in.read(buffer); - in.close(); - return buffer; - } - } - catch (java.io.IOException e) { - System.err.println("Exception trying to read srtmtiles.dat : " + e.getMessage()); - } - finally - { try { - in.close(); - } - catch (Exception e) {} // ignore + urls[t] = new URL(URL_PREFIX + tile.getTileName()); + } catch (MalformedURLException e) {} // ignore error, url stays null } - return null; + return urls; } } -- 2.43.0