From: Frédéric Perrin Date: Sat, 5 Oct 2019 15:39:26 +0000 (+0100) Subject: Use SRTM 1deg data from NASA servers X-Git-Tag: v19.2.fp1~1^2~1 X-Git-Url: http://gitweb.fperrin.net/?p=GpsPrune.git;a=commitdiff_plain;h=189a667821db055202dcb9f539ee26bd2af7ade4 Use SRTM 1deg data from NASA servers --- diff --git a/src/tim/prune/FunctionLibrary.java b/src/tim/prune/FunctionLibrary.java index 0441826..183c581 100644 --- a/src/tim/prune/FunctionLibrary.java +++ b/src/tim/prune/FunctionLibrary.java @@ -61,7 +61,6 @@ import tim.prune.function.settings.SetMapBgFunction; import tim.prune.function.settings.SetPathsFunction; import tim.prune.function.sew.SewTrackSegmentsFunction; import tim.prune.function.sew.SplitSegmentsFunction; -import tim.prune.function.srtm.DownloadSrtmFunction; import tim.prune.function.srtm.LookupSrtmFunction; import tim.prune.function.weather.GetWeatherForecastFunction; import tim.prune.load.AudioLoader; @@ -103,7 +102,6 @@ public abstract class FunctionLibrary public static GenericFunction FUNCTION_DELETE_BY_DATE = null; public static SingleNumericParameterFunction FUNCTION_INTERPOLATE = null; public static GenericFunction FUNCTION_LOOKUP_SRTM = null; - public static GenericFunction FUNCTION_DOWNLOAD_SRTM = null; public static GenericFunction FUNCTION_NEARBY_WIKIPEDIA = null; public static GenericFunction FUNCTION_SEARCH_WIKIPEDIA = null; public static GenericFunction FUNCTION_SEARCH_OSMPOIS = null; @@ -183,7 +181,6 @@ public abstract class FunctionLibrary FUNCTION_DELETE_BY_DATE = new DeleteByDateFunction(inApp); FUNCTION_INTERPOLATE = new InterpolateFunction(inApp); FUNCTION_LOOKUP_SRTM = new LookupSrtmFunction(inApp); - FUNCTION_DOWNLOAD_SRTM = new DownloadSrtmFunction(inApp); FUNCTION_NEARBY_WIKIPEDIA = new GetWikipediaFunction(inApp); FUNCTION_SEARCH_WIKIPEDIA = new SearchWikipediaNames(inApp); FUNCTION_SEARCH_OSMPOIS = new SearchOsmPoisFunction(inApp); diff --git a/src/tim/prune/config/Config.java b/src/tim/prune/config/Config.java index 84e5fd7..3939c00 100644 --- a/src/tim/prune/config/Config.java +++ b/src/tim/prune/config/Config.java @@ -105,6 +105,8 @@ public abstract class Config public static final String KEY_WAYPOINT_ICON_SIZE = "prune.waypointiconsize"; /** Id of selected timezone */ public static final String KEY_TIMEZONE_ID = "prune.timezoneid"; + /** Username/password to the Earthdata server for SRTM 1-arcsecond tiles */ + public static final String KEY_EARTHDATA_AUTH = "prune.earthdataauth"; /** Initialise the default properties */ diff --git a/src/tim/prune/function/srtm/DownloadSrtmFunction.java b/src/tim/prune/function/srtm/DownloadSrtmFunction.java index 0350ce3..311ff92 100644 --- a/src/tim/prune/function/srtm/DownloadSrtmFunction.java +++ b/src/tim/prune/function/srtm/DownloadSrtmFunction.java @@ -1,11 +1,5 @@ package tim.prune.function.srtm; -import java.io.File; -import java.io.FileOutputStream; -import java.io.IOException; -import java.io.InputStream; -import java.net.URL; -import java.net.URLConnection; import java.util.ArrayList; import javax.swing.JOptionPane; @@ -14,7 +8,6 @@ import tim.prune.App; import tim.prune.GenericFunction; import tim.prune.GpsPrune; import tim.prune.I18nManager; -import tim.prune.config.Config; import tim.prune.data.DoubleRange; import tim.prune.gui.ProgressDialog; @@ -28,19 +21,20 @@ public class DownloadSrtmFunction extends GenericFunction implements Runnable private ProgressDialog _progress = null; /** Flag to check whether this function is currently running or not */ private boolean _running = false; - + private SrtmSource _srtmSource = null; /** * Constructor * @param inApp App object */ - public DownloadSrtmFunction(App inApp) { + public DownloadSrtmFunction(App inApp, SrtmSource inSrtmSource) { super(inApp); + _srtmSource = inSrtmSource; } /** @return name key */ public String getNameKey() { - return "function.downloadsrtm"; + return "function.downloadsrtm."+_srtmSource.getName(); } /** @@ -48,6 +42,17 @@ public class DownloadSrtmFunction extends GenericFunction implements Runnable */ public void begin() { + if (! SrtmDiskCache.ensureCacheIsUsable()) + { + _app.showErrorMessage(getNameKey(), "error.downloadsrtm.nocache"); + return; + } + if (! _srtmSource.isReadyToUse()) + { + _app.showErrorMessage(getNameKey(), getNameKey() + ".needsetup"); + return; + } + _running = true; if (_progress == null) { _progress = new ProgressDialog(_parentFrame, getNameKey()); @@ -61,6 +66,14 @@ public class DownloadSrtmFunction extends GenericFunction implements Runnable * Run method using separate thread */ public void run() + { + ArrayList tileList = buildCoveringTiles(); + downloadTiles(tileList); + // Finished + _running = false; + } + + private ArrayList buildCoveringTiles() { // Compile list of tiles to get ArrayList tileList = new ArrayList(); @@ -89,18 +102,16 @@ public class DownloadSrtmFunction extends GenericFunction implements Runnable } } - downloadTiles(tileList); - // Finished - _running = false; + return tileList; } - /** * Download the tiles of SRTM data * @param inTileList list of tiles to get */ private void downloadTiles(ArrayList inTileList) { + String errorMessage = ""; // Update progress bar if (_progress != null) { @@ -108,62 +119,29 @@ public class DownloadSrtmFunction extends GenericFunction implements Runnable _progress.setValue(0); } - String errorMessage = null; - - // Check the cache is ok - final String diskCachePath = Config.getConfigString(Config.KEY_DISK_CACHE); - if (diskCachePath != null) - { - File srtmDir = new File(diskCachePath, "srtm"); - if (!srtmDir.exists() && !srtmDir.mkdir()) { - // can't create the srtm directory - errorMessage = I18nManager.getText("error.downloadsrtm.nocache"); - } - } - else { - // no cache set up - errorMessage = I18nManager.getText("error.downloadsrtm.nocache"); - } - - // Get urls for each tile - URL[] urls = TileFinder.getUrls(inTileList); int numDownloaded = 0; for (int t=0; t 0) { - _app.showErrorMessage(getNameKey(), "confirm.downloadsrtm.none"); + JOptionPane.showMessageDialog(_parentFrame, I18nManager.getText("confirm.downloadsrtm.none")); } } - - /** - * See whether the SRTM file is already available locally - * @param inUrl URL for online resource - * @return file object to write to, or null if already there - */ - private static File getFileToWrite(URL inUrl) - { - String diskCachePath = Config.getConfigString(Config.KEY_DISK_CACHE); - if (diskCachePath != null) - { - File srtmDir = new File(diskCachePath, "srtm"); - if (srtmDir.exists() && srtmDir.isDirectory() && srtmDir.canRead()) - { - File srtmFile = new File(srtmDir, new File(inUrl.getFile()).getName()); - if (!srtmFile.exists() || !srtmFile.canRead() || srtmFile.length() <= 400) { - return srtmFile; - } - } - } - return null; - } - - /** - * @return true if a thread is currently running - */ - public boolean isRunning() - { - return _running; - } } diff --git a/src/tim/prune/function/srtm/LookupSrtmFunction.java b/src/tim/prune/function/srtm/LookupSrtmFunction.java index 1599744..927d2cd 100644 --- a/src/tim/prune/function/srtm/LookupSrtmFunction.java +++ b/src/tim/prune/function/srtm/LookupSrtmFunction.java @@ -1,12 +1,6 @@ package tim.prune.function.srtm; -import java.io.File; -import java.io.FileInputStream; -import java.io.IOException; -import java.net.URL; import java.util.ArrayList; -import java.util.zip.ZipEntry; -import java.util.zip.ZipInputStream; import javax.swing.JOptionPane; @@ -15,14 +9,12 @@ import tim.prune.DataSubscriber; import tim.prune.GenericFunction; import tim.prune.I18nManager; import tim.prune.UpdateMessageBroker; -import tim.prune.config.Config; import tim.prune.data.Altitude; import tim.prune.data.DataPoint; import tim.prune.data.Field; import tim.prune.data.Track; import tim.prune.data.UnitSetLibrary; import tim.prune.gui.ProgressDialog; -import tim.prune.tips.TipManager; import tim.prune.undo.UndoLookupSrtm; /** @@ -38,13 +30,9 @@ public class LookupSrtmFunction extends GenericFunction implements Runnable private Track _track = null; /** Flag for whether this is a real track or a terrain one */ private boolean _normalTrack = true; - /** Flag set when any tiles had to be downloaded (rather than just loaded locally) */ - private boolean _hadToDownload = false; /** Flag to check whether this function is currently running or not */ private boolean _running = false; - /** Expected size of hgt file in bytes */ - private static final long HGT_SIZE = 2884802L; /** Altitude below which is considered void */ private static final int VOID_VAL = -32768; @@ -84,7 +72,10 @@ public class LookupSrtmFunction extends GenericFunction implements Runnable private void begin(Track inTrack, boolean inNormalTrack) { _running = true; - _hadToDownload = false; + if (! SrtmDiskCache.ensureCacheIsUsable()) + { + _app.showErrorMessage(getNameKey(), "error.cache.notthere"); + } if (_progress == null) { _progress = new ProgressDialog(_parentFrame, getNameKey()); } @@ -131,8 +122,7 @@ public class LookupSrtmFunction extends GenericFunction implements Runnable for (int i = 0; i < _track.getNumPoints(); i++) { // Consider points which don't have altitudes or have zero values - if (!_track.getPoint(i).hasAltitude() - || (overwriteZeros && _track.getPoint(i).getAltitude().getValue() == 0)) + if (needsAltitude(_track.getPoint(i), overwriteZeros)) { SrtmTile tile = new SrtmTile(_track.getPoint(i)); boolean alreadyGot = false; @@ -148,12 +138,23 @@ public class LookupSrtmFunction extends GenericFunction implements Runnable lookupValues(tileList, overwriteZeros); // Finished _running = false; - // Show tip if lots of online lookups were necessary - if (_hadToDownload) { - _app.showTip(TipManager.Tip_DownloadSrtm); - } } + /** + * true if we need to set the altitude of this point + */ + private boolean needsAltitude(DataPoint point, boolean overwriteZeros) + { + if (!point.hasAltitude()) + { + return true; + } + if (overwriteZeros && point.getAltitude().getValue() == 0) + { + return true; + } + return false; + } /** * Lookup the values from SRTM data @@ -170,94 +171,79 @@ public class LookupSrtmFunction extends GenericFunction implements Runnable _progress.setMaximum(inTileList.size()); _progress.setValue(0); } - String errorMessage = null; - // Get urls for each tile - URL[] urls = TileFinder.getUrls(inTileList); + String errorMessage = ""; for (int t=0; t 0) System.out.println(numVoids + " voids found"); + double altitude = 0.0; + switch (numVoids) { - heights[i] = inStream.read() * 256 + inStream.read(); - if (heights[i] >= 32768) {heights[i] -= 65536;} + case 0: altitude = bilinearInterpolate(fouralts, x, y); break; + case 1: altitude = bilinearInterpolate(fixVoid(fouralts), x, y); break; + case 2: + case 3: altitude = averageNonVoid(fouralts); break; + default: altitude = VOID_VAL; } - } - // else { - // System.out.println("length not ok: " + entry.getSize()); - // } - // Close stream from url - inStream.close(); - } - - if (entryOk) - { - // Loop over all points in track, try to apply altitude from array - for (int p = 0; p < _track.getNumPoints(); p++) - { - DataPoint point = _track.getPoint(p); - if (!point.hasAltitude() - || (inOverwriteZeros && point.getAltitude().getValue() == 0)) + // Special case for terrain tracks, don't interpolate voids yet + if (!_normalTrack && numVoids > 0) { + altitude = VOID_VAL; + } + if (altitude != VOID_VAL) { - if (new SrtmTile(point).equals(tile)) - { - double x = (point.getLongitude().getDouble() - tile.getLongitude()) * 1200; - double y = 1201 - (point.getLatitude().getDouble() - tile.getLatitude()) * 1200; - int idx1 = ((int)y)*1201 + (int)x; - try - { - int[] fouralts = {heights[idx1], heights[idx1+1], heights[idx1-1201], heights[idx1-1200]}; - int numVoids = (fouralts[0]==VOID_VAL?1:0) + (fouralts[1]==VOID_VAL?1:0) - + (fouralts[2]==VOID_VAL?1:0) + (fouralts[3]==VOID_VAL?1:0); - // if (numVoids > 0) System.out.println(numVoids + " voids found"); - double altitude = 0.0; - switch (numVoids) - { - case 0: altitude = bilinearInterpolate(fouralts, x, y); break; - case 1: altitude = bilinearInterpolate(fixVoid(fouralts), x, y); break; - case 2: - case 3: altitude = averageNonVoid(fouralts); break; - default: altitude = VOID_VAL; - } - // Special case for terrain tracks, don't interpolate voids yet - if (!_normalTrack && numVoids > 0) { - altitude = VOID_VAL; - } - if (altitude != VOID_VAL) - { - point.setFieldValue(Field.ALTITUDE, ""+altitude, false); - // depending on settings, this value may have been added as feet, we need to force metres - point.getAltitude().reset(new Altitude((int)altitude, UnitSetLibrary.UNITS_METRES)); - numAltitudesFound++; - } - } - catch (ArrayIndexOutOfBoundsException obe) { - // System.err.println("lat=" + point.getLatitude().getDouble() + ", x=" + x + ", y=" + y + ", idx=" + idx1); - } - } + point.setFieldValue(Field.ALTITUDE, ""+altitude, false); + // depending on settings, this value may have been added as feet, we need to force metres + point.getAltitude().reset(new Altitude((int)altitude, UnitSetLibrary.UNITS_METRES)); + numAltitudesFound++; } } + catch (ArrayIndexOutOfBoundsException obe) { + errorMessage += "Point not in tile? lat=" + point.getLatitude().getDouble() + ", x=" + x + ", y=" + y + ", idx=" + idx1+"\n"; + } } } - catch (IOException ioe) {errorMessage = ioe.getClass().getName() + " - " + ioe.getMessage(); - } } } @@ -266,6 +252,10 @@ public class LookupSrtmFunction extends GenericFunction implements Runnable return; } + if (! errorMessage.equals("")) { + _app.showErrorMessageNoLookup(getNameKey(), errorMessage); + return; + } if (numAltitudesFound > 0) { // Inform app including undo information @@ -278,9 +268,6 @@ public class LookupSrtmFunction extends GenericFunction implements Runnable I18nManager.getTextWithNumber("confirm.lookupsrtm", numAltitudesFound)); } } - else if (errorMessage != null) { - _app.showErrorMessageNoLookup(getNameKey(), errorMessage); - } else if (inTileList.size() > 0) { _app.showErrorMessage(getNameKey(), "error.lookupsrtm.nonefound"); } @@ -289,36 +276,6 @@ public class LookupSrtmFunction extends GenericFunction implements Runnable } } - /** - * See whether the SRTM file is already available locally first, then try online - * @param inUrl URL for online resource - * @return ZipInputStream either on the local file or on the downloaded zip file - */ - private ZipInputStream getStreamToHgtFile(URL inUrl) - throws IOException - { - String diskCachePath = Config.getConfigString(Config.KEY_DISK_CACHE); - if (diskCachePath != null) - { - File srtmDir = new File(diskCachePath, "srtm"); - if (srtmDir.exists() && srtmDir.isDirectory() && srtmDir.canRead()) - { - File srtmFile = new File(srtmDir, new File(inUrl.getFile()).getName()); - if (srtmFile.exists() && srtmFile.isFile() && srtmFile.canRead() - && srtmFile.length() > 400) - { - // System.out.println("Lookup: Using file " + srtmFile.getAbsolutePath()); - // File found, use this one - return new ZipInputStream(new FileInputStream(srtmFile)); - } - } - } - // System.out.println("Lookup: Trying online: " + inUrl.toString()); - _hadToDownload = true; - // MAYBE: Only download if we're in online mode? - return new ZipInputStream(inUrl.openStream()); - } - /** * Perform a bilinear interpolation on the given altitude array * @param inAltitudes array of four altitude values on corners of square (bl, br, tl, tr) diff --git a/src/tim/prune/function/srtm/Srtm3Source.java b/src/tim/prune/function/srtm/Srtm3Source.java new file mode 100644 index 0000000..14fad59 --- /dev/null +++ b/src/tim/prune/function/srtm/Srtm3Source.java @@ -0,0 +1,164 @@ +package tim.prune.function.srtm; + +import java.io.File; +import java.io.FileOutputStream; +import java.io.IOException; +import java.io.InputStream; +import java.net.MalformedURLException; +import java.net.URL; +import java.net.HttpURLConnection; + +import tim.prune.GpsPrune; +import tim.prune.I18nManager; + +public class Srtm3Source extends SrtmSource { + /** URL prefix for all tiles */ + private static final String URL_PREFIX = "https://dds.cr.usgs.gov/srtm/version2_1/SRTM3/"; + /** Directory names for each continent */ + private static final String[] CONTINENTS = {"", "Eurasia", "North_America", "Australia", + "Islands", "South_America", "Africa"}; + private byte[] _continents_lookup; + + + public Srtm3Source() + { + _continents_lookup = populateContinents(); + } + + public String getNameKey() + { + return "function.downloadsrtm." + getName(); + } + + public String getName() + { + return "SRTM3_v21"; + } + + protected String getSourceExtension() + { + return ".hgt.zip"; + } + + /** + * Read the dat file and get the contents + * @return byte array containing file contents + */ + private static byte[] populateContinents() + { + InputStream in = null; + try + { + // Need absolute path to dat file + in = Srtm3Source.class.getResourceAsStream("/tim/prune/function/srtm/srtmtiles.dat"); + if (in != null) + { + byte[] buffer = new byte[in.available()]; + in.read(buffer); + in.close(); + return buffer; + } + } + catch (java.io.IOException e) { + System.err.println("Exception trying to read srtmtiles.dat : " + e.getMessage()); + } + finally + { + try { + in.close(); + } + catch (Exception e) {} // ignore + } + return null; + } + + /** + * Get the Url for the given tile + * @param inTile Tile to get + * @return URL + */ + private URL buildUrl(SrtmTile inTile) + throws SrtmSourceException + { + + // Get byte from lookup array + int idx = (inTile.getLatitude() + 59)*360 + (inTile.getLongitude() + 180); + int dir; + try + { + dir = _continents_lookup[idx]; + } + catch (ArrayIndexOutOfBoundsException e) + { + throw new SrtmSourceException("Could not find continent for tile "+inTile.getTileName()); + } + try + { + return new URL(URL_PREFIX + CONTINENTS[dir] + "/" + inTile.getTileName() + getSourceExtension()); + } + catch (MalformedURLException e) + { + throw new SrtmSourceException("Could not build URL for tile "+inTile.getTileName()); + } + } + + public boolean isReadyToUse() + { + return true; + } + + public boolean downloadTile(SrtmTile inTile) + throws SrtmSourceException + { + int redirects = 5; + URL tileUrl = buildUrl(inTile); + File outputFile = getCacheFileName(inTile); + System.out.println("Download: Need to download: " + tileUrl); + + try + { + HttpURLConnection conn = (HttpURLConnection) tileUrl.openConnection(); + + // Define streams + FileOutputStream outStream = null; + InputStream inStream = null; + + conn.setRequestProperty("User-Agent", "GpsPrune v" + GpsPrune.VERSION_NUMBER); + + int status = conn.getResponseCode(); + if (status == 200) + { + inStream = conn.getInputStream(); + } + else if (status == 404) + { + throw new SrtmSourceException("Tile not found: "+conn.getURL()); + } + else + { + throw new SrtmSourceException("Invalid response from server: " +status+conn.getContent()); + } + + outStream = new FileOutputStream(outputFile); + + int c; + while ((c = inStream.read()) != -1) + { + outStream.write(c); + } + // Make sure streams are closed + try {inStream.close();} catch (Exception e) {} + try {outStream.close();} catch (Exception e) {} + return true; + } + catch (IOException e) + { + throw new SrtmSourceException("Error while downloading tile "+inTile.getTileName()+": "+e.getMessage()); + } + } + + public int getRowSize(SrtmTile inTile) + { + return 1201; + } +} diff --git a/src/tim/prune/function/srtm/SrtmDiskCache.java b/src/tim/prune/function/srtm/SrtmDiskCache.java new file mode 100644 index 0000000..80159e0 --- /dev/null +++ b/src/tim/prune/function/srtm/SrtmDiskCache.java @@ -0,0 +1,49 @@ +package tim.prune.function.srtm; + +import java.io.File; + +import tim.prune.config.Config; +import tim.prune.I18nManager; + +public class SrtmDiskCache { + + private static boolean _cacheIsUsable = false; + private static File _cacheDir = null; + + public static boolean ensureCacheIsUsable() + { + + if (_cacheIsUsable) + { + return true; + } + // Check the cache is ok + String diskCachePath = Config.getConfigString(Config.KEY_DISK_CACHE); + if (diskCachePath == null) + { + return false; + } + File srtmDir = new File(diskCachePath, "srtm"); + if (!srtmDir.exists() && !srtmDir.mkdir()) { + // can't create the srtm directory + return false; + } + _cacheIsUsable = true; + _cacheDir = srtmDir; + return true; + } + + public static File getCacheDir(String inSourceName) + { + if (_cacheDir == null) + { + ensureCacheIsUsable(); + } + File cacheDir = new File(_cacheDir, inSourceName); + if (!cacheDir.exists() && !cacheDir.mkdir()) { + // can't create the srtm directory + return null; + } + return cacheDir; + } +} diff --git a/src/tim/prune/function/srtm/SrtmGl1Source.java b/src/tim/prune/function/srtm/SrtmGl1Source.java new file mode 100644 index 0000000..51e6d77 --- /dev/null +++ b/src/tim/prune/function/srtm/SrtmGl1Source.java @@ -0,0 +1,176 @@ +package tim.prune.function.srtm; + +import java.io.File; +import java.io.FileOutputStream; +import java.io.IOException; +import java.io.InputStream; +import java.net.CookieHandler; +import java.net.CookieManager; +import java.net.CookiePolicy; +import java.net.MalformedURLException; +import java.net.URL; +import java.net.HttpURLConnection; + +import tim.prune.App; +import tim.prune.GpsPrune; +import tim.prune.config.Config; + +/** + * Create an account at: https://urs.earthdata.nasa.gov/users/new + * Data policy: https://lpdaac.usgs.gov/data/data-citation-and-policies/ + * + */ + +public class SrtmGl1Source extends SrtmSource { + /** URL prefix for all tiles */ + private static final String URL_PREFIX = "https://e4ftl01.cr.usgs.gov/MEASURES/SRTMGL1.003/2000.02.11/"; + /** Auth URL */ + private static final String AUTH_URL = "urs.earthdata.nasa.gov"; + + + public SrtmGl1Source() + { + CookieHandler.setDefault(new CookieManager(null, CookiePolicy.ACCEPT_ALL)); + } + + public String getName() + { + return "SRTMGL1_v003"; + } + + protected String getSourceExtension() + { + return ".SRTMGL1.hgt.zip"; + } + + private URL buildUrl(SrtmTile inTile) + throws SrtmSourceException + { + try { + return new URL(URL_PREFIX + inTile.getTileName() + getSourceExtension()); + } + catch (MalformedURLException e) + { + throw new SrtmSourceException(e.getMessage()); + } + } + + public boolean isReadyToUse() + { + return getAuth() != null; + } + + private String getAuth() + { + String authString = Config.getConfigString(Config.KEY_EARTHDATA_AUTH); + if (authString != null) + { + return "Basic " + authString; + } + else + { + return null; + } + } + + public boolean downloadTile(SrtmTile inTile) + throws SrtmSourceException + { + return downloadTile(inTile, getAuth()); + } + + private boolean downloadTile(SrtmTile inTile, String auth) + throws SrtmSourceException + { + URL tileUrl = buildUrl(inTile); + File outputFile = getCacheFileName(inTile); + System.out.println("Download: Need to download: " + tileUrl); + try + { + HttpURLConnection conn = (HttpURLConnection) tileUrl.openConnection(); + long fileLength = 0L; + + // Define streams + FileOutputStream outStream = null; + InputStream inStream = null; + + // Documentation about HTTP interface at: + // https://wiki.earthdata.nasa.gov/display/EL/How+To+Access+Data+With+Java + int redirects = 0; + + while (redirects < 10) { + redirects++; + + conn.setRequestProperty("User-Agent", "GpsPrune v" + GpsPrune.VERSION_NUMBER); + conn.setInstanceFollowRedirects(false); + conn.setUseCaches(false); + if (conn.getURL().getHost().equals(AUTH_URL)) + { + conn.setRequestProperty("Authorization", auth); + } + + int status = conn.getResponseCode(); + if (status == 200) + { + // Found the tile, we're good + inStream = conn.getInputStream(); + fileLength = conn.getContentLengthLong(); + break; + } + else if (status == 302) + { + // redirected to SSO server then back to original resource + String newUrl = conn.getHeaderField("Location"); + conn = (HttpURLConnection) (new URL(newUrl)).openConnection(); + } + else if (status == 404) + { + throw new SrtmSourceException("Tile " + inTile.getTileName() + " not found at " + conn.getURL()); + } + else + { + throw new SrtmSourceException("Invalid response from server: " + status + conn.getResponseMessage()); + } + } + + // _progress.setValue(t * 10 + 1); + outStream = new FileOutputStream(outputFile); + + // Copy all the bytes to the file + int c; + long written = 0L; + while ((c = inStream.read()) != -1) + { + outStream.write(c); + written++; + // _progress.setValue(t * 10 + 1 + (int) ((10 * written) / fileLength)); + } + // Make sure streams are closed + try {inStream.close();} catch (Exception e) {} + try {outStream.close();} catch (Exception e) {} + return true; + } + catch (IOException e) + { + throw new SrtmSourceException("Error while downloading tile " + inTile.getTileName() + ": "+e.getMessage()); + } + } + + public boolean testAuth(String auth) + throws SrtmSourceException + { + // The only thing special about this tile is that it's the smallest tile + // It covers small islands in Malaysia + SrtmTile testTile = new SrtmTile(7, 117); + if (isCached(testTile)) + { + getCacheFileName(testTile).delete(); + } + return downloadTile(testTile, auth); + } + + public int getRowSize(SrtmTile inTile) + { + return 3601; + } +} diff --git a/src/tim/prune/function/srtm/SrtmSource.java b/src/tim/prune/function/srtm/SrtmSource.java new file mode 100644 index 0000000..5651a3f --- /dev/null +++ b/src/tim/prune/function/srtm/SrtmSource.java @@ -0,0 +1,69 @@ +package tim.prune.function.srtm; + +import java.io.File; +import java.io.FileInputStream; +import java.io.FileOutputStream; +import java.io.IOException; +import java.util.zip.ZipEntry; +import java.util.zip.ZipInputStream; + +public abstract class SrtmSource { + public abstract String getName(); + public abstract boolean isReadyToUse(); + public abstract boolean downloadTile(SrtmTile inTile) + throws SrtmSourceException; + public abstract int getRowSize(SrtmTile inTile); + protected abstract String getSourceExtension(); + + public int[] getTileHeights(SrtmTile inTile) + throws SrtmSourceException + { + File cacheFileName = getCacheFileName(inTile); + if (cacheFileName == null) + { + throw new SrtmSourceException("Tile "+inTile.getTileName()+" not in cache"); + } + try + { + ZipInputStream inStream = new ZipInputStream(new FileInputStream(cacheFileName)); + ZipEntry entry = inStream.getNextEntry(); + int rowSize = getRowSize(inTile); + int tileSize = rowSize * rowSize; + if (entry.getSize() != 2 * tileSize) + { + throw new SrtmSourceException("Tile file "+cacheFileName+" does not have the expected size"); + } + int[] heights = new int[tileSize]; + // Read entire file contents into one byte array + for (int i = 0; i < heights.length; i++) + { + heights[i] = inStream.read() * 256 + inStream.read(); + if (heights[i] >= 32768) {heights[i] -= 65536;} + } + // Close stream + inStream.close(); + return heights; + } + catch (IOException e) + { + throw new SrtmSourceException("Failure opening "+cacheFileName+" for reading:"+e.getMessage()); + } + + } + + protected File getCacheDir() + { + return SrtmDiskCache.getCacheDir(getName()); + } + + protected File getCacheFileName(SrtmTile inTile) + { + String fileName = inTile.getTileName() + getSourceExtension(); + return new File(getCacheDir(), fileName); + } + + public boolean isCached(SrtmTile inTile) + { + return getCacheFileName(inTile).exists(); + } +} diff --git a/src/tim/prune/function/srtm/SrtmSourceException.java b/src/tim/prune/function/srtm/SrtmSourceException.java new file mode 100644 index 0000000..37b2db5 --- /dev/null +++ b/src/tim/prune/function/srtm/SrtmSourceException.java @@ -0,0 +1,7 @@ +package tim.prune.function.srtm; + +public class SrtmSourceException extends Exception { + public SrtmSourceException(String message) { + super(message); + } +} diff --git a/src/tim/prune/function/srtm/SrtmTile.java b/src/tim/prune/function/srtm/SrtmTile.java index 301bbaf..7bc1d38 100644 --- a/src/tim/prune/function/srtm/SrtmTile.java +++ b/src/tim/prune/function/srtm/SrtmTile.java @@ -67,7 +67,20 @@ public class SrtmTile + (_longitude >= 0?"E":"W") + (Math.abs(_longitude) < 100?"0":"") + (Math.abs(_longitude) < 10?"0":"") - + Math.abs(_longitude) - + ".hgt.zip"; + + Math.abs(_longitude); + } + + public SrtmSource findBestCachedSource() + { + SrtmSource[] sources = {new SrtmGl1Source(), + new Srtm3Source() }; + for (int i = 0; i < sources.length; i++) + { + if (sources[i].isCached(this)) + { + return sources[i]; + } + } + return null; } } diff --git a/src/tim/prune/function/srtm/TileFinder.java b/src/tim/prune/function/srtm/TileFinder.java deleted file mode 100644 index 60a9479..0000000 --- a/src/tim/prune/function/srtm/TileFinder.java +++ /dev/null @@ -1,82 +0,0 @@ -package tim.prune.function.srtm; - -import java.io.InputStream; -import java.net.MalformedURLException; -import java.net.URL; -import java.util.ArrayList; - - -/** - * Class to get the URLs of the SRTM tiles - * using the srtmtiles.dat file - */ -public abstract class TileFinder -{ - /** URL prefix for all tiles */ - private static final String URL_PREFIX = "https://dds.cr.usgs.gov/srtm/version2_1/SRTM3/"; - /** Directory names for each continent */ - private static final String[] CONTINENTS = {"", "Eurasia", "North_America", "Australia", - "Islands", "South_America", "Africa"}; - - - /** - * Get the Urls for the given list of tiles - * @param inTiles list of Tiles to get - * @return array of URLs - */ - public static URL[] getUrls(ArrayList inTiles) - { - if (inTiles == null || inTiles.size() < 1) {return null;} - URL[] urls = new URL[inTiles.size()]; - // Read dat file into array - byte[] lookup = readDatFile(); - for (int t=0; t 0) { - try { - urls[t] = new URL(URL_PREFIX + CONTINENTS[dir] + "/" + tile.getTileName()); - } catch (MalformedURLException e) {} // ignore error, url stays null - } - } catch (ArrayIndexOutOfBoundsException e) {} // ignore error, url stays null - } - return urls; - } - - /** - * Read the dat file and get the contents - * @return byte array containing file contents - */ - private static byte[] readDatFile() - { - InputStream in = null; - try - { - // Need absolute path to dat file - in = TileFinder.class.getResourceAsStream("/tim/prune/function/srtm/srtmtiles.dat"); - if (in != null) - { - byte[] buffer = new byte[in.available()]; - in.read(buffer); - in.close(); - return buffer; - } - } - catch (java.io.IOException e) { - System.err.println("Exception trying to read srtmtiles.dat : " + e.getMessage()); - } - finally - { - try { - in.close(); - } - catch (Exception e) {} // ignore - } - return null; - } -} diff --git a/src/tim/prune/gui/MenuManager.java b/src/tim/prune/gui/MenuManager.java index bce3b4d..89903d8 100644 --- a/src/tim/prune/gui/MenuManager.java +++ b/src/tim/prune/gui/MenuManager.java @@ -34,6 +34,9 @@ import tim.prune.function.SearchOpenCachingDeFunction; import tim.prune.function.browser.UrlGenerator; import tim.prune.function.browser.WebMapFunction; import tim.prune.function.search.SearchMapillaryFunction; +import tim.prune.function.srtm.DownloadSrtmFunction; +import tim.prune.function.srtm.SrtmGl1Source; +import tim.prune.function.srtm.Srtm3Source; /** * Class to manage the menu bar and tool bar, @@ -93,7 +96,7 @@ public class MenuManager implements DataSubscriber private JMenuItem _getGpsiesItem = null; private JMenuItem _uploadGpsiesItem = null; private JMenuItem _lookupSrtmItem = null; - private JMenuItem _downloadSrtmItem = null; + private JMenu _downloadSrtmMenu = null; private JMenuItem _nearbyWikipediaItem = null; private JMenuItem _nearbyOsmPoiItem = null; private JMenuItem _showPeakfinderItem = null; @@ -257,8 +260,15 @@ public class MenuManager implements DataSubscriber // SRTM _lookupSrtmItem = makeMenuItem(FunctionLibrary.FUNCTION_LOOKUP_SRTM, false); onlineMenu.add(_lookupSrtmItem); - _downloadSrtmItem = makeMenuItem(FunctionLibrary.FUNCTION_DOWNLOAD_SRTM, false); - onlineMenu.add(_downloadSrtmItem); + // Download SRTM sub-menu + _downloadSrtmMenu = new JMenu(I18nManager.getText("function.downloadsrtm")); + _downloadSrtmMenu.setEnabled(false); + JMenuItem downloadStrmGl1Item = makeMenuItem(new DownloadSrtmFunction(_app, new SrtmGl1Source())); + _downloadSrtmMenu.add(downloadStrmGl1Item); + JMenuItem downloadStrm3Item = makeMenuItem(new DownloadSrtmFunction(_app, new Srtm3Source())); + _downloadSrtmMenu.add(downloadStrm3Item); + onlineMenu.add(_downloadSrtmMenu); + // Get gpsies tracks _getGpsiesItem = makeMenuItem(FunctionLibrary.FUNCTION_GET_GPSIES, false); onlineMenu.add(_getGpsiesItem); @@ -889,7 +899,7 @@ public class MenuManager implements DataSubscriber _getWeatherItem.setEnabled(hasData); _findWaypointItem.setEnabled(hasData && _track.hasWaypoints()); // have we got a cache? - _downloadSrtmItem.setEnabled(hasData && Config.getConfigString(Config.KEY_DISK_CACHE) != null); + _downloadSrtmMenu.setEnabled(hasData && Config.getConfigString(Config.KEY_DISK_CACHE) != null); // have we got any timestamps? _deleteByDateItem.setEnabled(hasData && _track.hasData(Field.TIMESTAMP)); diff --git a/src/tim/prune/lang/prune-texts_en.properties b/src/tim/prune/lang/prune-texts_en.properties index 7693926..9e540eb 100644 --- a/src/tim/prune/lang/prune-texts_en.properties +++ b/src/tim/prune/lang/prune-texts_en.properties @@ -114,6 +114,9 @@ function.getgpsies=Get Gpsies tracks function.uploadgpsies=Upload track to Gpsies function.lookupsrtm=Get altitudes from SRTM function.downloadsrtm=Download SRTM tiles +function.downloadsrtm.SRTMGL1_v003=Download SRTM 1 arc-second tiles +function.downloadsrtm.SRTMGL1_v003.needsetup=An Earthdata account is necessary to download SRTM 1 arc-second tiles +function.downloadsrtm.SRTM3_v21=Download SRTM 3 arc-second tiles function.getwikipedia=Get nearby Wikipedia articles function.searchwikipedianames=Search Wikipedia by name function.searchosmpois=Get nearby OSM points