From bd60158f97e999cb18e37882b7048221001dbe7b Mon Sep 17 00:00:00 2001 From: thadh Date: Thu, 4 Oct 2012 08:09:10 -0700 Subject: [PATCH] Updated input locations. Moved pairs in builder. --- data/downloadInputs.sh | 20 +- .../engine/DictionaryBuilderMain.java | 204 +++++++++--------- todo.txt | 7 +- 3 files changed, 118 insertions(+), 113 deletions(-) diff --git a/data/downloadInputs.sh b/data/downloadInputs.sh index c14d460..5e7990f 100755 --- a/data/downloadInputs.sh +++ b/data/downloadInputs.sh @@ -8,8 +8,8 @@ echo "Note that unzipping is slow." L=en echo "Downloading from: http://dumps.wikimedia.org/${L}wiktionary/" -WIKI=${L}wiktionary-20120714-pages-articles.xml -curl --remote-name http://dumps.wikimedia.org/${L}wiktionary/20120714/${WIKI}.bz2 +WIKI=${L}wiktionary-20120930-pages-articles.xml +curl --remote-name http://dumps.wikimedia.org/${L}wiktionary/20120930/${WIKI}.bz2 bunzip2 ${WIKI}.bz2 mv ${WIKI} inputs/${L}wiktionary-pages-articles.xml @@ -21,29 +21,29 @@ mv ${CHEMNITZ} inputs/ L=fr echo "Downloading from: http://dumps.wikimedia.org/${L}wiktionary/" -WIKI=${L}wiktionary-20120719-pages-articles.xml -curl --remote-name http://dumps.wikimedia.org/${L}wiktionary/20120719/${WIKI}.bz2 +WIKI=${L}wiktionary-20120926-pages-articles.xml +curl --remote-name http://dumps.wikimedia.org/${L}wiktionary/20120926/${WIKI}.bz2 bunzip2 ${WIKI}.bz2 mv ${WIKI} inputs/${L}wiktionary-pages-articles.xml L=it echo "Downloading from: http://dumps.wikimedia.org/${L}wiktionary/" -WIKI=${L}wiktionary-20120720-pages-articles.xml -curl --remote-name http://dumps.wikimedia.org/${L}wiktionary/20120720/${WIKI}.bz2 +WIKI=${L}wiktionary-20120926-pages-articles.xml +curl --remote-name http://dumps.wikimedia.org/${L}wiktionary/20120926/${WIKI}.bz2 bunzip2 ${WIKI}.bz2 mv ${WIKI} inputs/${L}wiktionary-pages-articles.xml L=de echo "Downloading from: http://dumps.wikimedia.org/${L}wiktionary/" -WIKI=${L}wiktionary-20120714-pages-articles.xml -curl --remote-name http://dumps.wikimedia.org/${L}wiktionary/20120714/${WIKI}.bz2 +WIKI=${L}wiktionary-20120928-pages-articles.xml +curl --remote-name http://dumps.wikimedia.org/${L}wiktionary/20120928/${WIKI}.bz2 bunzip2 ${WIKI}.bz2 mv ${WIKI} inputs/${L}wiktionary-pages-articles.xml L=es echo "Downloading from: http://dumps.wikimedia.org/${L}wiktionary/" -WIKI=${L}wiktionary-20120718-pages-articles.xml -curl --remote-name http://dumps.wikimedia.org/${L}wiktionary/20120718/${WIKI}.bz2 +WIKI=${L}wiktionary-20120924-pages-articles.xml +curl --remote-name http://dumps.wikimedia.org/${L}wiktionary/20120924/${WIKI}.bz2 bunzip2 ${WIKI}.bz2 mv ${WIKI} inputs/${L}wiktionary-pages-articles.xml diff --git a/src/com/hughes/android/dictionary/engine/DictionaryBuilderMain.java b/src/com/hughes/android/dictionary/engine/DictionaryBuilderMain.java index 5fc5b3c..2677bb4 100644 --- a/src/com/hughes/android/dictionary/engine/DictionaryBuilderMain.java +++ b/src/com/hughes/android/dictionary/engine/DictionaryBuilderMain.java @@ -34,6 +34,107 @@ public class DictionaryBuilderMain extends TestCase { static final String INPUTS = "data/inputs/"; static final String STOPLISTS = "data/inputs/stoplists/"; static final String OUTPUTS = "data/outputs/"; + + // Build the non EN ones. + static final String[][] nonEnPairs = new String[][] { + {"EN"}, + {"DE"}, + {"IT"}, + {"FR"}, + + // The 3 I use most: + {"IT", "EN" }, + {"DE", "EN" }, + {"DE", "IT" }, + + + {"AR", "DE" }, + {"AR", "ES" }, + {"AR", "FR" }, + {"AR", "HE" }, + {"AR", "IT" }, + {"AR", "JA" }, + {"AR", "RU" }, + {"AR", "TR" }, // Turkish + {"AR", "ZH" }, + + {"DE", "AR" }, + {"DE", "FR" }, + {"DE", "CA" }, // Catalan + {"DE", "CS" }, // Czech + {"DE", "EO" }, // Esperanto + {"DE", "ES" }, + {"DE", "FR" }, + {"DE", "HE" }, + {"DE", "HU" }, // Hungarian + {"DE", "IT" }, + {"DE", "JA" }, + {"DE", "LA" }, // Latin + {"DE", "NL" }, // Dutch + {"DE", "PL" }, // Polish + {"DE", "RU" }, + {"DE", "SV" }, // Swedish + {"DE", "TR" }, // Turkish + {"DE", "ZH" }, + {"DE", "TA" }, // Tamil + + {"ES", "RU" }, // Spanish-Russian + + {"FR", "BG" }, // Bulgarian + {"FR", "CS" }, // Czech + {"FR", "DE" }, + {"FR", "ES" }, + {"FR", "IT" }, + {"FR", "JA" }, + {"FR", "LA" }, + {"FR", "NL" }, // Dutch + {"FR", "RU" }, + {"FR", "TR" }, // Turkish + {"FR", "ZH" }, + + {"IT", "DE" }, + {"IT", "EL" }, // Greek + {"IT", "ES" }, + {"IT", "FR" }, + {"IT", "HU" }, + {"IT", "JA" }, + {"IT", "LA" }, // Latin + {"IT", "LV" }, // Latvian + {"IT", "NL" }, + {"IT", "PL" }, + {"IT", "RU" }, + {"IT", "SV" }, + {"IT", "TR" }, // Turkish + {"IT", "ZH" }, + + {"JA", "ZH" }, + {"JA", "AR" }, + {"JA", "KO" }, + + {"ZH", "AR" }, + {"ZH", "DE" }, + {"ZH", "ES" }, + {"ZH", "FR" }, + {"ZH", "IT" }, + {"ZH", "KO" }, + + + {"NO", "SV" }, + {"NO", "FI" }, + {"FI", "SV" }, + + {"PL", "FR" }, // Polish + {"PL", "RU" }, // Polish + {"PL", "HU" }, // Polish + {"PL", "ES" }, // Polish + + {"TR", "EL" }, // Turkish, Greek + + {"FA", "HY" }, // Persian, Armenian, by request. + {"FA", "SV" }, // Persian, Swedish, by request. + + }; + static final Map isoToDedication = new LinkedHashMap(); @@ -97,7 +198,7 @@ public class DictionaryBuilderMain extends TestCase { result.add(String.format("--input%dWiktionaryLang=%s", i, lang1)); result.add(String.format("--input%dSkipLang=%s", i, lang1)); result.add(String.format("--input%dWebUrlTemplate=http://%s.wiktionary.org/wiki/%%s", i, lang1.toLowerCase())); - result.add(String.format("--input%dPageLimit=100", i)); + //result.add(String.format("--input%dPageLimit=100", i)); ++i; } else { System.err.println("Can't read file: " + wikiSplitFile); @@ -111,7 +212,7 @@ public class DictionaryBuilderMain extends TestCase { result.add(String.format("--input%dLangPattern=%s", i, "English")); result.add(String.format("--input%dLangCodePattern=%s", i, lang1.toLowerCase())); result.add(String.format("--input%dEnIndex=%d", i, 1)); - result.add(String.format("--input%dPageLimit=100", i)); + //result.add(String.format("--input%dPageLimit=100", i)); ++i; } @@ -210,105 +311,6 @@ public class DictionaryBuilderMain extends TestCase { final List allPairs = new ArrayList(); - // Build the non EN ones. - final String[][] nonEnPairs = new String[][] { - {"EN"}, - {"DE"}, - {"IT"}, - {"FR"}, - - // The 3 I use most: - {"IT", "EN" }, - {"DE", "EN" }, - {"DE", "IT" }, - - - {"AR", "DE" }, - {"AR", "ES" }, - {"AR", "FR" }, - {"AR", "HE" }, - {"AR", "IT" }, - {"AR", "JA" }, - {"AR", "RU" }, - {"AR", "TR" }, // Turkish - {"AR", "ZH" }, - - {"DE", "AR" }, - {"DE", "FR" }, - {"DE", "CA" }, // Catalan - {"DE", "CS" }, // Czech - {"DE", "EO" }, // Esperanto - {"DE", "ES" }, - {"DE", "FR" }, - {"DE", "HE" }, - {"DE", "HU" }, // Hungarian - {"DE", "IT" }, - {"DE", "JA" }, - {"DE", "LA" }, // Latin - {"DE", "NL" }, // Dutch - {"DE", "PL" }, // Polish - {"DE", "RU" }, - {"DE", "SV" }, // Swedish - {"DE", "TR" }, // Turkish - {"DE", "ZH" }, - {"DE", "TA" }, // Tamil - - {"ES", "RU" }, // Spanish-Russian - - {"FR", "BG" }, // Bulgarian - {"FR", "CS" }, // Czech - {"FR", "DE" }, - {"FR", "ES" }, - {"FR", "IT" }, - {"FR", "JA" }, - {"FR", "LA" }, - {"FR", "NL" }, // Dutch - {"FR", "RU" }, - {"FR", "TR" }, // Turkish - {"FR", "ZH" }, - - {"IT", "DE" }, - {"IT", "EL" }, // Greek - {"IT", "ES" }, - {"IT", "FR" }, - {"IT", "HU" }, - {"IT", "JA" }, - {"IT", "LA" }, // Latin - {"IT", "LV" }, // Latvian - {"IT", "NL" }, - {"IT", "PL" }, - {"IT", "RU" }, - {"IT", "SV" }, - {"IT", "TR" }, // Turkish - {"IT", "ZH" }, - - {"JA", "ZH" }, - {"JA", "AR" }, - {"JA", "KO" }, - - {"ZH", "AR" }, - {"ZH", "DE" }, - {"ZH", "ES" }, - {"ZH", "FR" }, - {"ZH", "IT" }, - {"ZH", "KO" }, - - - {"NO", "SV" }, - {"NO", "FI" }, - {"FI", "SV" }, - - {"PL", "FR" }, // Polish - {"PL", "RU" }, // Polish - {"PL", "HU" }, // Polish - {"PL", "ES" }, // Polish - - {"TR", "EL" }, // Turkish, Greek - - {"FA", "HY" }, // Persian, Armenian, by request. - {"FA", "SV" }, // Persian, Swedish, by request. - - }; allPairs.addAll(Arrays.asList(nonEnPairs)); // Add all the EN-XX pairs. diff --git a/todo.txt b/todo.txt index 085a52a..df462df 100644 --- a/todo.txt +++ b/todo.txt @@ -1,6 +1,8 @@ -EN dictionaries. +Compression +{{de-conj +{{term +{{etyl Spaces in links are done wrong: "perche mai",click "why on earth", see "why%20..." -De verbs. Delete it conjugation of entries. @@ -127,4 +129,5 @@ long click on see also works to link Links in HTML work: mostly, they don't open with the keyboard open when edit text loses focus, all highlighted: impossible if it's not focused. Web view search works. +EN dictionaries. \ No newline at end of file -- 2.43.0