X-Git-Url: http://gitweb.fperrin.net/?a=blobdiff_plain;f=src%2Fcom%2Fhughes%2Fandroid%2Fdictionary%2Fengine%2FDictionaryBuilderMain.java;h=9f41c83344bd8b6d03d36becebc2998b6b8d1386;hb=511f5633b8f21a7929b0658e06245bb5092a313d;hp=ea9e1a72f1e6446264057eb9746f315103deb751;hpb=67780552c216010c96a4823a2f2f44e6f206c8f2;p=DictionaryPC.git diff --git a/src/com/hughes/android/dictionary/engine/DictionaryBuilderMain.java b/src/com/hughes/android/dictionary/engine/DictionaryBuilderMain.java index ea9e1a7..9f41c83 100644 --- a/src/com/hughes/android/dictionary/engine/DictionaryBuilderMain.java +++ b/src/com/hughes/android/dictionary/engine/DictionaryBuilderMain.java @@ -34,6 +34,108 @@ public class DictionaryBuilderMain extends TestCase { static final String INPUTS = "data/inputs/"; static final String STOPLISTS = "data/inputs/stoplists/"; static final String OUTPUTS = "data/outputs/"; + + // Build the non EN ones. + static final String[][] nonEnPairs = new String[][] { + {"EN"}, + {"DE"}, + {"IT"}, + {"FR"}, + + // The 3 I use most: + {"IT", "EN" }, + {"DE", "EN" }, + {"DE", "IT" }, + + {"AR", "DE" }, + {"AR", "ES" }, + {"AR", "FR" }, + {"AR", "HE" }, + {"AR", "IT" }, + {"AR", "JA" }, + {"AR", "RU" }, + {"AR", "TR" }, // Turkish + {"AR", "ZH" }, + + {"DE", "AR" }, + {"DE", "FR" }, + {"DE", "CA" }, // Catalan + {"DE", "CS" }, // Czech + {"DE", "EO" }, // Esperanto + {"DE", "ES" }, + {"DE", "FR" }, + {"DE", "HE" }, + {"DE", "HU" }, // Hungarian + {"DE", "IT" }, + {"DE", "JA" }, + {"DE", "LA" }, // Latin + {"DE", "NL" }, // Dutch + {"DE", "PL" }, // Polish + {"DE", "RU" }, + {"DE", "SV" }, // Swedish + {"DE", "TR" }, // Turkish + {"DE", "ZH" }, + {"DE", "TA" }, // Tamil + + {"ES", "RU" }, // Spanish-Russian + + {"FR", "BG" }, // Bulgarian + {"FR", "CS" }, // Czech + {"FR", "DE" }, + {"FR", "ES" }, + {"FR", "IT" }, + {"FR", "JA" }, + {"FR", "LA" }, + {"FR", "NL" }, // Dutch + {"FR", "RU" }, + {"FR", "TR" }, // Turkish + {"FR", "ZH" }, + {"FR", "EL" }, + + {"IT", "DE" }, + {"IT", "EL" }, // Greek + {"IT", "ES" }, + {"IT", "FR" }, + {"IT", "HU" }, + {"IT", "JA" }, + {"IT", "LA" }, // Latin + {"IT", "LV" }, // Latvian + {"IT", "NL" }, + {"IT", "PL" }, + {"IT", "RU" }, + {"IT", "SV" }, + {"IT", "TR" }, // Turkish + {"IT", "ZH" }, + + {"JA", "ZH" }, + {"JA", "AR" }, + {"JA", "KO" }, + + {"ZH", "AR" }, + {"ZH", "DE" }, + {"ZH", "ES" }, + {"ZH", "FR" }, + {"ZH", "IT" }, + {"ZH", "KO" }, + + + {"NO", "SV" }, + {"NO", "FI" }, + {"FI", "SV" }, + + {"PL", "FR" }, // Polish + {"PL", "RU" }, // Polish + {"PL", "HU" }, // Polish + {"PL", "ES" }, // Polish + + {"TR", "EL" }, // Turkish, Greek + + {"FA", "HY" }, // Persian, Armenian, by request. + {"FA", "SV" }, // Persian, Swedish, by request. + {"NL", "PL" }, // Dutch, Polish, by request. + + }; + static final Map isoToDedication = new LinkedHashMap(); @@ -75,7 +177,48 @@ public class DictionaryBuilderMain extends TestCase { } static List getMainArgs(final String[] pair) { - final List result = new ArrayList(); + final List result = new ArrayList(); + + int i = 1; + + if (pair.length == 1) { + final String lang1 = pair[0]; + final String dictFile = String.format("%s/%s.quickdic", OUTPUTS, lang1); + result.add(String.format("--dictOut=%s", dictFile)); + result.add(String.format("--lang1=%s", lang1)); + result.add(String.format("--lang1Stoplist=%s", STOPLISTS + getStoplist(lang1))); + result.add(String.format("--dictInfo=Wikitionary-based %s dictionary.", lang1)); + + + final String wikiSplitFile = String.format("%s/wikiSplit/%s/%s.data", INPUTS, lang1.toLowerCase(), lang1); + if (new File(wikiSplitFile).canRead()) { + result.add(String.format("--input%d=%s", i, wikiSplitFile)); + result.add(String.format("--input%dName=%s.wiktionary.org", i, lang1.toLowerCase())); + result.add(String.format("--input%dFormat=%s", i, WholeSectionToHtmlParser.NAME)); + result.add(String.format("--input%dTitleIndex=%d", i, 1)); + result.add(String.format("--input%dWiktionaryLang=%s", i, lang1)); + result.add(String.format("--input%dSkipLang=%s", i, lang1)); + result.add(String.format("--input%dWebUrlTemplate=http://%s.wiktionary.org/wiki/%%s", i, lang1.toLowerCase())); + //result.add(String.format("--input%dPageLimit=100", i)); + ++i; + } else { + System.err.println("Can't read file: " + wikiSplitFile); + } + + if (lang1.equals("EN")) { + result.add(String.format("--input%d=%s/wikiSplit/en/%s.data", i, INPUTS, lang1)); + result.add(String.format("--input%dName=ENWiktionary.%s", i, lang1)) ; + result.add(String.format("--input%dFormat=enwiktionary", i)); + result.add(String.format("--input%dWiktionaryType=EnEnglish", i)); + result.add(String.format("--input%dLangPattern=%s", i, "English")); + result.add(String.format("--input%dLangCodePattern=%s", i, lang1.toLowerCase())); + result.add(String.format("--input%dEnIndex=%d", i, 1)); + //result.add(String.format("--input%dPageLimit=100", i)); + ++i; + } + + return result; + } // Single-lang dictionaries. final String lang1 = pair[0]; final String lang2 = pair[1]; @@ -87,8 +230,6 @@ public class DictionaryBuilderMain extends TestCase { result.add(String.format("--lang1Stoplist=%s", STOPLISTS + getStoplist(lang1))); result.add(String.format("--lang2Stoplist=%s", STOPLISTS + getStoplist(lang2))); - int i = 1; - // For a few langs, put the defs of the other language in DE/IT/FR using WholeSection. for (final String wikitionaryLang : Arrays.asList("EN", "DE", "IT", "FR")) { if (!Arrays.asList(pair).contains(wikitionaryLang)) { @@ -101,11 +242,12 @@ public class DictionaryBuilderMain extends TestCase { continue; } result.add(String.format("--input%d=%s", i, wikiSplitFile)); - result.add(String.format("--input%dName=%s.wiktionary.org (%s)", i, wikitionaryLang, foreignIso)); + result.add(String.format("--input%dName=%s.wiktionary.org", i, wikitionaryLang.toLowerCase())); result.add(String.format("--input%dFormat=%s", i, WholeSectionToHtmlParser.NAME)); result.add(String.format("--input%dTitleIndex=%d", i, Arrays.asList(pair).indexOf(foreignIso) + 1)); result.add(String.format("--input%dWiktionaryLang=%s", i, wikitionaryLang)); result.add(String.format("--input%dSkipLang=%s", i, foreignIso)); + result.add(String.format("--input%dWebUrlTemplate=http://%s.wiktionary.org/wiki/%%s", i, wikitionaryLang.toLowerCase())); ++i; } @@ -170,111 +312,13 @@ public class DictionaryBuilderMain extends TestCase { final List allPairs = new ArrayList(); - // Build the non EN ones. - final String[][] nonEnPairs = new String[][] { - - // The 3 I use most: - {"IT", "EN" }, - {"DE", "EN" }, - {"DE", "IT" }, - - - {"AR", "DE" }, - {"AR", "ES" }, - {"AR", "FR" }, - {"AR", "HE" }, - {"AR", "IT" }, - {"AR", "JA" }, - {"AR", "RU" }, - {"AR", "TR" }, // Turkish - {"AR", "ZH" }, - - {"DE", "AR" }, - {"DE", "FR" }, - {"DE", "CA" }, // Catalan - {"DE", "CS" }, // Czech - {"DE", "EO" }, // Esperanto - {"DE", "ES" }, - {"DE", "FR" }, - {"DE", "HE" }, - {"DE", "HU" }, // Hungarian - {"DE", "IT" }, - {"DE", "JA" }, - {"DE", "LA" }, // Latin - {"DE", "NL" }, // Dutch - {"DE", "PL" }, // Polish - {"DE", "RU" }, - {"DE", "SV" }, // Swedish - {"DE", "TR" }, // Turkish - {"DE", "ZH" }, - {"DE", "TA" }, // Tamil - - {"ES", "RU" }, // Spanish-Russian - - {"FR", "BG" }, // Bulgarian - {"FR", "CS" }, // Czech - {"FR", "DE" }, - {"FR", "ES" }, - {"FR", "IT" }, - {"FR", "JA" }, - {"FR", "LA" }, - {"FR", "NL" }, // Dutch - {"FR", "RU" }, - {"FR", "TR" }, // Turkish - {"FR", "ZH" }, - - {"IT", "DE" }, - {"IT", "EL" }, // Greek - {"IT", "ES" }, - {"IT", "FR" }, - {"IT", "HU" }, - {"IT", "JA" }, - {"IT", "LA" }, // Latin - {"IT", "LV" }, // Latvian - {"IT", "NL" }, - {"IT", "PL" }, - {"IT", "RU" }, - {"IT", "SV" }, - {"IT", "TR" }, // Turkish - {"IT", "ZH" }, - - {"JA", "ZH" }, - {"JA", "AR" }, - {"JA", "KO" }, - - {"ZH", "AR" }, - {"ZH", "DE" }, - {"ZH", "ES" }, - {"ZH", "FR" }, - {"ZH", "IT" }, - {"ZH", "KO" }, - - - {"NO", "SV" }, - {"NO", "FI" }, - {"FI", "SV" }, - - {"PL", "FR" }, // Polish - {"PL", "RU" }, // Polish - {"PL", "HU" }, // Polish - {"PL", "ES" }, // Polish - - {"TR", "EL" }, // Turkish, Greek - - {"FA", "HY" }, // Persian, Armenian, by request. - {"FA", "SV" }, // Persian, Swedish, by request. - - }; allPairs.addAll(Arrays.asList(nonEnPairs)); - // Add all the EN-XX pairs. for (final String isoCode : WiktionaryLangs.isoCodeToEnWikiName.keySet()) { - if (isoCode.equals("EN") || isoCode.equals("DE")) { - continue; + if (!isoCode.equals("EN")) { + allPairs.add(new String[] {"EN", isoCode}); } - allPairs.add(new String[] {"EN", isoCode}); } - allPairs.add(new String[] {"EN", "DE"}); final Set> done = new LinkedHashSet>(); @@ -286,7 +330,7 @@ public class DictionaryBuilderMain extends TestCase { } done.add(pairList); - if (!pairList.contains("IT") || !pairList.contains("EN")) { + if (!pairList.contains("EN") && !pairList.contains("EL")) { //continue; }