X-Git-Url: http://gitweb.fperrin.net/?a=blobdiff_plain;f=src%2Fcom%2Fhughes%2Fandroid%2Fdictionary%2Fengine%2FDictionaryBuilderMain.java;h=76ae5db4c78c7f3af671e6b44b73b7d476e8d67b;hb=434925c561467d7e204cd80d558f91a8a0a33082;hp=db5c757fe83a391848174eba0047e9d44a97c43c;hpb=d46f529d02bf4306a922c521d032f7620020b1e8;p=DictionaryPC.git diff --git a/src/com/hughes/android/dictionary/engine/DictionaryBuilderMain.java b/src/com/hughes/android/dictionary/engine/DictionaryBuilderMain.java index db5c757..76ae5db 100644 --- a/src/com/hughes/android/dictionary/engine/DictionaryBuilderMain.java +++ b/src/com/hughes/android/dictionary/engine/DictionaryBuilderMain.java @@ -14,6 +14,14 @@ package com.hughes.android.dictionary.engine; +import com.hughes.android.dictionary.parser.wiktionary.EnTranslationToTranslationParser; +import com.hughes.android.dictionary.parser.wiktionary.WholeSectionToHtmlParser; +import com.hughes.android.dictionary.parser.wiktionary.WiktionaryLangs; + +import junit.framework.TestCase; + +import java.io.File; +import java.util.ArrayList; import java.util.Arrays; import java.util.LinkedHashMap; import java.util.LinkedHashSet; @@ -21,57 +29,154 @@ import java.util.List; import java.util.Map; import java.util.Set; -import junit.framework.TestCase; - -import com.hughes.android.dictionary.parser.wiktionary.EnTranslationToTranslationParser; -import com.hughes.android.dictionary.parser.wiktionary.WiktionaryLangs; - public class DictionaryBuilderMain extends TestCase { static final String INPUTS = "data/inputs/"; static final String STOPLISTS = "data/inputs/stoplists/"; static final String OUTPUTS = "data/outputs/"; + + + static final Map isoToDedication = new LinkedHashMap(); + static { + isoToDedication.put("AF", "Afrikaans dictionary dedicated to Heiko and Mariëtte Horn."); + isoToDedication.put("HR", "Croatian dictionary dedicated to Ines Viskic and Miro Kresonja."); + isoToDedication.put("NL", "Dutch dictionary dedicated to Mike LeBeau."); + // German handled in file. + isoToDedication.put("EL", "Greek dictionary dedicated to Noah Egge."); + isoToDedication.put("IT", "Italian dictionary dedicated to Carolina Tropini, my favorite stardust in the whole universe! Ti amo!"); + isoToDedication.put("KO", "Korean dictionary dedicated to Ande Elwood--fall fashion und Fernsehturms!"); + isoToDedication.put("PT", "Portuguese dictionary dedicated to Carlos Melo, one Tough Mudder."); + isoToDedication.put("RO", "Romanian dictionary dedicated to Radu Teodorescu."); + isoToDedication.put("RU", "Russian dictionary dedicated to Maxim Aronin--best friend always!."); + isoToDedication.put("SR", "Serbian dictionary dedicated to Filip Crnogorac--thanks for the honey."); + isoToDedication.put("ES", "Spanish dictionary made especially for Carolina Tropini! <3 XoXoXXXXX!"); + isoToDedication.put("SV", "Swedish dictionary dedicated to Kajsa Palmblad--björn kramar!"); + } + private static String getDedication(String iso) { + return isoToDedication.containsKey(iso) ? "\n\n" + isoToDedication.get(iso) : ""; + } + + static final Map isoToStoplist = new LinkedHashMap(); + static { + isoToStoplist.put("DE", "de.txt"); + isoToStoplist.put("EN", "en.txt"); + isoToStoplist.put("ES", "es.txt"); + isoToStoplist.put("IT", "it.txt"); + isoToStoplist.put("FR", "fr.txt"); + } + private static String getStoplist(String iso) { + return isoToStoplist.containsKey(iso) ? isoToStoplist.get(iso) : "empty.txt"; + } + + static String getOtherLang(final String[] pair, final String first) { + assert Arrays.asList(pair).contains(first); + assert pair.length == 2; + return pair[0].equals(first) ? pair[1] : pair[0]; + } + + static List getMainArgs(final String[] pair) { + final List result = new ArrayList(); - public static void main(final String[] args) throws Exception { + final String lang1 = pair[0]; + final String lang2 = pair[1]; - // Builds all the dictionaries it can, outputs list to a text file. + final String dictFile = String.format("%s/%s-%s.quickdic", + OUTPUTS, lang1, lang2); - final Map isoToWikiName = new LinkedHashMap(WiktionaryLangs.isoCodeToWikiName); - isoToWikiName.remove("EN"); - isoToWikiName.remove("DE"); + result.add(String.format("--dictOut=%s", dictFile)); + result.add(String.format("--lang1Stoplist=%s", STOPLISTS + getStoplist(lang1))); + result.add(String.format("--lang2Stoplist=%s", STOPLISTS + getStoplist(lang2))); - final Map isoToDedication = new LinkedHashMap(); - isoToDedication.put("AF", "Afrikaans dictionary dedicated to Heiko and Mariëtte Horn."); - isoToDedication.put("HR", "Croatian dictionary dedicated to Ines Viskic and Miro Kresonja."); - isoToDedication.put("NL", "Dutch dictionary dedicated to Mike LeBeau."); - // German handled in file. - isoToDedication.put("EL", "Greek dictionary dedicated to Noah Egge."); - isoToDedication.put("IT", "Italian dictionary dedicated to Carolina Tropini, my favorite stardust in the whole universe! Ti amo!"); - isoToDedication.put("KO", "Korean dictionary dedicated to Ande Elwood--fall fashion und Fernsehturms!"); - isoToDedication.put("PT", "Portuguese dictionary dedicated to Carlos Melo, one Tough Mudder."); - isoToDedication.put("RO", "Romanian dictionary dedicated to Radu Teodorescu."); - isoToDedication.put("RU", "Russian dictionary dedicated to Maxim Aronin--best friend always!."); - isoToDedication.put("SR", "Serbian dictionary dedicated to Filip Crnogorac--thanks for the honey."); - isoToDedication.put("ES", "Spanish dictionary made especially for Carolina Tropini! <3 XoXoXXXXX!"); - isoToDedication.put("SV", "Swedish dictionary dedicated to Kajsa Palmblad--björn kramar!"); + int i = 1; + + // For a few langs, put the defs of the other language in DE/IT/FR using WholeSection. + for (final String wikitionaryLang : Arrays.asList("EN", "DE", "IT", "FR")) { + if (!Arrays.asList(pair).contains(wikitionaryLang)) { + continue; + } + final String foreignIso = getOtherLang(pair, wikitionaryLang); + final String wikiSplitFile = String.format("%s/wikiSplit/%s/%s.data", INPUTS, wikitionaryLang.toLowerCase(), foreignIso); + if (!new File(wikiSplitFile).canRead()) { + System.err.println("Can't read file: " + wikiSplitFile); + continue; + } + result.add(String.format("--input%d=%s", i, wikiSplitFile)); + result.add(String.format("--input%dName=%sWiktionary.WholeSections.%s", i, wikitionaryLang, foreignIso)); + result.add(String.format("--input%dFormat=%s", i, WholeSectionToHtmlParser.NAME)); + result.add(String.format("--input%dTitleIndex=%d", i, Arrays.asList(pair).indexOf(foreignIso) + 1)); + ++i; + } + + // Deal with the pairs where one is English. + if (Arrays.asList(pair).contains("EN")) { + final String foreignIso = getOtherLang(pair, "EN"); + + String foreignRegex = WiktionaryLangs.isoCodeToEnWikiName.get(foreignIso); + if (foreignIso.equals("ZH")) { + // HACK: The missing "e" prevents a full match, causing "Cantonese" to be appended to the entries. + foreignRegex = "Chinese|Mandarin|Cantones"; + } + + result.add(String.format("--lang1=%s", lang1)); + result.add(String.format("--lang2=%s", lang2)); + result.add(String.format("--dictInfo=(EN)Wikitionary-based EN-%s dictionary.%s", foreignIso, getDedication(foreignIso))); + + result.add(String.format("--input%d=%s/wikiSplit/en/%s.data", i, INPUTS, foreignIso)); + result.add(String.format("--input%dName=ENWiktionary.%s", i, foreignIso)) ; + result.add(String.format("--input%dFormat=enwiktionary", i)); + result.add(String.format("--input%dWiktionaryType=EnForeign", i)); + result.add(String.format("--input%dLangPattern=%s", i, foreignRegex)); + result.add(String.format("--input%dLangCodePattern=%s", i, foreignIso.toLowerCase())); + result.add(String.format("--input%dEnIndex=%d", i, Arrays.asList(pair).indexOf("EN") + 1)); + ++i; - final Map isoToStoplist = new LinkedHashMap(); - isoToStoplist.put("DE", "de.txt"); - isoToStoplist.put("EN", "en.txt"); - isoToStoplist.put("ES", "es.txt"); - isoToStoplist.put("IT", "it.txt"); - isoToStoplist.put("FR", "fr.txt"); + result.add(String.format("--input%d=%swikiSplit/en/EN.data", i, INPUTS)); + result.add(String.format("--input%dName=enwiktionary.english", i)); + result.add(String.format("--input%dFormat=enwiktionary", i)); + result.add(String.format("--input%dWiktionaryType=EnToTranslation", i)); + result.add(String.format("--input%dLangPattern=%s", i, foreignRegex)); + result.add(String.format("--input%dLangCodePattern=%s", i, foreignIso.toLowerCase())); + result.add(String.format("--input%dEnIndex=%d", i, Arrays.asList(pair).indexOf("EN") + 1)); + ++i; + + if (foreignIso.equals("DE")) { + result.add(String.format("--input%d=%sde-en_chemnitz.txt", i, INPUTS)); + result.add(String.format("--input%dName=chemnitz", i)); + result.add(String.format("--input%dCharset=UTF8", i)); + result.add(String.format("--input%dFormat=chemnitz", i)); + ++i; + } + + } else { + // Pairs without English. + result.add(String.format("--lang1=%s", lang1)); + result.add(String.format("--lang2=%s", lang2)); + result.add(String.format("--dictInfo=Wikitionary-based %s-%s dictionary.", lang1, lang2)); + + result.add(String.format("--input%d=%swikiSplit/en/EN.data", i, INPUTS)); + result.add(String.format("--input%dName=BETA!enwiktionary.%s-%s", i, lang1, lang2)); + result.add(String.format("--input%dFormat=%s", i, EnTranslationToTranslationParser.NAME)); + result.add(String.format("--input%dLangPattern1=%s", i, lang1)); + result.add(String.format("--input%dLangPattern2=%s", i, lang2)); + ++i; + } + + return result; + } - final Map isoToRegex = new LinkedHashMap(); - // HACK: The missing "e" prevents a full match, causing "Cantonese" to be appended to the entries. - isoToRegex.put("ZH", "Chinese|Mandarin|Cantones"); + public static void main(final String[] args) throws Exception { + final List allPairs = new ArrayList(); // Build the non EN ones. - final String[][] nonEnPairs = new String[][] { - - /* + + // The 3 I use most: + {"DE", "EN" }, + {"IT", "EN" }, + {"DE", "IT" }, + + {"AR", "DE" }, {"AR", "ES" }, {"AR", "FR" }, @@ -94,11 +199,13 @@ public class DictionaryBuilderMain extends TestCase { {"DE", "IT" }, {"DE", "JA" }, {"DE", "LA" }, // Latin + {"DE", "NL" }, // Dutch {"DE", "PL" }, // Polish {"DE", "RU" }, {"DE", "SV" }, // Swedish {"DE", "TR" }, // Turkish {"DE", "ZH" }, + {"DE", "TA" }, // Tamil {"FR", "BG" }, // Bulgarian @@ -148,14 +255,27 @@ public class DictionaryBuilderMain extends TestCase { {"PL", "RU" }, // Polish {"PL", "HU" }, // Polish {"PL", "ES" }, // Polish - - */ + {"TR", "EL" }, // Turkish, Greek + + {"FA", "HY" }, // Persian, Armenian, by request. + {"FA", "SV" }, // Persian, Swedish, by request. }; + allPairs.addAll(Arrays.asList(nonEnPairs)); + // Add all the EN-XX pairs. + for (final String isoCode : WiktionaryLangs.isoCodeToEnWikiName.keySet()) { + if (isoCode.equals("EN") || isoCode.equals("DE")) { + continue; + } + allPairs.add(new String[] {"EN", isoCode}); + } + allPairs.add(new String[] {"EN", "DE"}); + + final Set> done = new LinkedHashSet>(); - for (final String[] pair : nonEnPairs) { + for (final String[] pair : allPairs) { Arrays.sort(pair); final List pairList = Arrays.asList(pair); if (done.contains(pairList)) { @@ -163,127 +283,12 @@ public class DictionaryBuilderMain extends TestCase { } done.add(pairList); - final String lang1 = pair[0]; - final String lang2 = pair[1]; - - final String dictFile = String.format("%s/%s-%s_enwiktionary_BETA.quickdic", - OUTPUTS, lang1, lang2); - System.out.println("building dictFile: " + dictFile); - - if (!isoToStoplist.containsKey(lang1)) { - isoToStoplist.put(lang1, "empty.txt"); - } - if (!isoToStoplist.containsKey(lang2)) { - isoToStoplist.put(lang2, "empty.txt"); + if (!pairList.contains("IT") || !pairList.contains("EN")) { + //continue; } - DictionaryBuilder.main(new String[] { - String.format("--dictOut=%s", dictFile), - String.format("--lang1=%s", lang1), - String.format("--lang2=%s", lang2), - String.format("--lang1Stoplist=%s", STOPLISTS + isoToStoplist.get(lang1)), - String.format("--lang2Stoplist=%s", STOPLISTS + isoToStoplist.get(lang2)), - String.format("--dictInfo=(EN)Wikitionary-based %s-%s dictionary.", lang1, lang2), - - String.format("--input2=%swikiSplit/en/EN.data", INPUTS), - String.format("--input2Name=BETA!enwiktionary.%s-%s", lang1, lang2), - String.format("--input2Format=%s", EnTranslationToTranslationParser.NAME), - String.format("--input2LangPattern1=%s", lang1), - String.format("--input2LangPattern2=%s", lang2), - }); - } - if (1==1) { - //return; + DictionaryBuilder.main(getMainArgs(pair).toArray(new String[0])); } - - - // Now build the EN ones. - -// isoToWikiName.keySet().retainAll(Arrays.asList("UK", "HR", "FI")); - //isoToWikiName.clear(); - boolean go = false; - for (final String foreignIso : isoToWikiName.keySet()) { - if (foreignIso.equals("SL")) { - go = true; - } - if (!go) { - continue; - } - - final String dictFile = String.format("%s/EN-%s_enwiktionary.quickdic", OUTPUTS, foreignIso); - System.out.println("building dictFile: " + dictFile); - - if (!isoToStoplist.containsKey(foreignIso)) { - isoToStoplist.put(foreignIso, "empty.txt"); - } - if (!isoToDedication.containsKey(foreignIso)) { - isoToDedication.put(foreignIso, ""); - } - if (!isoToRegex.containsKey(foreignIso)) { - isoToRegex.put(foreignIso, isoToWikiName.get(foreignIso)); - } - - DictionaryBuilder.main(new String[] { - String.format("--dictOut=%s", dictFile), - String.format("--lang1=EN"), - String.format("--lang2=%s", foreignIso), - String.format("--lang1Stoplist=%s", STOPLISTS + isoToStoplist.get("EN")), - String.format("--lang2Stoplist=%s", STOPLISTS + isoToStoplist.get(foreignIso)), - String.format("--dictInfo=(EN)Wikitionary-based EN-%s dictionary.\n\n%s", foreignIso, isoToDedication.get(foreignIso)), - - "--input2=" + INPUTS + "wikiSplit/en/" + foreignIso + ".data", - "--input2Name=enwiktionary." + foreignIso, - "--input2Format=enwiktionary", - "--input2WiktionaryType=EnForeign", - "--input2LangPattern=" + isoToRegex.get(foreignIso), - "--input2LangCodePattern=" + foreignIso.toLowerCase(), - "--input2EnIndex=1", - - "--input3=" + INPUTS + "wikiSplit/en/EN.data", - "--input3Name=enwiktionary.english", - "--input3Format=enwiktionary", - "--input3WiktionaryType=EnToTranslation", - "--input3LangPattern=" + isoToRegex.get(foreignIso), - "--input3LangCodePattern=" + foreignIso.toLowerCase(), - "--input3EnIndex=1", - - }); - - } // foreignIso - - // Now special case German-English. - - final String dictFile = String.format("%s/DE-EN_chemnitz_enwiktionary.quickdic", OUTPUTS); - DictionaryBuilder.main(new String[] { - "--dictOut=" + dictFile, - "--lang1=DE", - "--lang2=EN", - String.format("--lang1Stoplist=%s", STOPLISTS + "de.txt"), - String.format("--lang2Stoplist=%s", STOPLISTS + "en.txt"), - "--dictInfo=@" + INPUTS + "de-en_chemnitz_enwiktionary.info", - - "--input4=" + INPUTS + "de-en_chemnitz.txt", - "--input4Name=chemnitz", - "--input4Charset=UTF8", - "--input4Format=chemnitz", - - "--input2=" + INPUTS + "wikiSplit/en/DE.data", - "--input2Name=enwiktionary.DE", - "--input2Format=enwiktionary", - "--input2WiktionaryType=EnForeign", - "--input2LangPattern=German", - "--input2LangCodePattern=de", - "--input2EnIndex=2", - - "--input3=" + INPUTS + "wikiSplit/en/EN.data", - "--input3Name=enwiktionary.english", - "--input3Format=enwiktionary", - "--input3WiktionaryType=EnToTranslation", - "--input3LangPattern=German", - "--input3LangCodePattern=de", - "--input3EnIndex=2", - }); - - } + } }