X-Git-Url: http://gitweb.fperrin.net/?a=blobdiff_plain;f=src%2Fcom%2Fhughes%2Fandroid%2Fdictionary%2Fengine%2FDictionaryBuilderMain.java;h=0faf144a37e474b755c569f02804ebf51d94999d;hb=0eaf1a63bc6d1145490b64d8c68e5a545401ec16;hp=833e5e97c4fe0a02e1431a1a24f91824e45824c9;hpb=317fb0c2a57c997af6f7f6111d6f423ba3adf1a0;p=DictionaryPC.git diff --git a/src/com/hughes/android/dictionary/engine/DictionaryBuilderMain.java b/src/com/hughes/android/dictionary/engine/DictionaryBuilderMain.java index 833e5e9..0faf144 100644 --- a/src/com/hughes/android/dictionary/engine/DictionaryBuilderMain.java +++ b/src/com/hughes/android/dictionary/engine/DictionaryBuilderMain.java @@ -14,177 +14,276 @@ package com.hughes.android.dictionary.engine; -import java.io.File; -import java.io.PrintWriter; -import java.io.RandomAccessFile; -import java.util.ArrayList; -import java.util.Collections; +import java.util.Arrays; +import java.util.LinkedHashMap; +import java.util.LinkedHashSet; import java.util.List; +import java.util.Map; +import java.util.Set; import junit.framework.TestCase; +import com.hughes.android.dictionary.parser.wiktionary.EnTranslationToTranslationParser; +import com.hughes.android.dictionary.parser.wiktionary.WiktionaryLangs; + public class DictionaryBuilderMain extends TestCase { - static final String INPUTS = "../DictionaryData/inputs/"; - static final String STOPLISTS = "../DictionaryData/inputs/stoplists/"; - static final String OUTPUTS = "../DictionaryData/outputs/"; - - static class Lang { - final String nameRegex; - final String isoCode; - final String wikiSplit; - final String stoplistFile; - public Lang(String nameRegex, String code, final String wikiSplit, final String stoplistFile) { - this.nameRegex = nameRegex; - this.isoCode = code; - this.wikiSplit = wikiSplit; - this.stoplistFile = stoplistFile; - } - } - - - public static void main(final String[] args) throws Exception { + static final String INPUTS = "data/inputs/"; + static final String STOPLISTS = "data/inputs/stoplists/"; + static final String OUTPUTS = "data/outputs/"; + + public static void main(final String[] args) throws Exception { + + // Builds all the dictionaries it can, outputs list to a text file. + + final Map isoToWikiName = new LinkedHashMap(WiktionaryLangs.isoCodeToWikiName); + isoToWikiName.remove("EN"); + isoToWikiName.remove("DE"); - Lang[] langs1 = new Lang[] { - new Lang("^English$", "EN", null, "en.txt"), - }; - Lang[] langs2 = new Lang[] { -// new Lang("^.*Italian.*$", "IT", "italian.data", "it.txt"), -// new Lang("^.*French.*$", "FR", "french.data", "empty.txt"), -// new Lang("^.*Spanish.*$", "ES", "spanish.data", "es.txt"), -// new Lang("^.*Greek.*$", "EL", "greek.data", "el.txt"), -// new Lang("^.*Japanese.*$", "JA", "japanese.data", "empty.txt"), -// new Lang("^.*Chinese.*$|^.*Mandarin.*$", "ZH", "mandarin.data", "empty.txt"), - new Lang("^.*Afrikaans.*$", "AF", "afrikaans.data", "empty.txt"), - new Lang("^.*Arabic.*$", "AR", "".data, "empty.txt"), - new Lang("^.*Hebrew.*$", "HE"), - new Lang("^.*Hindi.*$", "HI"), - new Lang("^.*Icelandic.*$", "IS"), - new Lang("^.*Irish.*$", "GA"), - new Lang("^.*Korean.*$", "KO"), - new Lang("^.*Maori.*$", "MI"), - new Lang("^.*Norwegian.*$", "NO"), - new Lang("^.*Persian.*$", "FA"), - new Lang("^.*Portuguese.*$", "PT"), - new Lang("^.*Romanian.*$", "RO"), - new Lang("^.*Russian.*$", "RU"), - new Lang("^.*Sanskrit.*$", "SA"), - new Lang("^.*Serbian.*$", "SR"), - new Lang("^.*Swedish.*$", "SV"), - new Lang("^.*Tajik.*$", "TG"), - new Lang("^.*Thai.*$", "TH"), - new Lang("^.*Tibetan.*$", "BO"), - new Lang("^.*Turkish.*$", "TR"), - new Lang("^.*Ukranian.*$", "UK"), - new Lang("^.*Vietnamese.*$", "VI"), - new Lang("^.*Welsh.*$", "CY"), - new Lang("^.*Zulu.*$", "ZU"), - new Lang("^.*Croation.*$", "HR"), - new Lang("^.*Czech.*$", "CS"), - new Lang("^.*Dutch.*$", "NL"), - new Lang("^.*Finnish.*$", "FI"), + final Map isoToDedication = new LinkedHashMap(); + isoToDedication.put("AF", "Afrikaans dictionary dedicated to Heiko and Mariëtte Horn."); + isoToDedication.put("HR", "Croatian dictionary dedicated to Ines Viskic and Miro Kresonja."); + isoToDedication.put("NL", "Dutch dictionary dedicated to Mike LeBeau."); + // German handled in file. + isoToDedication.put("EL", "Greek dictionary dedicated to Noah Egge."); + isoToDedication.put("IT", "Italian dictionary dedicated to Carolina Tropini, my favorite stardust in the whole universe! Ti amo!"); + isoToDedication.put("KO", "Korean dictionary dedicated to Ande Elwood--fall fashion und Fernsehturms!"); + isoToDedication.put("PT", "Portuguese dictionary dedicated to Carlos Melo, one Tough Mudder."); + isoToDedication.put("RO", "Romanian dictionary dedicated to Radu Teodorescu."); + isoToDedication.put("RU", "Russian dictionary dedicated to Maxim Aronin--best friend always!."); + isoToDedication.put("SR", "Serbian dictionary dedicated to Filip Crnogorac--thanks for the honey."); + isoToDedication.put("ES", "Spanish dictionary made especially for Carolina Tropini! <3 XoXoXXXXX!"); + isoToDedication.put("SV", "Swedish dictionary dedicated to Kajsa Palmblad--björn kramar!"); + + final Map isoToStoplist = new LinkedHashMap(); + isoToStoplist.put("DE", "de.txt"); + isoToStoplist.put("EN", "en.txt"); + isoToStoplist.put("ES", "es.txt"); + isoToStoplist.put("IT", "it.txt"); + isoToStoplist.put("FR", "fr.txt"); + + final Map isoToRegex = new LinkedHashMap(); + // HACK: The missing "e" prevents a full match, causing "Cantonese" to be appended to the entries. + isoToRegex.put("ZH", "Chinese|Mandarin|Cantones"); + + + // Build the non EN ones. + + final String[][] nonEnPairs = new String[][] { + /* - new Lang("^German$", "DE"), - new Lang("^Armenian$", "HY"), - new Lang("^English$", "EN"), - new Lang("^Kurdish$", "KU"), - new Lang("^Lithuanian$", "LT"), - new Lang("^Malay$", "MS"), - new Lang("^Mongolian$", "MN"), - new Lang("^Somali$", "SO"), - new Lang("^Sudanese$", "SU"), - new Lang("^Yiddish$", "YI"), + {"AR", "DE" }, + {"AR", "ES" }, + {"AR", "FR" }, + {"AR", "HE" }, + {"AR", "IT" }, + {"AR", "JA" }, + {"AR", "RU" }, + {"AR", "TR" }, // Turkish + {"AR", "ZH" }, + + {"DE", "AR" }, + {"DE", "FR" }, + {"DE", "CA" }, // Catalan + {"DE", "CS" }, // Czech + {"DE", "EO" }, // Esperanto + {"DE", "ES" }, + {"DE", "FR" }, + {"DE", "HE" }, + {"DE", "HU" }, // Hungarian + {"DE", "IT" }, + {"DE", "JA" }, + {"DE", "LA" }, // Latin + {"DE", "PL" }, // Polish + {"DE", "RU" }, + {"DE", "SV" }, // Swedish + {"DE", "TR" }, // Turkish + {"DE", "ZH" }, + + + {"FR", "BG" }, // Bulgarian + {"FR", "CS" }, // Czech + {"FR", "DE" }, + {"FR", "ES" }, + {"FR", "IT" }, + {"FR", "JA" }, + {"FR", "LA" }, + {"FR", "NL" }, // Dutch + {"FR", "RU" }, + {"FR", "TR" }, // Turkish + {"FR", "ZH" }, + + {"IT", "DE" }, + {"IT", "EL" }, // Greek + {"IT", "ES" }, + {"IT", "FR" }, + {"IT", "HU" }, + {"IT", "JA" }, + {"IT", "LA" }, // Latin + {"IT", "LV" }, // Latvian + {"IT", "NL" }, + {"IT", "PL" }, + {"IT", "RU" }, + {"IT", "SV" }, + {"IT", "TR" }, // Turkish + {"IT", "ZH" }, + + {"JA", "ZH" }, + {"JA", "AR" }, + {"JA", "KO" }, + + {"ZH", "AR" }, + {"ZH", "DE" }, + {"ZH", "ES" }, + {"ZH", "FR" }, + {"ZH", "IT" }, + {"ZH", "KO" }, + + + {"NO", "SV" }, + {"NO", "FI" }, + {"FI", "SV" }, + + {"PL", "FR" }, // Polish + {"PL", "RU" }, // Polish + {"PL", "HU" }, // Polish + {"PL", "ES" }, // Polish + */ + + }; - for (final Lang lang1 : langs1) { - for (final Lang lang2 : langs2) { - if (lang1.nameRegex.equals(lang2.nameRegex)) { - continue; - } + final Set> done = new LinkedHashSet>(); + for (final String[] pair : nonEnPairs) { + Arrays.sort(pair); + final List pairList = Arrays.asList(pair); + if (done.contains(pairList)) { + continue; + } + done.add(pairList); + + final String lang1 = pair[0]; + final String lang2 = pair[1]; + + final String dictFile = String.format("%s/%s-%s_enwiktionary_BETA.quickdic", + OUTPUTS, lang1, lang2); + System.out.println("building dictFile: " + dictFile); + + if (!isoToStoplist.containsKey(lang1)) { + isoToStoplist.put(lang1, "empty.txt"); + } + if (!isoToStoplist.containsKey(lang2)) { + isoToStoplist.put(lang2, "empty.txt"); + } + + DictionaryBuilder.main(new String[] { + String.format("--dictOut=%s", dictFile), + String.format("--lang1=%s", lang1), + String.format("--lang2=%s", lang2), + String.format("--lang1Stoplist=%s", STOPLISTS + isoToStoplist.get(lang1)), + String.format("--lang2Stoplist=%s", STOPLISTS + isoToStoplist.get(lang2)), + String.format("--dictInfo=(EN)Wikitionary-based %s-%s dictionary.", lang1, lang2), + + String.format("--input2=%swikiSplit/en/EN.data", INPUTS), + String.format("--input2Name=BETA!enwiktionary.%s-%s", lang1, lang2), + String.format("--input2Format=%s", EnTranslationToTranslationParser.NAME), + String.format("--input2LangPattern1=%s", lang1), + String.format("--input2LangPattern2=%s", lang2), + }); + } + if (1==1) { + //return; + } + + + // Now build the EN ones. + +// isoToWikiName.keySet().retainAll(Arrays.asList("UK", "HR", "FI")); + //isoToWikiName.clear(); + boolean go = false; + for (final String foreignIso : isoToWikiName.keySet()) { + if (foreignIso.equals("SL")) { + go = true; + } + if (!go) { + continue; + } + + final String dictFile = String.format("%s/EN-%s_enwiktionary.quickdic", OUTPUTS, foreignIso); + System.out.println("building dictFile: " + dictFile); - int enIndex = -1; - Lang nonEnglish = null; - if (lang2.isoCode.equals("EN")) { - enIndex = 2; - nonEnglish = lang1; + if (!isoToStoplist.containsKey(foreignIso)) { + isoToStoplist.put(foreignIso, "empty.txt"); } - if (lang1.isoCode.equals("EN")) { - enIndex = 1; - nonEnglish = lang2; + if (!isoToDedication.containsKey(foreignIso)) { + isoToDedication.put(foreignIso, ""); } - assert nonEnglish != null; - - final String dictFile = String.format(OUTPUTS + "/%s-%s_enwiktionary.quickdic", lang1.isoCode, lang2.isoCode); - System.out.println("building dictFile: " + dictFile); + if (!isoToRegex.containsKey(foreignIso)) { + isoToRegex.put(foreignIso, isoToWikiName.get(foreignIso)); + } + DictionaryBuilder.main(new String[] { String.format("--dictOut=%s", dictFile), - String.format("--lang1=%s", lang1.isoCode), - String.format("--lang2=%s", lang2.isoCode), - String.format("--lang1Stoplist=%s", STOPLISTS + lang1.stoplistFile), - String.format("--lang2Stoplist=%s", STOPLISTS + lang2.stoplistFile), - String.format("--dictInfo=(EN)Wikitionary-based %s-%s dictionary", lang1.isoCode, lang2.isoCode), - - "--input2=" + INPUTS + "enWikiSplit/" + nonEnglish.wikiSplit, - "--input2Name=enwiktionary." + nonEnglish.wikiSplit, + String.format("--lang1=EN"), + String.format("--lang2=%s", foreignIso), + String.format("--lang1Stoplist=%s", STOPLISTS + isoToStoplist.get("EN")), + String.format("--lang2Stoplist=%s", STOPLISTS + isoToStoplist.get(foreignIso)), + String.format("--dictInfo=(EN)Wikitionary-based EN-%s dictionary.\n\n%s", foreignIso, isoToDedication.get(foreignIso)), + + "--input2=" + INPUTS + "wikiSplit/en/" + foreignIso + ".data", + "--input2Name=enwiktionary." + foreignIso, "--input2Format=enwiktionary", - "--input2LangPattern=" + nonEnglish.nameRegex, - "--input2LangCodePattern=" + nonEnglish.isoCode.toLowerCase(), - "--input2EnIndex=" + enIndex, + "--input2WiktionaryType=EnForeign", + "--input2LangPattern=" + isoToRegex.get(foreignIso), + "--input2LangCodePattern=" + foreignIso.toLowerCase(), + "--input2EnIndex=1", - "--input3=" + INPUTS + "enWikiSplit/english.data", + "--input3=" + INPUTS + "wikiSplit/en/EN.data", "--input3Name=enwiktionary.english", "--input3Format=enwiktionary", - "--input3LangPattern=" + nonEnglish.nameRegex, - "--input3LangCodePattern=" + (enIndex == 1 ? lang2.isoCode : lang1.isoCode).toLowerCase(), - "--input3EnIndex=" + enIndex, + "--input3WiktionaryType=EnToTranslation", + "--input3LangPattern=" + isoToRegex.get(foreignIso), + "--input3LangCodePattern=" + foreignIso.toLowerCase(), + "--input3EnIndex=1", }); - // Print the entries for diffing. - final RandomAccessFile raf = new RandomAccessFile(new File(dictFile), "r"); - final Dictionary dict = new Dictionary(raf); - final PrintWriter textOut = new PrintWriter(new File(dictFile + ".text")); - final List sorted = new ArrayList(dict.pairEntries); - Collections.sort(sorted); - for (final PairEntry pairEntry : sorted) { - textOut.println(pairEntry.getRawText(false)); - } - textOut.close(); - raf.close(); + } // foreignIso - } // langs2 - } // langs1 + // Now special case German-English. + final String dictFile = String.format("%s/DE-EN_chemnitz_enwiktionary.quickdic", OUTPUTS); DictionaryBuilder.main(new String[] { - "--dictOut=" + OUTPUTS + "DE-EN_all_free.quickdic", + "--dictOut=" + dictFile, "--lang1=DE", "--lang2=EN", - "--dictInfo=@" + INPUTS + "de-en_all_free.info", - - "--input1=" + INPUTS + "de-en_chemnitz.txt", - "--input1Name=chemnitz", - "--input1Charset=UTF8", - "--input1Format=chemnitz", - }); + String.format("--lang1Stoplist=%s", STOPLISTS + "de.txt"), + String.format("--lang2Stoplist=%s", STOPLISTS + "en.txt"), + "--dictInfo=@" + INPUTS + "de-en_chemnitz_enwiktionary.info", - DictionaryBuilder.main(new String[] { - "--dictOut=" + OUTPUTS + "de-en_all.quickdic", - "--lang1=DE", - "--lang2=EN", - "--dictInfo=@" + INPUTS + "de-en_all.info", - - "--input2=" + INPUTS + "de-en_chemnitz.txt", - "--input2Name=dictcc", - "--input2Charset=UTF8", - "--input2Format=chemnitz", + "--input4=" + INPUTS + "de-en_chemnitz.txt", + "--input4Name=chemnitz", + "--input4Charset=UTF8", + "--input4Format=chemnitz", + + "--input2=" + INPUTS + "wikiSplit/en/DE.data", + "--input2Name=enwiktionary.DE", + "--input2Format=enwiktionary", + "--input2WiktionaryType=EnForeign", + "--input2LangPattern=German", + "--input2LangCodePattern=de", + "--input2EnIndex=2", - "--input3=" + INPUTS + "/NONFREE/de-en_dictcc.txt", - "--input3Name=dictcc", - "--input3Charset=UTF8", - "--input3Format=dictcc", + "--input3=" + INPUTS + "wikiSplit/en/EN.data", + "--input3Name=enwiktionary.english", + "--input3Format=enwiktionary", + "--input3WiktionaryType=EnToTranslation", + "--input3LangPattern=German", + "--input3LangCodePattern=de", + "--input3EnIndex=2", }); - + } - + }