X-Git-Url: http://gitweb.fperrin.net/?a=blobdiff_plain;f=src%2Fcom%2Fhughes%2Fandroid%2Fdictionary%2Fengine%2FDictionaryBuilderMain.java;h=0731ffb2237b4af66b51b825032f23b4999f5cb4;hb=7d5ada9329d101b59b55691dd2f63ce3e3860011;hp=20bf4ba4437909286baf5b6593199c19ee8a6b8d;hpb=33d9d23508af89ec8e33c07908f4a3c228ac862a;p=DictionaryPC.git diff --git a/src/com/hughes/android/dictionary/engine/DictionaryBuilderMain.java b/src/com/hughes/android/dictionary/engine/DictionaryBuilderMain.java index 20bf4ba..0731ffb 100644 --- a/src/com/hughes/android/dictionary/engine/DictionaryBuilderMain.java +++ b/src/com/hughes/android/dictionary/engine/DictionaryBuilderMain.java @@ -14,37 +14,34 @@ package com.hughes.android.dictionary.engine; -import java.io.File; -import java.io.PrintWriter; -import java.io.RandomAccessFile; -import java.util.ArrayList; -import java.util.Collections; import java.util.LinkedHashMap; -import java.util.List; import java.util.Map; import junit.framework.TestCase; +import com.hughes.android.dictionary.parser.wiktionary.WiktionaryLangs; + public class DictionaryBuilderMain extends TestCase { - static final String INPUTS = "../DictionaryData/inputs/"; - static final String STOPLISTS = "../DictionaryData/inputs/stoplists/"; - static final String OUTPUTS = "../DictionaryData/outputs/"; + static final String INPUTS = "data/inputs/"; + static final String STOPLISTS = "data/inputs/stoplists/"; + static final String OUTPUTS = "data/outputs/"; public static void main(final String[] args) throws Exception { - final Map isoToWikiName = new LinkedHashMap(Language.isoCodeToWikiName); + // Builds all the dictionaries it can, outputs list to a text file. + + final Map isoToWikiName = new LinkedHashMap(WiktionaryLangs.isoCodeToWikiName); isoToWikiName.remove("EN"); isoToWikiName.remove("DE"); final Map isoToDedication = new LinkedHashMap(); isoToDedication.put("AF", "Afrikaans dictionary dedicated to Heiko and Mariëtte Horn."); - isoToDedication.put("HR", "Croation dictionary dedicated to Ines Viskic and Miro Kresonja."); + isoToDedication.put("HR", "Croatian dictionary dedicated to Ines Viskic and Miro Kresonja."); isoToDedication.put("NL", "Dutch dictionary dedicated to Mike LeBeau."); // German handled in file. isoToDedication.put("EL", "Greek dictionary dedicated to Noah Egge."); isoToDedication.put("IT", "Italian dictionary dedicated to Carolina Tropini, my favorite stardust in the whole universe! Ti amo!"); - isoToDedication.put("JA", "Japanese dictionary dedicated to Akane Watanabe."); isoToDedication.put("KO", "Korean dictionary dedicated to Ande Elwood--fall fashion und Fernsehturms!"); isoToDedication.put("PT", "Portuguese dictionary dedicated to Carlos Melo, one Tough Mudder."); isoToDedication.put("RO", "Romanian dictionary dedicated to Radu Teodorescu."); @@ -61,19 +58,21 @@ public class DictionaryBuilderMain extends TestCase { isoToStoplist.put("FR", "fr.txt"); final Map isoToRegex = new LinkedHashMap(); - isoToRegex.put("ZH", ".*Chinese.*|.*Mandarin.*|.*Cantonese.*"); + // HACK: The missing "e" prevents a full match, causing "Cantonese" to be appended to the entries. + isoToRegex.put("ZH", "Chinese|Mandarin|Cantones"); - boolean go = false; - isoToWikiName.clear(); +// isoToWikiName.keySet().retainAll(Arrays.asList("UK", "HR", "FI")); + //isoToWikiName.clear(); + boolean go = false; for (final String foreignIso : isoToWikiName.keySet()) { - if (foreignIso.equals("GA")) { + if (foreignIso.equals("BO")) { go = true; } if (!go) { continue; } - final String dictFile = String.format(OUTPUTS + "/EN-%s_enwiktionary.quickdic", foreignIso); + final String dictFile = String.format("%s/EN-%s_enwiktionary.quickdic", OUTPUTS, foreignIso); System.out.println("building dictFile: " + dictFile); if (!isoToStoplist.containsKey(foreignIso)) { @@ -83,7 +82,7 @@ public class DictionaryBuilderMain extends TestCase { isoToDedication.put(foreignIso, ""); } if (!isoToRegex.containsKey(foreignIso)) { - isoToRegex.put(foreignIso, ".*" + isoToWikiName.get(foreignIso) + ".*"); + isoToRegex.put(foreignIso, isoToWikiName.get(foreignIso)); } DictionaryBuilder.main(new String[] { @@ -92,16 +91,16 @@ public class DictionaryBuilderMain extends TestCase { String.format("--lang2=%s", foreignIso), String.format("--lang1Stoplist=%s", STOPLISTS + isoToStoplist.get("EN")), String.format("--lang2Stoplist=%s", STOPLISTS + isoToStoplist.get(foreignIso)), - String.format("--dictInfo=(EN)Wikitionary-based EN-%s dictionary. %s", foreignIso, isoToDedication.get(foreignIso)), + String.format("--dictInfo=(EN)Wikitionary-based EN-%s dictionary.\n\n%s", foreignIso, isoToDedication.get(foreignIso)), - "--input2=" + INPUTS + "enWikiSplit/" + foreignIso + ".data", + "--input2=" + INPUTS + "eikiSplit/en/" + foreignIso + ".data", "--input2Name=enwiktionary." + foreignIso, "--input2Format=enwiktionary", "--input2LangPattern=" + isoToRegex.get(foreignIso), "--input2LangCodePattern=" + foreignIso.toLowerCase(), "--input2EnIndex=1", - "--input3=" + INPUTS + "enWikiSplit/EN.data", + "--input3=" + INPUTS + "wikiSplit/en/EN.data", "--input3Name=enwiktionary.english", "--input3Format=enwiktionary", "--input3LangPattern=" + isoToRegex.get(foreignIso), @@ -110,49 +109,37 @@ public class DictionaryBuilderMain extends TestCase { }); - // Print the entries for diffing. - final RandomAccessFile raf = new RandomAccessFile(new File(dictFile), "r"); - final Dictionary dict = new Dictionary(raf); - final PrintWriter textOut = new PrintWriter(new File(dictFile + ".text")); - final List sorted = new ArrayList(dict.pairEntries); - Collections.sort(sorted); - for (final PairEntry pairEntry : sorted) { - textOut.println(pairEntry.getRawText(false)); - } - textOut.close(); - raf.close(); - } // foreignIso + final String dictFile = String.format("%s/DE-EN_chemnitz_enwiktionary.quickdic", OUTPUTS); DictionaryBuilder.main(new String[] { - "--dictOut=" + OUTPUTS + "DE-EN_chemnitz_enwiktionary.quickdic", + "--dictOut=" + dictFile, "--lang1=DE", "--lang2=EN", - String.format("--lang1Stoplist=%s", STOPLISTS + isoToStoplist.get("DE")), - String.format("--lang2Stoplist=%s", STOPLISTS + isoToStoplist.get("EN")), + String.format("--lang1Stoplist=%s", STOPLISTS + "de.txt"), + String.format("--lang2Stoplist=%s", STOPLISTS + "en.txt"), "--dictInfo=@" + INPUTS + "de-en_chemnitz_enwiktionary.info", - "--input1=" + INPUTS + "de-en_chemnitz.txt", - "--input1Name=chemnitz", - "--input1Charset=UTF8", - "--input1Format=chemnitz", + "--input4=" + INPUTS + "de-en_chemnitz.txt", + "--input4Name=chemnitz", + "--input4Charset=UTF8", + "--input4Format=chemnitz", - "--input2=" + INPUTS + "enWikiSplit/DE.data", + "--input2=" + INPUTS + "wikiSplit/en/DE.data", "--input2Name=enwiktionary.DE", "--input2Format=enwiktionary", - "--input2LangPattern=" + isoToRegex.get("DE"), + "--input2LangPattern=German", "--input2LangCodePattern=de", "--input2EnIndex=2", - "--input3=" + INPUTS + "enWikiSplit/EN.data", + "--input3=" + INPUTS + "wikiSplit/en/EN.data", "--input3Name=enwiktionary.english", "--input3Format=enwiktionary", - "--input3LangPattern=" + isoToRegex.get("DE"), + "--input3LangPattern=German", "--input3LangCodePattern=de", "--input3EnIndex=2", - }); - + } - + }