1 // Copyright 2011 Google Inc. All Rights Reserved.
3 // Licensed under the Apache License, Version 2.0 (the "License");
4 // you may not use this file except in compliance with the License.
5 // You may obtain a copy of the License at
7 // http://www.apache.org/licenses/LICENSE-2.0
9 // Unless required by applicable law or agreed to in writing, software
10 // distributed under the License is distributed on an "AS IS" BASIS,
11 // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 // See the License for the specific language governing permissions and
13 // limitations under the License.
15 package com.hughes.android.dictionary.engine;
17 import java.util.Arrays;
18 import java.util.LinkedHashMap;
19 import java.util.LinkedHashSet;
20 import java.util.List;
24 import junit.framework.TestCase;
26 import com.hughes.android.dictionary.parser.wiktionary.EnTranslationToTranslationParser;
27 import com.hughes.android.dictionary.parser.wiktionary.WiktionaryLangs;
29 public class DictionaryBuilderMain extends TestCase {
31 static final String INPUTS = "data/inputs/";
32 static final String STOPLISTS = "data/inputs/stoplists/";
33 static final String OUTPUTS = "data/outputs/";
35 public static <E> void main(final String[] args) throws Exception {
37 // Builds all the dictionaries it can, outputs list to a text file.
39 final Map<String,String> isoToWikiName = new LinkedHashMap<String, String>(WiktionaryLangs.isoCodeToWikiName);
40 isoToWikiName.remove("EN");
41 isoToWikiName.remove("DE");
43 final Map<String,String> isoToDedication = new LinkedHashMap<String, String>();
44 isoToDedication.put("AF", "Afrikaans dictionary dedicated to Heiko and Mariëtte Horn.");
45 isoToDedication.put("HR", "Croatian dictionary dedicated to Ines Viskic and Miro Kresonja.");
46 isoToDedication.put("NL", "Dutch dictionary dedicated to Mike LeBeau.");
47 // German handled in file.
48 isoToDedication.put("EL", "Greek dictionary dedicated to Noah Egge.");
49 isoToDedication.put("IT", "Italian dictionary dedicated to Carolina Tropini, my favorite stardust in the whole universe! Ti amo!");
50 isoToDedication.put("KO", "Korean dictionary dedicated to Ande Elwood--fall fashion und Fernsehturms!");
51 isoToDedication.put("PT", "Portuguese dictionary dedicated to Carlos Melo, one Tough Mudder.");
52 isoToDedication.put("RO", "Romanian dictionary dedicated to Radu Teodorescu.");
53 isoToDedication.put("RU", "Russian dictionary dedicated to Maxim Aronin--best friend always!.");
54 isoToDedication.put("SR", "Serbian dictionary dedicated to Filip Crnogorac--thanks for the honey.");
55 isoToDedication.put("ES", "Spanish dictionary made especially for Carolina Tropini! <3 XoXoXXXXX!");
56 isoToDedication.put("SV", "Swedish dictionary dedicated to Kajsa Palmblad--björn kramar!");
58 final Map<String,String> isoToStoplist = new LinkedHashMap<String, String>();
59 isoToStoplist.put("DE", "de.txt");
60 isoToStoplist.put("EN", "en.txt");
61 isoToStoplist.put("ES", "es.txt");
62 isoToStoplist.put("IT", "it.txt");
63 isoToStoplist.put("FR", "fr.txt");
65 final Map<String,String> isoToRegex = new LinkedHashMap<String, String>();
66 // HACK: The missing "e" prevents a full match, causing "Cantonese" to be appended to the entries.
67 isoToRegex.put("ZH", "Chinese|Mandarin|Cantones");
70 // Build the non EN ones.
72 final String[][] nonEnPairs = new String[][] {
82 {"AR", "TR" }, // Turkish
87 {"DE", "CA" }, // Catalan
88 {"DE", "CS" }, // Czech
89 {"DE", "EO" }, // Esperanto
93 {"DE", "HU" }, // Hungarian
96 {"DE", "LA" }, // Latin
97 {"DE", "PL" }, // Polish
99 {"DE", "SV" }, // Swedish
100 {"DE", "TR" }, // Turkish
104 {"FR", "BG" }, // Bulgarian
105 {"FR", "CS" }, // Czech
111 {"FR", "NL" }, // Dutch
113 {"FR", "TR" }, // Turkish
117 {"IT", "EL" }, // Greek
122 {"IT", "LA" }, // Latin
123 {"IT", "LV" }, // Latvian
128 {"IT", "TR" }, // Turkish
147 {"PL", "FR" }, // Polish
148 {"PL", "RU" }, // Polish
149 {"PL", "HU" }, // Polish
150 {"PL", "ES" }, // Polish
157 final Set<List<String>> done = new LinkedHashSet<List<String>>();
158 for (final String[] pair : nonEnPairs) {
160 final List<String> pairList = Arrays.asList(pair);
161 if (done.contains(pairList)) {
166 final String lang1 = pair[0];
167 final String lang2 = pair[1];
169 final String dictFile = String.format("%s/%s-%s_enwiktionary_BETA.quickdic",
170 OUTPUTS, lang1, lang2);
171 System.out.println("building dictFile: " + dictFile);
173 if (!isoToStoplist.containsKey(lang1)) {
174 isoToStoplist.put(lang1, "empty.txt");
176 if (!isoToStoplist.containsKey(lang2)) {
177 isoToStoplist.put(lang2, "empty.txt");
180 DictionaryBuilder.main(new String[] {
181 String.format("--dictOut=%s", dictFile),
182 String.format("--lang1=%s", lang1),
183 String.format("--lang2=%s", lang2),
184 String.format("--lang1Stoplist=%s", STOPLISTS + isoToStoplist.get(lang1)),
185 String.format("--lang2Stoplist=%s", STOPLISTS + isoToStoplist.get(lang2)),
186 String.format("--dictInfo=(EN)Wikitionary-based %s-%s dictionary.", lang1, lang2),
188 String.format("--input2=%swikiSplit/en/EN.data", INPUTS),
189 String.format("--input2Name=BETA!enwiktionary.%s-%s", lang1, lang2),
190 String.format("--input2Format=%s", EnTranslationToTranslationParser.NAME),
191 String.format("--input2LangPattern1=%s", lang1),
192 String.format("--input2LangPattern2=%s", lang2),
200 // Now build the EN ones.
202 // isoToWikiName.keySet().retainAll(Arrays.asList("UK", "HR", "FI"));
203 //isoToWikiName.clear();
205 for (final String foreignIso : isoToWikiName.keySet()) {
206 if (foreignIso.equals("SL")) {
213 final String dictFile = String.format("%s/EN-%s_enwiktionary.quickdic", OUTPUTS, foreignIso);
214 System.out.println("building dictFile: " + dictFile);
216 if (!isoToStoplist.containsKey(foreignIso)) {
217 isoToStoplist.put(foreignIso, "empty.txt");
219 if (!isoToDedication.containsKey(foreignIso)) {
220 isoToDedication.put(foreignIso, "");
222 if (!isoToRegex.containsKey(foreignIso)) {
223 isoToRegex.put(foreignIso, isoToWikiName.get(foreignIso));
226 DictionaryBuilder.main(new String[] {
227 String.format("--dictOut=%s", dictFile),
228 String.format("--lang1=EN"),
229 String.format("--lang2=%s", foreignIso),
230 String.format("--lang1Stoplist=%s", STOPLISTS + isoToStoplist.get("EN")),
231 String.format("--lang2Stoplist=%s", STOPLISTS + isoToStoplist.get(foreignIso)),
232 String.format("--dictInfo=(EN)Wikitionary-based EN-%s dictionary.\n\n%s", foreignIso, isoToDedication.get(foreignIso)),
234 "--input2=" + INPUTS + "wikiSplit/en/" + foreignIso + ".data",
235 "--input2Name=enwiktionary." + foreignIso,
236 "--input2Format=enwiktionary",
237 "--input2WiktionaryType=EnForeign",
238 "--input2LangPattern=" + isoToRegex.get(foreignIso),
239 "--input2LangCodePattern=" + foreignIso.toLowerCase(),
242 "--input3=" + INPUTS + "wikiSplit/en/EN.data",
243 "--input3Name=enwiktionary.english",
244 "--input3Format=enwiktionary",
245 "--input3WiktionaryType=EnToTranslation",
246 "--input3LangPattern=" + isoToRegex.get(foreignIso),
247 "--input3LangCodePattern=" + foreignIso.toLowerCase(),
254 // Now special case German-English.
256 final String dictFile = String.format("%s/DE-EN_chemnitz_enwiktionary.quickdic", OUTPUTS);
257 DictionaryBuilder.main(new String[] {
258 "--dictOut=" + dictFile,
261 String.format("--lang1Stoplist=%s", STOPLISTS + "de.txt"),
262 String.format("--lang2Stoplist=%s", STOPLISTS + "en.txt"),
263 "--dictInfo=@" + INPUTS + "de-en_chemnitz_enwiktionary.info",
265 "--input4=" + INPUTS + "de-en_chemnitz.txt",
266 "--input4Name=chemnitz",
267 "--input4Charset=UTF8",
268 "--input4Format=chemnitz",
270 "--input2=" + INPUTS + "wikiSplit/en/DE.data",
271 "--input2Name=enwiktionary.DE",
272 "--input2Format=enwiktionary",
273 "--input2WiktionaryType=EnForeign",
274 "--input2LangPattern=German",
275 "--input2LangCodePattern=de",
278 "--input3=" + INPUTS + "wikiSplit/en/EN.data",
279 "--input3Name=enwiktionary.english",
280 "--input3Format=enwiktionary",
281 "--input3WiktionaryType=EnToTranslation",
282 "--input3LangPattern=German",
283 "--input3LangCodePattern=de",