1 // Copyright 2011 Google Inc. All Rights Reserved.
3 // Licensed under the Apache License, Version 2.0 (the "License");
4 // you may not use this file except in compliance with the License.
5 // You may obtain a copy of the License at
7 // http://www.apache.org/licenses/LICENSE-2.0
9 // Unless required by applicable law or agreed to in writing, software
10 // distributed under the License is distributed on an "AS IS" BASIS,
11 // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 // See the License for the specific language governing permissions and
13 // limitations under the License.
15 package com.hughes.android.dictionary.engine;
18 import java.io.FileNotFoundException;
19 import java.io.IOException;
20 import java.io.PrintWriter;
21 import java.io.RandomAccessFile;
22 import java.util.ArrayList;
23 import java.util.Arrays;
24 import java.util.Collections;
25 import java.util.LinkedHashMap;
26 import java.util.List;
29 import junit.framework.TestCase;
31 public class DictionaryBuilderMain extends TestCase {
33 static final String INPUTS = "data/inputs/";
34 static final String STOPLISTS = "data/inputs/stoplists/";
35 static final String OUTPUTS = "data/outputs/";
37 public static void main(final String[] args) throws Exception {
39 final Map<String,String> isoToWikiName = new LinkedHashMap<String, String>(Language.isoCodeToWikiName);
40 isoToWikiName.remove("EN");
41 isoToWikiName.remove("DE");
43 final Map<String,String> isoToDedication = new LinkedHashMap<String, String>();
44 isoToDedication.put("AF", "Afrikaans dictionary dedicated to Heiko and Mariëtte Horn.");
45 isoToDedication.put("HR", "Croation dictionary dedicated to Ines Viskic and Miro Kresonja.");
46 isoToDedication.put("NL", "Dutch dictionary dedicated to Mike LeBeau.");
47 // German handled in file.
48 isoToDedication.put("EL", "Greek dictionary dedicated to Noah Egge.");
49 isoToDedication.put("IT", "Italian dictionary dedicated to Carolina Tropini, my favorite stardust in the whole universe! Ti amo!");
50 isoToDedication.put("KO", "Korean dictionary dedicated to Ande Elwood--fall fashion und Fernsehturms!");
51 isoToDedication.put("PT", "Portuguese dictionary dedicated to Carlos Melo, one Tough Mudder.");
52 isoToDedication.put("RO", "Romanian dictionary dedicated to Radu Teodorescu.");
53 isoToDedication.put("RU", "Russian dictionary dedicated to Maxim Aronin--best friend always!.");
54 isoToDedication.put("SR", "Serbian dictionary dedicated to Filip Crnogorac--thanks for the honey.");
55 isoToDedication.put("ES", "Spanish dictionary made especially for Carolina Tropini! <3 XoXoXXXXX!");
56 isoToDedication.put("SV", "Swedish dictionary dedicated to Kajsa Palmblad--björn kramar!");
58 final Map<String,String> isoToStoplist = new LinkedHashMap<String, String>();
59 isoToStoplist.put("DE", "de.txt");
60 isoToStoplist.put("EN", "en.txt");
61 isoToStoplist.put("ES", "es.txt");
62 isoToStoplist.put("IT", "it.txt");
63 isoToStoplist.put("FR", "fr.txt");
65 final Map<String,String> isoToRegex = new LinkedHashMap<String, String>();
66 // HACK: The missing "e" prevents a full match, causing "Cantonese" to be appended to the entries.
67 isoToRegex.put("ZH", "Chinese|Mandarin|Cantones");
69 // isoToWikiName.keySet().retainAll(Arrays.asList("UK", "HR", "FI"));
70 //isoToWikiName.clear();
72 for (final String foreignIso : isoToWikiName.keySet()) {
73 if (foreignIso.equals("SK")) {
80 final String dictFile = String.format(OUTPUTS + "/EN-%s_enwiktionary.quickdic", foreignIso);
81 System.out.println("building dictFile: " + dictFile);
83 if (!isoToStoplist.containsKey(foreignIso)) {
84 isoToStoplist.put(foreignIso, "empty.txt");
86 if (!isoToDedication.containsKey(foreignIso)) {
87 isoToDedication.put(foreignIso, "");
89 if (!isoToRegex.containsKey(foreignIso)) {
90 isoToRegex.put(foreignIso, isoToWikiName.get(foreignIso));
93 DictionaryBuilder.main(new String[] {
94 String.format("--dictOut=%s", dictFile),
95 String.format("--lang1=EN"),
96 String.format("--lang2=%s", foreignIso),
97 String.format("--lang1Stoplist=%s", STOPLISTS + isoToStoplist.get("EN")),
98 String.format("--lang2Stoplist=%s", STOPLISTS + isoToStoplist.get(foreignIso)),
99 String.format("--dictInfo=(EN)Wikitionary-based EN-%s dictionary. %s", foreignIso, isoToDedication.get(foreignIso)),
101 "--input2=" + INPUTS + "enWikiSplit/" + foreignIso + ".data",
102 "--input2Name=enwiktionary." + foreignIso,
103 "--input2Format=enwiktionary",
104 "--input2LangPattern=" + isoToRegex.get(foreignIso),
105 "--input2LangCodePattern=" + foreignIso.toLowerCase(),
108 "--input3=" + INPUTS + "enWikiSplit/EN.data",
109 "--input3Name=enwiktionary.english",
110 "--input3Format=enwiktionary",
111 "--input3LangPattern=" + isoToRegex.get(foreignIso),
112 "--input3LangCodePattern=" + foreignIso.toLowerCase(),
117 // Print the entries for diffing.
118 printToText(dictFile);
122 final String dictFile = OUTPUTS + "DE-EN_chemnitz_enwiktionary.quickdic";
123 DictionaryBuilder.main(new String[] {
124 "--dictOut=" + dictFile,
127 String.format("--lang1Stoplist=%s", STOPLISTS + "de.txt"),
128 String.format("--lang2Stoplist=%s", STOPLISTS + "en.txt"),
129 "--dictInfo=@" + INPUTS + "de-en_chemnitz_enwiktionary.info",
131 "--input4=" + INPUTS + "de-en_chemnitz.txt",
132 "--input4Name=chemnitz",
133 "--input4Charset=UTF8",
134 "--input4Format=chemnitz",
136 "--input2=" + INPUTS + "enWikiSplit/DE.data",
137 "--input2Name=enwiktionary.DE",
138 "--input2Format=enwiktionary",
139 "--input2LangPattern=German",
140 "--input2LangCodePattern=de",
143 "--input3=" + INPUTS + "enWikiSplit/EN.data",
144 "--input3Name=enwiktionary.english",
145 "--input3Format=enwiktionary",
146 "--input3LangPattern=German",
147 "--input3LangCodePattern=de",
150 printToText(dictFile);
154 static void printToText(final String dictFile) throws IOException {
155 final RandomAccessFile raf = new RandomAccessFile(new File(dictFile), "r");
156 final Dictionary dict = new Dictionary(raf);
157 final PrintWriter textOut = new PrintWriter(new File(dictFile + ".text"));
158 final List<PairEntry> sorted = new ArrayList<PairEntry>(dict.pairEntries);
159 Collections.sort(sorted);
160 for (final PairEntry pairEntry : sorted) {
161 textOut.println(pairEntry.getRawText(false));