1 // Copyright 2011 Google Inc. All Rights Reserved.
3 // Licensed under the Apache License, Version 2.0 (the "License");
4 // you may not use this file except in compliance with the License.
5 // You may obtain a copy of the License at
7 // http://www.apache.org/licenses/LICENSE-2.0
9 // Unless required by applicable law or agreed to in writing, software
10 // distributed under the License is distributed on an "AS IS" BASIS,
11 // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 // See the License for the specific language governing permissions and
13 // limitations under the License.
15 package com.hughes.android.dictionary.engine;
18 import java.io.FileNotFoundException;
19 import java.io.IOException;
20 import java.io.PrintStream;
21 import java.io.RandomAccessFile;
23 import com.hughes.android.dictionary.parser.wiktionary.EnTranslationToTranslationParser;
24 import com.hughes.android.dictionary.parser.wiktionary.WholeSectionToHtmlParser;
25 import com.hughes.util.FileUtil;
27 import junit.framework.TestCase;
29 public class DictionaryBuilderTest extends TestCase {
31 public static final String TEST_INPUTS = "testdata/inputs/";
32 public static final String WIKISPLIT_EN = "data/inputs/wikiSplit/en/";
33 public static final String STOPLISTS = "data/inputs/stoplists/";
34 public static final String GOLDENS = "testdata/goldens/";
36 public static final String TEST_OUTPUTS = "testdata/outputs/";
38 public void doTestCustomDict(final String name, final String lang1,
39 final String lang2, final String inputFile) throws Exception {
40 final File result = new File(TEST_OUTPUTS + name);
41 System.out.println("Writing to: " + result);
42 DictionaryBuilder.main(new String[] {
43 "--dictOut=" + result.getAbsolutePath(),
46 "--lang1Stoplist=" + STOPLISTS + "empty.txt",
47 "--lang2Stoplist=" + STOPLISTS + "empty.txt",
50 "--input1=testdata/inputs/" + inputFile,
51 "--input1Name=my_input_" + name,
52 "--input1Charset=ISO-8859-1",
53 "--input1Format=tab_separated",
55 "--print=" + result.getPath() + ".text",
58 checkGolden(name, result);
61 public void test_FR_NL() throws Exception {
62 doTestCustomDict("QuickDic-FR-NL.quickdic", "FR", "NL", "QuickDic-FR-NL.txt");
65 public void testWiktionary_en_de2fr() throws Exception {
66 wiktionaryTestWithEnTrans2Trans("wiktionary.de_fr.quickdic", "DE", "FR");
69 public void wiktionaryTestWithEnTrans2Trans(final String name, final String lang1,
70 final String lang2) throws Exception {
71 final File result = new File(TEST_OUTPUTS + name);
72 System.out.println("Writing to: " + result);
73 DictionaryBuilder.main(new String[] {
74 "--dictOut=" + result.getAbsolutePath(),
77 "--lang1Stoplist=" + STOPLISTS + "empty.txt",
78 "--lang2Stoplist=" + STOPLISTS + "empty.txt",
79 "--dictInfo=SomeWikiDataTrans2Trans",
81 "--input4=" + WIKISPLIT_EN + "EN.data",
82 "--input4Name=" + name,
83 "--input4Format=" + EnTranslationToTranslationParser.NAME,
84 "--input4LangPattern1=" + lang1,
85 "--input4LangPattern2=" + lang2,
86 "--input4PageLimit=1000",
88 "--print=" + result.getPath() + ".text",
91 checkGolden(name, result);
94 public void testWiktionary_WholeSection_DE() throws Exception {
95 wiktionaryTestWithWholeSectionToHtml("wiktionary.WholeSection.DE.quickdic", "DE", 100);
98 public void testWiktionary_WholeSection_EN() throws Exception {
99 wiktionaryTestWithWholeSectionToHtml("wiktionary.WholeSection.EN.quickdic", "EN", 100);
102 public void testWiktionary_WholeSection_IT() throws Exception {
103 // Have to run to 800 to get a few verb conjugations (including essere!)
104 wiktionaryTestWithWholeSectionToHtml("wiktionary.WholeSection.IT.quickdic", "IT", 800);
107 public void wiktionaryTestWithWholeSectionToHtml(final String name, final String langCode, final int pageLimit) throws Exception {
108 final File result = new File(TEST_OUTPUTS + name);
109 System.out.println("Writing to: " + result);
110 DictionaryBuilder.main(new String[] {
111 "--dictOut=" + result.getAbsolutePath(),
112 "--lang1=" + langCode,
114 "--lang1Stoplist=" + STOPLISTS + "empty.txt",
115 "--lang2Stoplist=" + STOPLISTS + "empty.txt",
116 "--dictInfo=SomeWikiDataWholeSection",
118 "--input4=" + WIKISPLIT_EN + langCode + ".data",
119 "--input4Name=" + name,
120 "--input4Format=" + WholeSectionToHtmlParser.NAME,
121 "--input4WiktionaryLang=EN",
122 "--input4SkipLang=" + langCode,
123 "--input4TitleIndex=" + "1",
124 "--input4PageLimit=" + pageLimit,
126 "--print=" + result.getPath() + ".text",
128 checkGolden(name, result);
132 public void testWiktionary_IT_EN() throws Exception {
133 wiktionaryTestWithLangToEn("wiktionary.it_en.quickdic", "IT", "it.txt",
134 "EN.data", "enwiktionary.english", "Italian", "it");
137 public void testWiktionary_ZH_EN() throws Exception {
138 wiktionaryTestWithLangToEn("wiktionary.zh_en.quickdic", "ZH", "empty.txt",
139 // These missing "e" prevents a complete match, forcing the name to be printed
140 "EN.data", "enwiktionary.english", "Chinese|Mandarin|Cantones", "zh");
143 public void testWiktionary_DE_EN() throws Exception {
144 wiktionaryTestWithLangToEn("wiktionary.de_en.quickdic", "DE", "de.txt",
145 "EN.data", "enwiktionary.english", "German", "it");
148 public void testWiktionary_IT_IT() throws Exception {
149 wiktionaryTestWithLangToEn("wiktionary.it_it.quickdic", "IT", "it.txt",
150 "IT.data", "enwiktionary.italian", "Italian", "it");
154 public void testWiktionary_FR_FR() throws Exception {
155 wiktionaryTestWithLangToEn("wiktionary.fr_fr.quickdic", "FR", "fr.txt",
156 "FR.data", "enwiktionary.french", "French", "fr");
161 public void testWiktionary_AR_AR() throws Exception {
162 wiktionaryTestWithLangToEn("wiktionary.ar_ar.quickdic", "AR", "empty.txt",
163 "AR.data", "enwiktionary.arabic", "Arabic", "ar");
167 public void testWiktionary_ZH_ZH() throws Exception {
168 wiktionaryTestWithLangToEn("wiktionary.zh_zh.quickdic", "ZH", "empty.txt",
169 // These missing "e" prevents a complete match, forcing the name to be printed.
170 "ZH.data", "enwiktionary.chinese", "Chinese|Mandarin|Cantones", "zh");
174 public void testWiktionary_DE_DE() throws Exception {
175 wiktionaryTestWithLangToEn("wiktionary.de_de.quickdic", "DE", "de.txt",
176 "DE.data", "enwiktionary.german", "German", "it");
180 public void testWiktionary_TH_TH() throws Exception {
181 wiktionaryTestWithLangToEn("wiktionary.th_th.quickdic", "TH", "empty.txt",
182 // These missing "e" prevents a complete match, forcing the name to be printed.
183 "TH.data", "enwiktionary.thai", "Thai", "th");
186 public void wiktionaryTestWithLangToEn(final String name, final String lang1,
187 final String stoplist, final String data, final String dictName,
188 final String langPattern, final String langCode) throws Exception {
189 final File result = new File(TEST_OUTPUTS + name);
190 System.out.println("Writing to: " + result);
191 final String type = data.equals("EN.data") ? "EnToTranslation" : "EnForeign";
192 DictionaryBuilder.main(new String[] {
193 "--dictOut=" + result.getAbsolutePath(),
196 "--lang1Stoplist=" + STOPLISTS + stoplist,
197 "--lang2Stoplist=" + STOPLISTS + "en.txt",
198 "--dictInfo=SomeWikiData",
200 "--input4=" + WIKISPLIT_EN + data,
201 "--input4Name=" + dictName,
202 "--input4Format=enwiktionary",
203 "--input4WiktionaryType=" + type,
204 "--input4LangPattern=" + langPattern,
205 "--input4LangCodePattern=" + langCode,
207 "--input4PageLimit=1000",
209 "--print=" + result.getPath() + ".text",
212 checkGolden(name, result);
215 public void testGermanCombined() throws Exception {
216 final String name = "de-en.quickdic";
217 final File result = new File(TEST_OUTPUTS + name);
218 System.out.println("Writing to: " + result);
219 DictionaryBuilder.main(new String[] {
220 "--dictOut=" + result.getAbsolutePath(),
223 "--dictInfo=@" + TEST_INPUTS + "de-en_dictInfo.txt",
225 "--input1=" + TEST_INPUTS + "de-en_chemnitz_100",
226 "--input1Name=chemnitz",
227 "--input1Charset=UTF8",
228 "--input1Format=chemnitz",
230 "--input2=" + TEST_INPUTS + "de-en_dictcc_simulated",
231 "--input2Name=dictcc",
232 "--input2Charset=UTF8",
233 "--input2Format=tab_separated",
235 "--print=" + result.getPath() + ".text",
238 checkGolden(name, result);
241 private void checkGolden(final String dictName, final File dictFile)
242 throws IOException, FileNotFoundException {
244 assertFilesEqual(GOLDENS + dictName + ".text", dictFile.getPath() + ".text");
247 final Dictionary dict = new Dictionary(new RandomAccessFile(dictFile.getAbsolutePath(), "r"));
248 final PrintStream out = new PrintStream(new File(dictFile.getPath() + ".text"));
251 assertFilesEqual(GOLDENS + dictName + ".text", dictFile.getPath() + ".text");
255 void assertFilesEqual(final String expected, final String actual) throws IOException {
256 final String expectedString = FileUtil.readToString(new File(expected));
257 final String actualString = FileUtil.readToString(new File(actual));
258 assertEquals(expectedString, actualString);