1 // Copyright 2011 Google Inc. All Rights Reserved.
3 // Licensed under the Apache License, Version 2.0 (the "License");
4 // you may not use this file except in compliance with the License.
5 // You may obtain a copy of the License at
7 // http://www.apache.org/licenses/LICENSE-2.0
9 // Unless required by applicable law or agreed to in writing, software
10 // distributed under the License is distributed on an "AS IS" BASIS,
11 // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 // See the License for the specific language governing permissions and
13 // limitations under the License.
15 package com.hughes.android.dictionary.engine;
18 import java.io.FileNotFoundException;
19 import java.io.IOException;
20 import java.io.PrintStream;
21 import java.io.RandomAccessFile;
22 import java.util.Collections;
24 import com.hughes.android.dictionary.parser.wiktionary.EnTranslationToTranslationParser;
25 import com.hughes.android.dictionary.parser.wiktionary.WholeSectionToHtmlParser;
26 import com.hughes.util.FileUtil;
28 import junit.framework.TestCase;
30 public class DictionaryBuilderTest extends TestCase {
32 public static final String TEST_INPUTS = "testdata/inputs/";
33 public static final String WIKISPLIT = "data/inputs/wikiSplit/";
34 public static final String WIKISPLIT_EN = "data/inputs/wikiSplit/en/";
35 public static final String STOPLISTS = "data/inputs/stoplists/";
36 public static final String GOLDENS = "testdata/goldens/";
38 public static final String TEST_OUTPUTS = "testdata/outputs/";
40 public void testItConj() throws Exception {
41 final String toParse = "{{it-conj-are|d|avere|pres2s=dai|pres3s=dà|pres3p=danno|prem1s=diedi|prem1s2=detti|prem2s=desti|prem3s=diede|prem3s2=dette|prem1p=demmo|prem2p=deste|prem3p=diedero|prem3p2=dettero|fut1s=darò|fut2s=darai|fut3s=darà|fut1p=daremo|fut2p=darete|fut3p=daranno|cond1s=darei|cond2s=daresti|cond3s=darebbe|cond1p=daremmo|cond2p=dareste|cond3p=darebbero|sub123s=dia|sub3p=diano|impsub12s=dessi|impsub3s=desse|impsub1p=dessimo|impsub2p=deste|impsub3p=dessero|imp2s=dà|imp2s2=dai|imp2s3=da'|imp3s=dia|imp3p=diano}}\n" +
42 "{{it-conj-are|accus|avere}}\n" +
43 "{{it-conj-care|pag|avere or essere}}\n" +
44 "{{it-conj-iare|studi|avere}}\n" +
45 "{{it-conj-iare-b|avvi|avere}}\n" +
46 "{{it-conj-ciare|pronunc|avere}}\n" +
47 "{{it-conj-ere|sed|essere|pres1s=siedo|pres1s2=seggo|pres2s=siedi|pres3s=siede|pres3p=siedono|pres3p2=seggono|fut1s2=siederò|fut2s2=siederai|fut3s2=siederà|fut1p2=siederemo|fut2p2=siederete|fut3p2=siederanno|cond1s2=siederei|cond2s2=siederesti|cond3s2=siederebbe|cond1p2=siederemmo|cond2p2=siedereste|cond3p2=siederebbero|sub123s=sieda|sub3p=siedano|imp2s=siedi|imp3s=sieda|imp3s2=segga|imp3p=siedano|imp3p2=seggano}}\n" +
48 "{{it-conj-ere|persuad|avere|pastp=persuaso|prem1s=persuasi|prem3s=persuase|prem3s2=''|prem3p=persuasero|prem3p2=''}}\n" +
49 "{{it-conj-ere|abbatt|avere}}\n" +
50 "{{it-conj-ire|copr|avere|pastp=coperto|prem1s2=copersi|prem3s2=coperse|prem3p2=copersero}}\n" +
51 "{{it-conj-ire-b|prefer|avere}}\n" +
52 "{{it-conj-urre|prod|avere}}\n" +
53 "{{it-conj-arsi|lav}}\n" +
54 "{{it-conj-ersi|abbatt}}\n" +
55 "{{it-conj-iarsi|annoi}}\n" +
56 "{{it-conj-carsi|coniug}}\n" +
57 "{{it-conj-ciarsi|affacc}}\n" +
58 "{{it-conj-irsi|vest}}\n" +
59 "{{it-conj-irsi-b|fer}}\n" +
60 "{{it-conj-ursi|rid|essere}}\n" +
61 "{{it-conj-cire|ricuc|avere}}\n" +
62 "{{it-conj-iarsi-b|riavvi|essere}}" +
63 "{{it-conj-fare|putre|avere}}\n" +
64 "{{it-conj-cirsi|cuc|essere}}\n" +
65 "{{it-conj-ere|smett|avere|pastp=smesso|prem1s=smisi|prem3s=smise|prem3s2=''|prem3p=smisero|prem3p2=''}}\n" +
66 "{{term||[[cor#Latin|Cor]] [[Carolus#Latin|Carolī]]|Charles' heart}}\n" +
67 "{{term|sc=Grek|λόγος|tr=lógos||word}}\n" +
68 "{{term|verbo|verbō|for the word}}\n"
70 final DictionaryBuilder db = new DictionaryBuilder("", Language.en, Language.it, "", "", Collections.singleton("X"), Collections.singleton("X"));
71 WholeSectionToHtmlParser parser = new WholeSectionToHtmlParser(db.indexBuilders.get(0), null, "EN", "IT", "http://en.wiktionary.org/wiki/%s");
72 parser.title = "dummyTitle";
73 parser.entrySource = new EntrySource(0, "dummySource", 0);
74 parser.parseSection("dummyHeading", toParse);
77 final String dictName = "testItConj.html";
78 final PrintStream out = new PrintStream(new File(TEST_OUTPUTS, dictName));
79 db.dictionary.print(out);
82 assertFilesEqual(GOLDENS + dictName, TEST_OUTPUTS + dictName);
85 public void doTestCustomDict(final String name, final String lang1,
86 final String lang2, final String inputFile) throws Exception {
87 final File result = new File(TEST_OUTPUTS + name);
88 System.out.println("Writing to: " + result);
89 DictionaryBuilder.main(new String[] {
90 "--dictOut=" + result.getAbsolutePath(),
93 "--lang1Stoplist=" + STOPLISTS + "empty.txt",
94 "--lang2Stoplist=" + STOPLISTS + "empty.txt",
97 "--input1=testdata/inputs/" + inputFile,
98 "--input1Name=my_input_" + name,
99 "--input1Charset=ISO-8859-1",
100 "--input1Format=tab_separated",
102 "--print=" + result.getPath() + ".text",
105 checkGolden(name, result);
108 public void test_FR_NL() throws Exception {
109 doTestCustomDict("QuickDic-FR-NL.quickdic", "FR", "NL", "QuickDic-FR-NL.txt");
112 public void testWiktionary_en_de2fr() throws Exception {
113 wiktionaryTestWithEnTrans2Trans("wiktionary.de_fr.quickdic", "DE", "FR");
116 public void wiktionaryTestWithEnTrans2Trans(final String name, final String lang1,
117 final String lang2) throws Exception {
118 final File result = new File(TEST_OUTPUTS + name);
119 System.out.println("Writing to: " + result);
120 DictionaryBuilder.main(new String[] {
121 "--dictOut=" + result.getAbsolutePath(),
124 "--lang1Stoplist=" + STOPLISTS + "empty.txt",
125 "--lang2Stoplist=" + STOPLISTS + "empty.txt",
126 "--dictInfo=SomeWikiDataTrans2Trans",
128 "--input4=" + WIKISPLIT_EN + "EN.data",
129 "--input4Name=" + name,
130 "--input4Format=" + EnTranslationToTranslationParser.NAME,
131 "--input4LangPattern1=" + lang1,
132 "--input4LangPattern2=" + lang2,
133 "--input4PageLimit=1000",
135 "--print=" + result.getPath() + ".text",
138 checkGolden(name, result);
141 public void testWiktionary_WholeSection_DE() throws Exception {
142 enWiktionaryTestWithWholeSectionToHtml("enwiktionary.WholeSection.DE.quickdic", "DE", 100);
145 public void testWiktionary_WholeSection_EN() throws Exception {
146 enWiktionaryTestWithWholeSectionToHtml("enwiktionary.WholeSection.EN.quickdic", "EN", 100);
149 public void testWiktionary_WholeSection_IT() throws Exception {
150 // Have to run to 800 to get a few verb conjugations (including essere!)
151 enWiktionaryTestWithWholeSectionToHtml("enwiktionary.WholeSection.IT.quickdic", "IT", 800);
154 public void enWiktionaryTestWithWholeSectionToHtml(final String name, final String langCode, final int pageLimit) throws Exception {
155 final File result = new File(TEST_OUTPUTS + name);
156 System.out.println("Writing to: " + result);
157 DictionaryBuilder.main(new String[] {
158 "--dictOut=" + result.getAbsolutePath(),
159 "--lang1=" + langCode,
161 "--lang1Stoplist=" + STOPLISTS + "empty.txt",
162 "--lang2Stoplist=" + STOPLISTS + "empty.txt",
163 "--dictInfo=SomeWikiDataWholeSection",
165 "--input4=" + WIKISPLIT_EN + langCode + ".data",
166 "--input4Name=" + name,
167 "--input4Format=" + WholeSectionToHtmlParser.NAME,
168 "--input4WiktionaryLang=EN",
169 "--input4SkipLang=" + langCode,
170 "--input4TitleIndex=" + "1",
171 "--input4PageLimit=" + pageLimit,
173 "--print=" + result.getPath() + ".text",
175 checkGolden(name, result);
178 //-----------------------------------------------------------------
180 public void testSingleLang_EN() throws Exception {
181 wiktionaryTestSingleLang("SingleLang_EN.quickdic", "EN", 100);
184 public void testSingleLang_DE() throws Exception {
185 wiktionaryTestSingleLang("SingleLang_DE.quickdic", "DE", 100);
188 public void testSingleLang_IT() throws Exception {
189 wiktionaryTestSingleLang("SingleLang_IT.quickdic", "IT", 100);
192 public void testSingleLang_FR() throws Exception {
193 wiktionaryTestSingleLang("SingleLang_FR.quickdic", "FR", 100);
196 public void wiktionaryTestSingleLang(final String name, final String langCode, final int pageLimit) throws Exception {
197 final File result = new File(TEST_OUTPUTS + name);
198 System.out.println("Writing to: " + result);
199 DictionaryBuilder.main(new String[] {
200 "--dictOut=" + result.getAbsolutePath(),
201 "--lang1=" + langCode,
202 "--lang1Stoplist=" + STOPLISTS + "empty.txt",
203 "--dictInfo=SomeWikiDataWholeSection",
204 "--input4=" + WIKISPLIT + langCode.toLowerCase() + "/" + langCode + ".data",
205 "--input4Name=" + name,
206 "--input4Format=" + WholeSectionToHtmlParser.NAME,
207 "--input4WiktionaryLang=" + langCode,
208 "--input4SkipLang=" + langCode,
209 "--input4TitleIndex=" + "1",
210 "--input4PageLimit=" + pageLimit,
211 "--print=" + result.getPath() + ".text",
213 checkGolden(name, result);
216 //-----------------------------------------------------------------
218 public void testWiktionary_IT_EN() throws Exception {
219 wiktionaryTestWithLangToEn("wiktionary.it_en.quickdic", "IT", "it.txt",
220 "EN.data", "enwiktionary.english", "Italian", "it", 1000);
223 public void testWiktionary_cmn_EN() throws Exception {
224 wiktionaryTestWithLangToEn("wiktionary.cmn_en.quickdic", "cmn", "empty.txt",
225 // These missing "e" prevents a complete match, forcing the name to be printed
226 "EN.data", "enwiktionary.english", "Chinese|Mandarin", "cmn", 1000);
229 public void testWiktionary_DE_EN() throws Exception {
230 wiktionaryTestWithLangToEn("wiktionary.de_en.quickdic", "DE", "de.txt",
231 "EN.data", "enwiktionary.english", "German", "de", 1000);
234 public void testWiktionary_IT_IT() throws Exception {
235 wiktionaryTestWithLangToEn("wiktionary.it_it.quickdic", "IT", "it.txt",
236 "IT.data", "enwiktionary.italian", "Italian", "it", 1000);
240 public void testWiktionary_FR_FR() throws Exception {
241 wiktionaryTestWithLangToEn("wiktionary.fr_fr.quickdic", "FR", "fr.txt",
242 "FR.data", "enwiktionary.french", "French", "fr", 1000);
247 public void testWiktionary_AR_AR() throws Exception {
248 // Arabic is really big for some reason, use fewer pages.
249 wiktionaryTestWithLangToEn("wiktionary.ar_ar.quickdic", "AR", "empty.txt",
250 "AR.data", "enwiktionary.arabic", "Arabic", "ar", 200);
254 public void testWiktionary_cmn_cmn() throws Exception {
255 wiktionaryTestWithLangToEn("wiktionary.cmn_cmn.quickdic", "cmn", "empty.txt",
256 // These missing "e" prevents a complete match, forcing the name to be printed.
257 "cmn.data", "enwiktionary.chinese", "Chinese|Mandarin", "cmn", 1000);
261 public void testWiktionary_DE_DE() throws Exception {
262 wiktionaryTestWithLangToEn("wiktionary.de_de.quickdic", "DE", "de.txt",
263 "DE.data", "enwiktionary.german", "German", "de", 1000);
267 public void testWiktionary_TH_TH() throws Exception {
268 wiktionaryTestWithLangToEn("wiktionary.th_th.quickdic", "TH", "empty.txt",
269 // These missing "e" prevents a complete match, forcing the name to be printed.
270 "TH.data", "enwiktionary.thai", "Thai", "th", 1000);
273 public void wiktionaryTestWithLangToEn(final String name, final String lang1,
274 final String stoplist, final String data, final String dictName,
275 final String langPattern, final String langCode, int pageLimit) throws Exception {
276 final File result = new File(TEST_OUTPUTS + name);
277 System.out.println("Writing to: " + result);
278 final String type = data.equals("EN.data") ? "EnToTranslation" : "EnForeign";
279 DictionaryBuilder.main(new String[] {
280 "--dictOut=" + result.getAbsolutePath(),
283 "--lang1Stoplist=" + STOPLISTS + stoplist,
284 "--lang2Stoplist=" + STOPLISTS + "en.txt",
285 "--dictInfo=SomeWikiData",
287 "--input4=" + WIKISPLIT_EN + data,
288 "--input4Name=" + dictName,
289 "--input4Format=enwiktionary",
290 "--input4WiktionaryType=" + type,
291 "--input4LangPattern=" + langPattern,
292 "--input4LangCodePattern=" + langCode,
294 "--input4PageLimit=" + pageLimit,
296 "--print=" + result.getPath() + ".text",
299 checkGolden(name, result);
302 public void testGermanCombined() throws Exception {
303 final String name = "de-en.quickdic";
304 final File result = new File(TEST_OUTPUTS + name);
305 System.out.println("Writing to: " + result);
306 DictionaryBuilder.main(new String[] {
307 "--dictOut=" + result.getAbsolutePath(),
310 "--dictInfo=@" + TEST_INPUTS + "de-en_dictInfo.txt",
312 "--input1=" + TEST_INPUTS + "de-en_chemnitz_100",
313 "--input1Name=chemnitz",
314 "--input1Charset=UTF8",
315 "--input1Format=chemnitz",
317 "--input2=" + TEST_INPUTS + "de-en_dictcc_simulated",
318 "--input2Name=dictcc",
319 "--input2Charset=UTF8",
320 "--input2Format=tab_separated",
322 "--print=" + result.getPath() + ".text",
325 checkGolden(name, result);
328 public void testItalianTurkish() throws Exception {
329 final String name = "it-tr_dictcc.quickdic";
330 final File result = new File(TEST_OUTPUTS + name);
331 System.out.println("Writing to: " + result);
332 DictionaryBuilder.main(new String[] {
333 "--dictOut=" + result.getAbsolutePath(),
336 "--dictInfo=it-tr_dictcc_simulated",
338 "--input1=" + TEST_INPUTS + "it-tr_dictcc_simulated.txt",
339 "--input1Name=dictcc",
340 "--input1Charset=UTF8",
341 "--input1Format=tab_separated",
343 "--print=" + result.getPath() + ".text",
346 checkGolden(name, result);
349 private void checkGolden(final String dictName, final File dictFile)
350 throws IOException, FileNotFoundException {
352 assertFilesEqual(GOLDENS + dictName + ".text", dictFile.getPath() + ".text");
355 final Dictionary dict = new Dictionary(new RandomAccessFile(dictFile.getAbsolutePath(), "r"));
356 final PrintStream out = new PrintStream(new File(dictFile.getPath() + ".text"));
359 assertFilesEqual(GOLDENS + dictName + ".text", dictFile.getPath() + ".text");
363 void assertFilesEqual(final String expected, final String actual) throws IOException {
364 final String expectedString = FileUtil.readToString(new File(expected));
365 final String actualString = FileUtil.readToString(new File(actual));
366 assertEquals(expectedString, actualString);