src/com/hughes/android/dictionary/engine/DictionaryBuilderTest.java

   1 // Copyright 2011 Google Inc. All Rights Reserved.
   2 //
   3 // Licensed under the Apache License, Version 2.0 (the "License");
   4 // you may not use this file except in compliance with the License.
   5 // You may obtain a copy of the License at
   6 //
   7 //     http://www.apache.org/licenses/LICENSE-2.0
   8 //
   9 // Unless required by applicable law or agreed to in writing, software
  10 // distributed under the License is distributed on an "AS IS" BASIS,
  11 // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  12 // See the License for the specific language governing permissions and
  13 // limitations under the License.
  14
  15 package com.hughes.android.dictionary.engine;
  16
  17 import java.io.File;
  18 import java.io.FileNotFoundException;
  19 import java.io.IOException;
  20 import java.io.PrintStream;
  21 import java.io.RandomAccessFile;
  22 import java.util.Collections;
  23
  24 import com.hughes.android.dictionary.parser.wiktionary.EnTranslationToTranslationParser;
  25 import com.hughes.android.dictionary.parser.wiktionary.WholeSectionToHtmlParser;
  26 import com.hughes.util.FileUtil;
  27
  28 import junit.framework.TestCase;
  29
  30 public class DictionaryBuilderTest extends TestCase {
  31
  32   public static final String TEST_INPUTS = "testdata/inputs/";
  33   public static final String WIKISPLIT = "data/inputs/wikiSplit/";
  34   public static final String WIKISPLIT_EN = "data/inputs/wikiSplit/en/";
  35   public static final String STOPLISTS = "data/inputs/stoplists/";
  36   public static final String GOLDENS = "testdata/goldens/";
  37
  38   public static final String TEST_OUTPUTS = "testdata/outputs/";
  39
  40   public void testItConj() throws Exception {
  41       final String toParse = "{{it-conj-are|d|avere|pres2s=dai|pres3s=dà|pres3p=danno|prem1s=diedi|prem1s2=detti|prem2s=desti|prem3s=diede|prem3s2=dette|prem1p=demmo|prem2p=deste|prem3p=diedero|prem3p2=dettero|fut1s=darò|fut2s=darai|fut3s=darà|fut1p=daremo|fut2p=darete|fut3p=daranno|cond1s=darei|cond2s=daresti|cond3s=darebbe|cond1p=daremmo|cond2p=dareste|cond3p=darebbero|sub123s=dia|sub3p=diano|impsub12s=dessi|impsub3s=desse|impsub1p=dessimo|impsub2p=deste|impsub3p=dessero|imp2s=dà|imp2s2=dai|imp2s3=da'|imp3s=dia|imp3p=diano}}\n" +
  42               "{{it-conj-are|accus|avere}}\n" +
  43               "{{it-conj-care|pag|avere or essere}}\n" +
  44               "{{it-conj-iare|studi|avere}}\n" +
  45               "{{it-conj-iare-b|avvi|avere}}\n" +
  46               "{{it-conj-ciare|pronunc|avere}}\n" +
  47               "{{it-conj-ere|sed|essere|pres1s=siedo|pres1s2=seggo|pres2s=siedi|pres3s=siede|pres3p=siedono|pres3p2=seggono|fut1s2=siederò|fut2s2=siederai|fut3s2=siederà|fut1p2=siederemo|fut2p2=siederete|fut3p2=siederanno|cond1s2=siederei|cond2s2=siederesti|cond3s2=siederebbe|cond1p2=siederemmo|cond2p2=siedereste|cond3p2=siederebbero|sub123s=sieda|sub3p=siedano|imp2s=siedi|imp3s=sieda|imp3s2=segga|imp3p=siedano|imp3p2=seggano}}\n" +
  48               "{{it-conj-ere|persuad|avere|pastp=persuaso|prem1s=persuasi|prem3s=persuase|prem3s2=''|prem3p=persuasero|prem3p2=''}}\n" +
  49               "{{it-conj-ere|abbatt|avere}}\n" +
  50               "{{it-conj-ire|copr|avere|pastp=coperto|prem1s2=copersi|prem3s2=coperse|prem3p2=copersero}}\n" +
  51               "{{it-conj-ire-b|prefer|avere}}\n" +
  52               "{{it-conj-urre|prod|avere}}\n" +
  53               "{{it-conj-arsi|lav}}\n" +
  54               "{{it-conj-ersi|abbatt}}\n" +
  55               "{{it-conj-iarsi|annoi}}\n" +
  56               "{{it-conj-carsi|coniug}}\n" +
  57               "{{it-conj-ciarsi|affacc}}\n" +
  58               "{{it-conj-irsi|vest}}\n" +
  59               "{{it-conj-irsi-b|fer}}\n" +
  60               "{{it-conj-ursi|rid|essere}}\n" +
  61               "{{it-conj-cire|ricuc|avere}}\n" +
  62               "{{it-conj-iarsi-b|riavvi|essere}}" +
  63               "{{it-conj-fare|putre|avere}}\n" +
  64               "{{it-conj-cirsi|cuc|essere}}\n" +
  65               "{{it-conj-ere|smett|avere|pastp=smesso|prem1s=smisi|prem3s=smise|prem3s2=''|prem3p=smisero|prem3p2=''}}\n" +
  66               "{{term||[[cor#Latin|Cor]] [[Carolus#Latin|Carolī]]|Charles' heart}}\n" +
  67               "{{term|sc=Grek|λόγος|tr=lógos||word}}\n" +
  68               "{{term|verbo|verbō|for the word}}\n"
  69               ;
  70       final DictionaryBuilder db = new DictionaryBuilder("", Language.en, Language.it,  "", "", Collections.singleton("X"), Collections.singleton("X"));
  71       WholeSectionToHtmlParser parser = new WholeSectionToHtmlParser(db.indexBuilders.get(0), null, "EN", "IT", "http://en.wiktionary.org/wiki/%s");
  72       parser.title = "dummyTitle";
  73       parser.entrySource = new EntrySource(0, "dummySource", 0);
  74       parser.parseSection("dummyHeading", toParse);
  75       db.build();
  76
  77       final String dictName = "testItConj.html";
  78       final PrintStream out = new PrintStream(new File(TEST_OUTPUTS, dictName));
  79       db.dictionary.print(out);
  80       out.close();
  81
  82       assertFilesEqual(GOLDENS + dictName, TEST_OUTPUTS + dictName);
  83   }
  84
  85   public void doTestCustomDict(final String name, final String lang1,
  86       final String lang2, final String inputFile) throws Exception {
  87     final File result = new File(TEST_OUTPUTS + name);
  88     System.out.println("Writing to: " + result);
  89     DictionaryBuilder.main(new String[] {
  90         "--dictOut=" + result.getAbsolutePath(),
  91         "--lang1=" + lang1,
  92         "--lang2=" + lang2,
  93         "--lang1Stoplist=" + STOPLISTS + "empty.txt",
  94         "--lang2Stoplist=" + STOPLISTS + "empty.txt",
  95         "--dictInfo=bleh.",
  96
  97         "--input1=testdata/inputs/" + inputFile,
  98         "--input1Name=my_input_" + name,
  99         "--input1Charset=ISO-8859-1",
 100         "--input1Format=tab_separated",
 101
 102         "--print=" + result.getPath() + ".text",
 103     });
 104
 105     checkGolden(name, result);
 106   }
 107
 108   public void test_FR_NL() throws Exception {
 109     doTestCustomDict("QuickDic-FR-NL.quickdic", "FR", "NL", "QuickDic-FR-NL.txt");
 110   }
 111
 112   public void testWiktionary_en_de2fr() throws Exception {
 113     wiktionaryTestWithEnTrans2Trans("wiktionary.de_fr.quickdic", "DE", "FR");
 114   }
 115
 116   public void wiktionaryTestWithEnTrans2Trans(final String name, final String lang1,
 117       final String lang2) throws Exception {
 118     final File result = new File(TEST_OUTPUTS + name);
 119     System.out.println("Writing to: " + result);
 120     DictionaryBuilder.main(new String[] {
 121         "--dictOut=" + result.getAbsolutePath(),
 122         "--lang1=" + lang1,
 123         "--lang2=" + lang2,
 124         "--lang1Stoplist=" + STOPLISTS + "empty.txt",
 125         "--lang2Stoplist=" + STOPLISTS + "empty.txt",
 126         "--dictInfo=SomeWikiDataTrans2Trans",
 127
 128         "--input4=" + WIKISPLIT_EN + "EN.data",
 129         "--input4Name=" + name,
 130         "--input4Format=" + EnTranslationToTranslationParser.NAME,
 131         "--input4LangPattern1=" + lang1,
 132         "--input4LangPattern2=" + lang2,
 133         "--input4PageLimit=1000",
 134
 135         "--print=" + result.getPath() + ".text",
 136     });
 137
 138     checkGolden(name, result);
 139   }
 140
 141   public void testWiktionary_WholeSection_DE() throws Exception {
 142     enWiktionaryTestWithWholeSectionToHtml("enwiktionary.WholeSection.DE.quickdic", "DE", 100);
 143   }
 144
 145   public void testWiktionary_WholeSection_EN() throws Exception {
 146     enWiktionaryTestWithWholeSectionToHtml("enwiktionary.WholeSection.EN.quickdic", "EN", 100);
 147   }
 148
 149   public void testWiktionary_WholeSection_IT() throws Exception {
 150     // Have to run to 800 to get a few verb conjugations (including essere!)
 151     enWiktionaryTestWithWholeSectionToHtml("enwiktionary.WholeSection.IT.quickdic", "IT", 800);
 152   }
 153
 154   public void enWiktionaryTestWithWholeSectionToHtml(final String name, final String langCode, final int pageLimit) throws Exception {
 155     final File result = new File(TEST_OUTPUTS + name);
 156     System.out.println("Writing to: " + result);
 157     DictionaryBuilder.main(new String[] {
 158         "--dictOut=" + result.getAbsolutePath(),
 159         "--lang1=" + langCode,
 160         "--lang2=" + "EN",
 161         "--lang1Stoplist=" + STOPLISTS + "empty.txt",
 162         "--lang2Stoplist=" + STOPLISTS + "empty.txt",
 163         "--dictInfo=SomeWikiDataWholeSection",
 164
 165         "--input4=" + WIKISPLIT_EN + langCode + ".data",
 166         "--input4Name=" + name,
 167         "--input4Format=" + WholeSectionToHtmlParser.NAME,
 168         "--input4WiktionaryLang=EN",
 169         "--input4SkipLang=" + langCode,
 170         "--input4TitleIndex=" + "1",
 171         "--input4PageLimit=" + pageLimit,
 172
 173         "--print=" + result.getPath() + ".text",
 174     });
 175     checkGolden(name, result);
 176   }
 177
 178   //-----------------------------------------------------------------
 179
 180   public void testSingleLang_EN() throws Exception {
 181       wiktionaryTestSingleLang("SingleLang_EN.quickdic", "EN", 100);
 182   }
 183
 184   public void testSingleLang_DE() throws Exception {
 185       wiktionaryTestSingleLang("SingleLang_DE.quickdic", "DE", 100);
 186   }
 187
 188   public void testSingleLang_IT() throws Exception {
 189       wiktionaryTestSingleLang("SingleLang_IT.quickdic", "IT", 100);
 190   }
 191
 192   public void testSingleLang_FR() throws Exception {
 193       wiktionaryTestSingleLang("SingleLang_FR.quickdic", "FR", 100);
 194   }
 195
 196   public void wiktionaryTestSingleLang(final String name, final String langCode, final int pageLimit) throws Exception {
 197       final File result = new File(TEST_OUTPUTS + name);
 198       System.out.println("Writing to: " + result);
 199       DictionaryBuilder.main(new String[] {
 200           "--dictOut=" + result.getAbsolutePath(),
 201           "--lang1=" + langCode,
 202           "--lang1Stoplist=" + STOPLISTS + "empty.txt",
 203           "--dictInfo=SomeWikiDataWholeSection",
 204           "--input4=" + WIKISPLIT + langCode.toLowerCase() + "/" + langCode + ".data",
 205           "--input4Name=" + name,
 206           "--input4Format=" + WholeSectionToHtmlParser.NAME,
 207           "--input4WiktionaryLang=" + langCode,
 208           "--input4SkipLang=" + langCode,
 209           "--input4TitleIndex=" + "1",
 210           "--input4PageLimit=" + pageLimit,
 211           "--print=" + result.getPath() + ".text",
 212       });
 213       checkGolden(name, result);
 214     }
 215
 216   //-----------------------------------------------------------------
 217
 218   public void testWiktionary_IT_EN() throws Exception {
 219     wiktionaryTestWithLangToEn("wiktionary.it_en.quickdic", "IT", "it.txt",
 220         "EN.data", "enwiktionary.english", "Italian", "it", 1000);
 221   }
 222
 223   public void testWiktionary_cmn_EN() throws Exception {
 224     wiktionaryTestWithLangToEn("wiktionary.cmn_en.quickdic", "cmn", "empty.txt",
 225         // These missing "e" prevents a complete match, forcing the name to be printed
 226         "EN.data", "enwiktionary.english", "Chinese|Mandarin", "cmn", 1000);
 227   }
 228
 229   public void testWiktionary_DE_EN() throws Exception {
 230     wiktionaryTestWithLangToEn("wiktionary.de_en.quickdic", "DE", "de.txt",
 231         "EN.data", "enwiktionary.english", "German", "de", 1000);
 232   }
 233
 234   public void testWiktionary_IT_IT() throws Exception {
 235     wiktionaryTestWithLangToEn("wiktionary.it_it.quickdic", "IT", "it.txt",
 236         "IT.data", "enwiktionary.italian", "Italian", "it", 1000);
 237   }
 238
 239   // French
 240   public void testWiktionary_FR_FR() throws Exception {
 241     wiktionaryTestWithLangToEn("wiktionary.fr_fr.quickdic", "FR", "fr.txt",
 242         "FR.data", "enwiktionary.french", "French", "fr", 1000);
 243   }
 244
 245
 246   // Arabic
 247   public void testWiktionary_AR_AR() throws Exception {
 248       // Arabic is really big for some reason, use fewer pages.
 249     wiktionaryTestWithLangToEn("wiktionary.ar_ar.quickdic", "AR", "empty.txt",
 250         "AR.data", "enwiktionary.arabic", "Arabic", "ar", 200);
 251   }
 252
 253   // Chinese
 254   public void testWiktionary_cmn_cmn() throws Exception {
 255     wiktionaryTestWithLangToEn("wiktionary.cmn_cmn.quickdic", "cmn", "empty.txt",
 256         // These missing "e" prevents a complete match, forcing the name to be printed.
 257         "cmn.data", "enwiktionary.chinese", "Chinese|Mandarin", "cmn", 1000);
 258   }
 259
 260   // German
 261   public void testWiktionary_DE_DE() throws Exception {
 262     wiktionaryTestWithLangToEn("wiktionary.de_de.quickdic", "DE", "de.txt",
 263         "DE.data", "enwiktionary.german", "German", "de", 1000);
 264   }
 265
 266   // Thai
 267   public void testWiktionary_TH_TH() throws Exception {
 268     wiktionaryTestWithLangToEn("wiktionary.th_th.quickdic", "TH", "empty.txt",
 269         // These missing "e" prevents a complete match, forcing the name to be printed.
 270         "TH.data", "enwiktionary.thai", "Thai", "th", 1000);
 271   }
 272
 273   public void wiktionaryTestWithLangToEn(final String name, final String lang1,
 274       final String stoplist, final String data, final String dictName,
 275       final String langPattern, final String langCode, int pageLimit) throws Exception {
 276     final File result = new File(TEST_OUTPUTS + name);
 277     System.out.println("Writing to: " + result);
 278     final String type = data.equals("EN.data") ? "EnToTranslation" : "EnForeign";
 279     DictionaryBuilder.main(new String[] {
 280         "--dictOut=" + result.getAbsolutePath(),
 281         "--lang1=" + lang1,
 282         "--lang2=EN",
 283         "--lang1Stoplist=" + STOPLISTS + stoplist,
 284         "--lang2Stoplist=" + STOPLISTS + "en.txt",
 285         "--dictInfo=SomeWikiData",
 286
 287         "--input4=" + WIKISPLIT_EN + data,
 288         "--input4Name=" + dictName,
 289         "--input4Format=enwiktionary",
 290         "--input4WiktionaryType=" + type,
 291         "--input4LangPattern=" + langPattern,
 292         "--input4LangCodePattern=" + langCode,
 293         "--input4EnIndex=2",
 294         "--input4PageLimit=" + pageLimit,
 295
 296         "--print=" + result.getPath() + ".text",
 297     });
 298
 299     checkGolden(name, result);
 300   }
 301
 302   public void testGermanCombined() throws Exception {
 303     final String name = "de-en.quickdic";
 304     final File result = new File(TEST_OUTPUTS + name);
 305     System.out.println("Writing to: " + result);
 306     DictionaryBuilder.main(new String[] {
 307         "--dictOut=" + result.getAbsolutePath(),
 308         "--lang1=DE",
 309         "--lang2=EN",
 310         "--dictInfo=@" + TEST_INPUTS + "de-en_dictInfo.txt",
 311
 312         "--input1=" + TEST_INPUTS + "de-en_chemnitz_100",
 313         "--input1Name=chemnitz",
 314         "--input1Charset=UTF8",
 315         "--input1Format=chemnitz",
 316
 317         "--input2=" + TEST_INPUTS + "de-en_dictcc_simulated",
 318         "--input2Name=dictcc",
 319         "--input2Charset=UTF8",
 320         "--input2Format=tab_separated",
 321
 322         "--print=" + result.getPath() + ".text",
 323     });
 324
 325     checkGolden(name, result);
 326   }
 327
 328   public void testItalianTurkish() throws Exception {
 329       final String name = "it-tr_dictcc.quickdic";
 330       final File result = new File(TEST_OUTPUTS + name);
 331       System.out.println("Writing to: " + result);
 332       DictionaryBuilder.main(new String[] {
 333           "--dictOut=" + result.getAbsolutePath(),
 334           "--lang1=IT",
 335           "--lang2=TR",
 336           "--dictInfo=it-tr_dictcc_simulated",
 337
 338           "--input1=" + TEST_INPUTS + "it-tr_dictcc_simulated.txt",
 339           "--input1Name=dictcc",
 340           "--input1Charset=UTF8",
 341           "--input1Format=tab_separated",
 342
 343           "--print=" + result.getPath() + ".text",
 344       });
 345
 346       checkGolden(name, result);
 347     }
 348
 349   private void checkGolden(final String dictName, final File dictFile)
 350       throws IOException, FileNotFoundException {
 351     // Check it once:
 352     assertFilesEqual(GOLDENS + dictName + ".text", dictFile.getPath() + ".text");
 353
 354     // Check it again.
 355     final Dictionary dict = new Dictionary(new RandomAccessFile(dictFile.getAbsolutePath(), "r"));
 356     final PrintStream out = new PrintStream(new File(dictFile.getPath() + ".text"));
 357     dict.print(out);
 358     out.close();
 359     assertFilesEqual(GOLDENS + dictName + ".text", dictFile.getPath() + ".text");
 360   }
 361
 362
 363   void assertFilesEqual(final String expected, final String actual) throws IOException {
 364     final String expectedString = FileUtil.readToString(new File(expected));
 365     final String actualString = FileUtil.readToString(new File(actual));
 366     assertEquals(expectedString, actualString);
 367   }
 368
 369
 370 }