src/com/hughes/android/dictionary/engine/DictionaryBuilderTest.java

   1 // Copyright 2011 Google Inc. All Rights Reserved.
   2 //
   3 // Licensed under the Apache License, Version 2.0 (the "License");
   4 // you may not use this file except in compliance with the License.
   5 // You may obtain a copy of the License at
   6 //
   7 //     http://www.apache.org/licenses/LICENSE-2.0
   8 //
   9 // Unless required by applicable law or agreed to in writing, software
  10 // distributed under the License is distributed on an "AS IS" BASIS,
  11 // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  12 // See the License for the specific language governing permissions and
  13 // limitations under the License.
  14
  15 package com.hughes.android.dictionary.engine;
  16
  17 import java.io.File;
  18 import java.io.FileNotFoundException;
  19 import java.io.IOException;
  20 import java.io.PrintStream;
  21 import java.io.RandomAccessFile;
  22 import java.util.Collections;
  23
  24 import com.hughes.android.dictionary.parser.wiktionary.EnTranslationToTranslationParser;
  25 import com.hughes.android.dictionary.parser.wiktionary.WholeSectionToHtmlParser;
  26 import com.hughes.util.FileUtil;
  27
  28 import junit.framework.TestCase;
  29
  30 public class DictionaryBuilderTest extends TestCase {
  31
  32   public static final String TEST_INPUTS = "testdata/inputs/";
  33   public static final String WIKISPLIT_EN = "data/inputs/wikiSplit/en/";
  34   public static final String STOPLISTS = "data/inputs/stoplists/";
  35   public static final String GOLDENS = "testdata/goldens/";
  36
  37   public static final String TEST_OUTPUTS = "testdata/outputs/";
  38
  39   public void testItConj() throws Exception {
  40       final String toParse = "{{it-conj-are|d|avere|pres2s=dai|pres3s=dà|pres3p=danno|prem1s=diedi|prem1s2=detti|prem2s=desti|prem3s=diede|prem3s2=dette|prem1p=demmo|prem2p=deste|prem3p=diedero|prem3p2=dettero|fut1s=darò|fut2s=darai|fut3s=darà|fut1p=daremo|fut2p=darete|fut3p=daranno|cond1s=darei|cond2s=daresti|cond3s=darebbe|cond1p=daremmo|cond2p=dareste|cond3p=darebbero|sub123s=dia|sub3p=diano|impsub12s=dessi|impsub3s=desse|impsub1p=dessimo|impsub2p=deste|impsub3p=dessero|imp2s=dà|imp2s2=dai|imp2s3=da'|imp3s=dia|imp3p=diano}}\n" +
  41               "{{it-conj-are|accus|avere}}\n" +
  42               "{{it-conj-care|pag|avere or essere}}\n" +
  43               "{{it-conj-iare|studi|avere}}\n" +
  44               "{{it-conj-iare-b|avvi|avere}}\n" +
  45               "{{it-conj-ciare|pronunc|avere}}\n" +
  46               "{{it-conj-ere|sed|essere|pres1s=siedo|pres1s2=seggo|pres2s=siedi|pres3s=siede|pres3p=siedono|pres3p2=seggono|fut1s2=siederò|fut2s2=siederai|fut3s2=siederà|fut1p2=siederemo|fut2p2=siederete|fut3p2=siederanno|cond1s2=siederei|cond2s2=siederesti|cond3s2=siederebbe|cond1p2=siederemmo|cond2p2=siedereste|cond3p2=siederebbero|sub123s=sieda|sub3p=siedano|imp2s=siedi|imp3s=sieda|imp3s2=segga|imp3p=siedano|imp3p2=seggano}}\n" +
  47               "{{it-conj-ere|persuad|avere|pastp=persuaso|prem1s=persuasi|prem3s=persuase|prem3s2=''|prem3p=persuasero|prem3p2=''}}\n" +
  48               "{{it-conj-ere|abbatt|avere}}\n" +
  49               "{{it-conj-ire|copr|avere|pastp=coperto|prem1s2=copersi|prem3s2=coperse|prem3p2=copersero}}\n" +
  50               "{{it-conj-ire-b|prefer|avere}}\n" +
  51               "{{it-conj-urre|prod|avere}}\n" +
  52               "{{it-conj-arsi|lav}}\n" +
  53               "{{it-conj-ersi|abbatt}}\n" +
  54               "{{it-conj-iarsi|annoi}}\n" +
  55               "{{it-conj-carsi|coniug}\n" +
  56               "{{it-conj-ciarsi|affacc}\n" +
  57               "{{it-conj-irsi|vest}}\n" +
  58               "{{it-conj-irsi-b|fer}}\n" +
  59               "{{it-conj-ursi|rid|essere}}\n" +
  60               "{{it-conj-cire|ricuc|avere}}\n" +
  61               "{{it-conj-iarsi-b|riavvi|essere}}" +
  62               "{{it-conj-fare|putre|avere}}\n" +
  63               "{{it-conj-cirsi|cuc|essere}}\n"
  64               ;
  65       final DictionaryBuilder db = new DictionaryBuilder("", Language.en, Language.it,  "", "", Collections.singleton("X"), Collections.singleton("X"));
  66       WholeSectionToHtmlParser parser = new WholeSectionToHtmlParser(db.indexBuilders.get(0), "EN", "IT");
  67       parser.title = "dummyTitle";
  68       parser.parseSection("dummyHeading", toParse);
  69       db.build();
  70
  71       final String dictName = "testItConj.html";
  72       final PrintStream out = new PrintStream(new File(TEST_OUTPUTS, dictName));
  73       db.dictionary.print(out);
  74       out.close();
  75
  76       assertFilesEqual(GOLDENS + dictName, TEST_OUTPUTS + dictName);
  77   }
  78
  79   public void doTestCustomDict(final String name, final String lang1,
  80       final String lang2, final String inputFile) throws Exception {
  81     final File result = new File(TEST_OUTPUTS + name);
  82     System.out.println("Writing to: " + result);
  83     DictionaryBuilder.main(new String[] {
  84         "--dictOut=" + result.getAbsolutePath(),
  85         "--lang1=" + lang1,
  86         "--lang2=" + lang2,
  87         "--lang1Stoplist=" + STOPLISTS + "empty.txt",
  88         "--lang2Stoplist=" + STOPLISTS + "empty.txt",
  89         "--dictInfo=bleh.",
  90
  91         "--input1=testdata/inputs/" + inputFile,
  92         "--input1Name=my_input_" + name,
  93         "--input1Charset=ISO-8859-1",
  94         "--input1Format=tab_separated",
  95
  96         "--print=" + result.getPath() + ".text",
  97     });
  98
  99     checkGolden(name, result);
 100   }
 101
 102   public void test_FR_NL() throws Exception {
 103     doTestCustomDict("QuickDic-FR-NL.quickdic", "FR", "NL", "QuickDic-FR-NL.txt");
 104   }
 105
 106   public void testWiktionary_en_de2fr() throws Exception {
 107     wiktionaryTestWithEnTrans2Trans("wiktionary.de_fr.quickdic", "DE", "FR");
 108   }
 109
 110   public void wiktionaryTestWithEnTrans2Trans(final String name, final String lang1,
 111       final String lang2) throws Exception {
 112     final File result = new File(TEST_OUTPUTS + name);
 113     System.out.println("Writing to: " + result);
 114     DictionaryBuilder.main(new String[] {
 115         "--dictOut=" + result.getAbsolutePath(),
 116         "--lang1=" + lang1,
 117         "--lang2=" + lang2,
 118         "--lang1Stoplist=" + STOPLISTS + "empty.txt",
 119         "--lang2Stoplist=" + STOPLISTS + "empty.txt",
 120         "--dictInfo=SomeWikiDataTrans2Trans",
 121
 122         "--input4=" + WIKISPLIT_EN + "EN.data",
 123         "--input4Name=" + name,
 124         "--input4Format=" + EnTranslationToTranslationParser.NAME,
 125         "--input4LangPattern1=" + lang1,
 126         "--input4LangPattern2=" + lang2,
 127         "--input4PageLimit=1000",
 128
 129         "--print=" + result.getPath() + ".text",
 130     });
 131
 132     checkGolden(name, result);
 133   }
 134
 135   public void testWiktionary_WholeSection_DE() throws Exception {
 136     wiktionaryTestWithWholeSectionToHtml("wiktionary.WholeSection.DE.quickdic", "DE", 100);
 137   }
 138
 139   public void testWiktionary_WholeSection_EN() throws Exception {
 140     wiktionaryTestWithWholeSectionToHtml("wiktionary.WholeSection.EN.quickdic", "EN", 100);
 141   }
 142
 143   public void testWiktionary_WholeSection_IT() throws Exception {
 144     // Have to run to 800 to get a few verb conjugations (including essere!)
 145     wiktionaryTestWithWholeSectionToHtml("wiktionary.WholeSection.IT.quickdic", "IT", 800);
 146   }
 147
 148   public void wiktionaryTestWithWholeSectionToHtml(final String name, final String langCode, final int pageLimit) throws Exception {
 149     final File result = new File(TEST_OUTPUTS + name);
 150     System.out.println("Writing to: " + result);
 151     DictionaryBuilder.main(new String[] {
 152         "--dictOut=" + result.getAbsolutePath(),
 153         "--lang1=" + langCode,
 154         "--lang2=" + "EN",
 155         "--lang1Stoplist=" + STOPLISTS + "empty.txt",
 156         "--lang2Stoplist=" + STOPLISTS + "empty.txt",
 157         "--dictInfo=SomeWikiDataWholeSection",
 158
 159         "--input4=" + WIKISPLIT_EN + langCode + ".data",
 160         "--input4Name=" + name,
 161         "--input4Format=" + WholeSectionToHtmlParser.NAME,
 162         "--input4WiktionaryLang=EN",
 163         "--input4SkipLang=" + langCode,
 164         "--input4TitleIndex=" + "1",
 165         "--input4PageLimit=" + pageLimit,
 166
 167         "--print=" + result.getPath() + ".text",
 168     });
 169     checkGolden(name, result);
 170   }
 171
 172
 173   public void testWiktionary_IT_EN() throws Exception {
 174     wiktionaryTestWithLangToEn("wiktionary.it_en.quickdic", "IT", "it.txt",
 175         "EN.data", "enwiktionary.english", "Italian", "it");
 176   }
 177
 178   public void testWiktionary_ZH_EN() throws Exception {
 179     wiktionaryTestWithLangToEn("wiktionary.zh_en.quickdic", "ZH", "empty.txt",
 180         // These missing "e" prevents a complete match, forcing the name to be printed
 181         "EN.data", "enwiktionary.english", "Chinese|Mandarin|Cantones", "zh");
 182   }
 183
 184   public void testWiktionary_DE_EN() throws Exception {
 185     wiktionaryTestWithLangToEn("wiktionary.de_en.quickdic", "DE", "de.txt",
 186         "EN.data", "enwiktionary.english", "German", "it");
 187   }
 188
 189   public void testWiktionary_IT_IT() throws Exception {
 190     wiktionaryTestWithLangToEn("wiktionary.it_it.quickdic", "IT", "it.txt",
 191         "IT.data", "enwiktionary.italian", "Italian", "it");
 192   }
 193
 194   // French
 195   public void testWiktionary_FR_FR() throws Exception {
 196     wiktionaryTestWithLangToEn("wiktionary.fr_fr.quickdic", "FR", "fr.txt",
 197         "FR.data", "enwiktionary.french", "French", "fr");
 198   }
 199
 200
 201   // Arabic
 202   public void testWiktionary_AR_AR() throws Exception {
 203     wiktionaryTestWithLangToEn("wiktionary.ar_ar.quickdic", "AR", "empty.txt",
 204         "AR.data", "enwiktionary.arabic", "Arabic", "ar");
 205   }
 206
 207   // Chinese
 208   public void testWiktionary_ZH_ZH() throws Exception {
 209     wiktionaryTestWithLangToEn("wiktionary.zh_zh.quickdic", "ZH", "empty.txt",
 210         // These missing "e" prevents a complete match, forcing the name to be printed.
 211         "ZH.data", "enwiktionary.chinese", "Chinese|Mandarin|Cantones", "zh");
 212   }
 213
 214   // German
 215   public void testWiktionary_DE_DE() throws Exception {
 216     wiktionaryTestWithLangToEn("wiktionary.de_de.quickdic", "DE", "de.txt",
 217         "DE.data", "enwiktionary.german", "German", "it");
 218   }
 219
 220   // Thai
 221   public void testWiktionary_TH_TH() throws Exception {
 222     wiktionaryTestWithLangToEn("wiktionary.th_th.quickdic", "TH", "empty.txt",
 223         // These missing "e" prevents a complete match, forcing the name to be printed.
 224         "TH.data", "enwiktionary.thai", "Thai", "th");
 225   }
 226
 227   public void wiktionaryTestWithLangToEn(final String name, final String lang1,
 228       final String stoplist, final String data, final String dictName,
 229       final String langPattern, final String langCode) throws Exception {
 230     final File result = new File(TEST_OUTPUTS + name);
 231     System.out.println("Writing to: " + result);
 232     final String type = data.equals("EN.data") ? "EnToTranslation" : "EnForeign";
 233     DictionaryBuilder.main(new String[] {
 234         "--dictOut=" + result.getAbsolutePath(),
 235         "--lang1=" + lang1,
 236         "--lang2=EN",
 237         "--lang1Stoplist=" + STOPLISTS + stoplist,
 238         "--lang2Stoplist=" + STOPLISTS + "en.txt",
 239         "--dictInfo=SomeWikiData",
 240
 241         "--input4=" + WIKISPLIT_EN + data,
 242         "--input4Name=" + dictName,
 243         "--input4Format=enwiktionary",
 244         "--input4WiktionaryType=" + type,
 245         "--input4LangPattern=" + langPattern,
 246         "--input4LangCodePattern=" + langCode,
 247         "--input4EnIndex=2",
 248         "--input4PageLimit=1000",
 249
 250         "--print=" + result.getPath() + ".text",
 251     });
 252
 253     checkGolden(name, result);
 254   }
 255
 256   public void testGermanCombined() throws Exception {
 257     final String name = "de-en.quickdic";
 258     final File result = new File(TEST_OUTPUTS + name);
 259     System.out.println("Writing to: " + result);
 260     DictionaryBuilder.main(new String[] {
 261         "--dictOut=" + result.getAbsolutePath(),
 262         "--lang1=DE",
 263         "--lang2=EN",
 264         "--dictInfo=@" + TEST_INPUTS + "de-en_dictInfo.txt",
 265
 266         "--input1=" + TEST_INPUTS + "de-en_chemnitz_100",
 267         "--input1Name=chemnitz",
 268         "--input1Charset=UTF8",
 269         "--input1Format=chemnitz",
 270
 271         "--input2=" + TEST_INPUTS + "de-en_dictcc_simulated",
 272         "--input2Name=dictcc",
 273         "--input2Charset=UTF8",
 274         "--input2Format=tab_separated",
 275
 276         "--print=" + result.getPath() + ".text",
 277     });
 278
 279     checkGolden(name, result);
 280   }
 281
 282   private void checkGolden(final String dictName, final File dictFile)
 283       throws IOException, FileNotFoundException {
 284     // Check it once:
 285     assertFilesEqual(GOLDENS + dictName + ".text", dictFile.getPath() + ".text");
 286
 287     // Check it again.
 288     final Dictionary dict = new Dictionary(new RandomAccessFile(dictFile.getAbsolutePath(), "r"));
 289     final PrintStream out = new PrintStream(new File(dictFile.getPath() + ".text"));
 290     dict.print(out);
 291     out.close();
 292     assertFilesEqual(GOLDENS + dictName + ".text", dictFile.getPath() + ".text");
 293   }
 294
 295
 296   void assertFilesEqual(final String expected, final String actual) throws IOException {
 297     final String expectedString = FileUtil.readToString(new File(expected));
 298     final String actualString = FileUtil.readToString(new File(actual));
 299     assertEquals(expectedString, actualString);
 300   }
 301
 302
 303 }