import java.io.IOException;
import java.io.PrintStream;
import java.io.RandomAccessFile;
+import java.util.Collections;
+import com.hughes.android.dictionary.parser.wiktionary.EnTranslationToTranslationParser;
+import com.hughes.android.dictionary.parser.wiktionary.WholeSectionToHtmlParser;
import com.hughes.util.FileUtil;
import junit.framework.TestCase;
public class DictionaryBuilderTest extends TestCase {
- public static final String TEST_INPUTS = "../DictionaryData/testdata/inputs/";
- public static final String WIKISPLIT = "../DictionaryData/inputs/enWikiSplit/";
- public static final String STOPLISTS = "../DictionaryData/inputs/stoplists/";
- public static final String GOLDENS = "../DictionaryData/testdata/goldens/";
+ public static final String TEST_INPUTS = "testdata/inputs/";
+ public static final String WIKISPLIT_EN = "data/inputs/wikiSplit/en/";
+ public static final String STOPLISTS = "data/inputs/stoplists/";
+ public static final String GOLDENS = "testdata/goldens/";
- public static final String TEST_OUTPUTS = "../DictionaryData/testdata/outputs/";
+ public static final String TEST_OUTPUTS = "testdata/outputs/";
- public void testWiktionaryItalianFromItalian() throws Exception {
- final String name = "wiktionary.it_it.quickdic";
+ public void testItConj() throws Exception {
+ final String toParse = "{{it-conj-are|d|avere|pres2s=dai|pres3s=dà|pres3p=danno|prem1s=diedi|prem1s2=detti|prem2s=desti|prem3s=diede|prem3s2=dette|prem1p=demmo|prem2p=deste|prem3p=diedero|prem3p2=dettero|fut1s=darò|fut2s=darai|fut3s=darà|fut1p=daremo|fut2p=darete|fut3p=daranno|cond1s=darei|cond2s=daresti|cond3s=darebbe|cond1p=daremmo|cond2p=dareste|cond3p=darebbero|sub123s=dia|sub3p=diano|impsub12s=dessi|impsub3s=desse|impsub1p=dessimo|impsub2p=deste|impsub3p=dessero|imp2s=dà|imp2s2=dai|imp2s3=da'|imp3s=dia|imp3p=diano}}\n" +
+ "{{it-conj-are|accus|avere}}\n" +
+ "{{it-conj-care|pag|avere or essere}}\n" +
+ "{{it-conj-iare|studi|avere}}\n" +
+ "{{it-conj-iare-b|avvi|avere}}\n" +
+ "{{it-conj-ciare|pronunc|avere}}\n" +
+ "{{it-conj-ere|sed|essere|pres1s=siedo|pres1s2=seggo|pres2s=siedi|pres3s=siede|pres3p=siedono|pres3p2=seggono|fut1s2=siederò|fut2s2=siederai|fut3s2=siederà|fut1p2=siederemo|fut2p2=siederete|fut3p2=siederanno|cond1s2=siederei|cond2s2=siederesti|cond3s2=siederebbe|cond1p2=siederemmo|cond2p2=siedereste|cond3p2=siederebbero|sub123s=sieda|sub3p=siedano|imp2s=siedi|imp3s=sieda|imp3s2=segga|imp3p=siedano|imp3p2=seggano}}\n" +
+ "{{it-conj-ere|persuad|avere|pastp=persuaso|prem1s=persuasi|prem3s=persuase|prem3s2=''|prem3p=persuasero|prem3p2=''}}\n" +
+ "{{it-conj-ere|abbatt|avere}}\n" +
+ "{{it-conj-ire|copr|avere|pastp=coperto|prem1s2=copersi|prem3s2=coperse|prem3p2=copersero}}\n" +
+ "{{it-conj-ire-b|prefer|avere}}\n" +
+ "{{it-conj-urre|prod|avere}}\n" +
+ "{{it-conj-arsi|lav}}\n" +
+ "{{it-conj-ersi|abbatt}}\n" +
+ "{{it-conj-iarsi|annoi}}\n" +
+ "{{it-conj-carsi|coniug}}\n" +
+ "{{it-conj-ciarsi|affacc}}\n" +
+ "{{it-conj-irsi|vest}}\n" +
+ "{{it-conj-irsi-b|fer}}\n" +
+ "{{it-conj-ursi|rid|essere}}\n" +
+ "{{it-conj-cire|ricuc|avere}}\n" +
+ "{{it-conj-iarsi-b|riavvi|essere}}" +
+ "{{it-conj-fare|putre|avere}}\n" +
+ "{{it-conj-cirsi|cuc|essere}}\n"
+ ;
+ final DictionaryBuilder db = new DictionaryBuilder("", Language.en, Language.it, "", "", Collections.singleton("X"), Collections.singleton("X"));
+ WholeSectionToHtmlParser parser = new WholeSectionToHtmlParser(db.indexBuilders.get(0), "EN", "IT");
+ parser.title = "dummyTitle";
+ parser.entrySource = new EntrySource(0, "dummySource", 0);
+ parser.parseSection("dummyHeading", toParse);
+ db.build();
+
+ final String dictName = "testItConj.html";
+ final PrintStream out = new PrintStream(new File(TEST_OUTPUTS, dictName));
+ db.dictionary.print(out);
+ out.close();
+
+ assertFilesEqual(GOLDENS + dictName, TEST_OUTPUTS + dictName);
+ }
+
+ public void doTestCustomDict(final String name, final String lang1,
+ final String lang2, final String inputFile) throws Exception {
final File result = new File(TEST_OUTPUTS + name);
System.out.println("Writing to: " + result);
DictionaryBuilder.main(new String[] {
"--dictOut=" + result.getAbsolutePath(),
- "--lang1=IT",
- "--lang2=EN",
- "--lang1Stoplist=" + STOPLISTS + "it.txt",
- "--lang2Stoplist=" + STOPLISTS + "en.txt",
- "--dictInfo=SomeWikiData",
+ "--lang1=" + lang1,
+ "--lang2=" + lang2,
+ "--lang1Stoplist=" + STOPLISTS + "empty.txt",
+ "--lang2Stoplist=" + STOPLISTS + "empty.txt",
+ "--dictInfo=bleh.",
+
+ "--input1=testdata/inputs/" + inputFile,
+ "--input1Name=my_input_" + name,
+ "--input1Charset=ISO-8859-1",
+ "--input1Format=tab_separated",
- "--input4=" + WIKISPLIT + "italian.data",
- "--input4Name=enwiktionary.italian",
- "--input4Format=enwiktionary",
- "--input4LangPattern=Italian",
- "--input4LangCodePattern=it",
- "--input4EnIndex=2",
+ "--print=" + result.getPath() + ".text",
+ });
+
+ checkGolden(name, result);
+ }
+
+ public void test_FR_NL() throws Exception {
+ doTestCustomDict("QuickDic-FR-NL.quickdic", "FR", "NL", "QuickDic-FR-NL.txt");
+ }
+
+ public void testWiktionary_en_de2fr() throws Exception {
+ wiktionaryTestWithEnTrans2Trans("wiktionary.de_fr.quickdic", "DE", "FR");
+ }
+
+ public void wiktionaryTestWithEnTrans2Trans(final String name, final String lang1,
+ final String lang2) throws Exception {
+ final File result = new File(TEST_OUTPUTS + name);
+ System.out.println("Writing to: " + result);
+ DictionaryBuilder.main(new String[] {
+ "--dictOut=" + result.getAbsolutePath(),
+ "--lang1=" + lang1,
+ "--lang2=" + lang2,
+ "--lang1Stoplist=" + STOPLISTS + "empty.txt",
+ "--lang2Stoplist=" + STOPLISTS + "empty.txt",
+ "--dictInfo=SomeWikiDataTrans2Trans",
+
+ "--input4=" + WIKISPLIT_EN + "EN.data",
+ "--input4Name=" + name,
+ "--input4Format=" + EnTranslationToTranslationParser.NAME,
+ "--input4LangPattern1=" + lang1,
+ "--input4LangPattern2=" + lang2,
"--input4PageLimit=1000",
"--print=" + result.getPath() + ".text",
checkGolden(name, result);
}
- public void testWiktionaryItalianFromEnglish() throws Exception {
- final String name = "wiktionary.it_en.quickdic";
+ public void testWiktionary_WholeSection_DE() throws Exception {
+ wiktionaryTestWithWholeSectionToHtml("wiktionary.WholeSection.DE.quickdic", "DE", 100);
+ }
+
+ public void testWiktionary_WholeSection_EN() throws Exception {
+ wiktionaryTestWithWholeSectionToHtml("wiktionary.WholeSection.EN.quickdic", "EN", 100);
+ }
+
+ public void testWiktionary_WholeSection_IT() throws Exception {
+ // Have to run to 800 to get a few verb conjugations (including essere!)
+ wiktionaryTestWithWholeSectionToHtml("wiktionary.WholeSection.IT.quickdic", "IT", 800);
+ }
+
+ public void wiktionaryTestWithWholeSectionToHtml(final String name, final String langCode, final int pageLimit) throws Exception {
final File result = new File(TEST_OUTPUTS + name);
System.out.println("Writing to: " + result);
DictionaryBuilder.main(new String[] {
"--dictOut=" + result.getAbsolutePath(),
- "--lang1=IT",
+ "--lang1=" + langCode,
+ "--lang2=" + "EN",
+ "--lang1Stoplist=" + STOPLISTS + "empty.txt",
+ "--lang2Stoplist=" + STOPLISTS + "empty.txt",
+ "--dictInfo=SomeWikiDataWholeSection",
+
+ "--input4=" + WIKISPLIT_EN + langCode + ".data",
+ "--input4Name=" + name,
+ "--input4Format=" + WholeSectionToHtmlParser.NAME,
+ "--input4WiktionaryLang=EN",
+ "--input4SkipLang=" + langCode,
+ "--input4TitleIndex=" + "1",
+ "--input4PageLimit=" + pageLimit,
+
+ "--print=" + result.getPath() + ".text",
+ });
+ checkGolden(name, result);
+ }
+
+
+ public void testWiktionary_IT_EN() throws Exception {
+ wiktionaryTestWithLangToEn("wiktionary.it_en.quickdic", "IT", "it.txt",
+ "EN.data", "enwiktionary.english", "Italian", "it");
+ }
+
+ public void testWiktionary_ZH_EN() throws Exception {
+ wiktionaryTestWithLangToEn("wiktionary.zh_en.quickdic", "ZH", "empty.txt",
+ // These missing "e" prevents a complete match, forcing the name to be printed
+ "EN.data", "enwiktionary.english", "Chinese|Mandarin|Cantones", "zh");
+ }
+
+ public void testWiktionary_DE_EN() throws Exception {
+ wiktionaryTestWithLangToEn("wiktionary.de_en.quickdic", "DE", "de.txt",
+ "EN.data", "enwiktionary.english", "German", "it");
+ }
+
+ public void testWiktionary_IT_IT() throws Exception {
+ wiktionaryTestWithLangToEn("wiktionary.it_it.quickdic", "IT", "it.txt",
+ "IT.data", "enwiktionary.italian", "Italian", "it");
+ }
+
+ // French
+ public void testWiktionary_FR_FR() throws Exception {
+ wiktionaryTestWithLangToEn("wiktionary.fr_fr.quickdic", "FR", "fr.txt",
+ "FR.data", "enwiktionary.french", "French", "fr");
+ }
+
+
+ // Arabic
+ public void testWiktionary_AR_AR() throws Exception {
+ wiktionaryTestWithLangToEn("wiktionary.ar_ar.quickdic", "AR", "empty.txt",
+ "AR.data", "enwiktionary.arabic", "Arabic", "ar");
+ }
+
+ // Chinese
+ public void testWiktionary_ZH_ZH() throws Exception {
+ wiktionaryTestWithLangToEn("wiktionary.zh_zh.quickdic", "ZH", "empty.txt",
+ // These missing "e" prevents a complete match, forcing the name to be printed.
+ "ZH.data", "enwiktionary.chinese", "Chinese|Mandarin|Cantones", "zh");
+ }
+
+ // German
+ public void testWiktionary_DE_DE() throws Exception {
+ wiktionaryTestWithLangToEn("wiktionary.de_de.quickdic", "DE", "de.txt",
+ "DE.data", "enwiktionary.german", "German", "it");
+ }
+
+ // Thai
+ public void testWiktionary_TH_TH() throws Exception {
+ wiktionaryTestWithLangToEn("wiktionary.th_th.quickdic", "TH", "empty.txt",
+ // These missing "e" prevents a complete match, forcing the name to be printed.
+ "TH.data", "enwiktionary.thai", "Thai", "th");
+ }
+
+ public void wiktionaryTestWithLangToEn(final String name, final String lang1,
+ final String stoplist, final String data, final String dictName,
+ final String langPattern, final String langCode) throws Exception {
+ final File result = new File(TEST_OUTPUTS + name);
+ System.out.println("Writing to: " + result);
+ final String type = data.equals("EN.data") ? "EnToTranslation" : "EnForeign";
+ DictionaryBuilder.main(new String[] {
+ "--dictOut=" + result.getAbsolutePath(),
+ "--lang1=" + lang1,
"--lang2=EN",
- "--lang1Stoplist=" + STOPLISTS + "it.txt",
+ "--lang1Stoplist=" + STOPLISTS + stoplist,
"--lang2Stoplist=" + STOPLISTS + "en.txt",
"--dictInfo=SomeWikiData",
- "--input3=" + WIKISPLIT + "english.data",
- "--input3Name=enwiktionary.english",
- "--input3Format=enwiktionary",
- "--input3LangPattern=Italian",
- "--input3LangCodePattern=it",
- "--input3EnIndex=2",
- "--input3PageLimit=1000",
+ "--input4=" + WIKISPLIT_EN + data,
+ "--input4Name=" + dictName,
+ "--input4Format=enwiktionary",
+ "--input4WiktionaryType=" + type,
+ "--input4LangPattern=" + langPattern,
+ "--input4LangCodePattern=" + langCode,
+ "--input4EnIndex=2",
+ "--input4PageLimit=1000",
"--print=" + result.getPath() + ".text",
});
checkGolden(name, result);
}
-
public void testGermanCombined() throws Exception {
final String name = "de-en.quickdic";
final File result = new File(TEST_OUTPUTS + name);
"--input2=" + TEST_INPUTS + "de-en_dictcc_simulated",
"--input2Name=dictcc",
"--input2Charset=UTF8",
- "--input2Format=dictcc",
+ "--input2Format=tab_separated",
"--print=" + result.getPath() + ".text",
});
// Check it again.
final Dictionary dict = new Dictionary(new RandomAccessFile(dictFile.getAbsolutePath(), "r"));
- final PrintStream out = new PrintStream(new File(dictFile.getName() + ".text"));
+ final PrintStream out = new PrintStream(new File(dictFile.getPath() + ".text"));
dict.print(out);
out.close();
assertFilesEqual(GOLDENS + dictName + ".text", dictFile.getPath() + ".text");
projects / DictionaryPC.git / blobdiff