From: Thad Hughes Date: Sun, 20 May 2012 23:31:08 +0000 (-0700) Subject: DictionaryBuilder prints sortable langs, JP->JA fix. X-Git-Url: http://gitweb.fperrin.net/?p=DictionaryPC.git;a=commitdiff_plain;h=9930633d9597c9dde5c040b7c24ba729f2b5d889 DictionaryBuilder prints sortable langs, JP->JA fix. --- diff --git a/src/com/hughes/android/dictionary/engine/CheckDictionariesMain.java b/src/com/hughes/android/dictionary/engine/CheckDictionariesMain.java index 846aaf1..e4d5c0c 100644 --- a/src/com/hughes/android/dictionary/engine/CheckDictionariesMain.java +++ b/src/com/hughes/android/dictionary/engine/CheckDictionariesMain.java @@ -1,8 +1,6 @@ package com.hughes.android.dictionary.engine; -import java.io.BufferedWriter; import java.io.File; -import java.io.FileWriter; import java.io.IOException; import java.io.PrintWriter; import java.io.RandomAccessFile; @@ -12,6 +10,7 @@ import java.util.Collections; import java.util.List; import com.hughes.android.dictionary.DictionaryInfo; +import com.hughes.android.dictionary.parser.wiktionary.WiktionaryLangs; public class CheckDictionariesMain { @@ -25,6 +24,7 @@ public class CheckDictionariesMain { // dictionaryInfoOut.println("# LANG_1\t%LANG_2\tFILENAME\tVERSION_CODE\tFILESIZE\tNUM_MAIN_WORDS_1\tNUM_MAIN_WORDS_2\tNUM_ALL_WORDS_1\tNUM_ALL_WORDS_2"); final File[] files = dictDir.listFiles(); + final List dictNames = new ArrayList(); Arrays.sort(files); for (final File dictFile : files) { if (!dictFile.getName().endsWith("quickdic")) { @@ -56,19 +56,25 @@ public class CheckDictionariesMain { // Find the stats. System.out.println("Stats..."); + final String lang1 = WiktionaryLangs.isoCodeToWikiName.get(dictionaryInfo.indexInfos.get(0).shortName); + final String lang2 = WiktionaryLangs.isoCodeToWikiName.get(dictionaryInfo.indexInfos.get(1).shortName); + dictNames.add(String.format("%s-%s\n", lang1, lang2)); final String row = dictionaryInfo.append(new StringBuilder()).toString(); if (!zipFile.canRead()) { System.err.println("Couldn't read zipfile: " + zipFile); } System.out.println(row + "\n"); + dictionaryInfoOut.println(row); dictionaryInfoOut.flush(); raf.close(); - } + Collections.sort(dictNames); + System.out.println(dictNames.toString().replaceAll(",", " *")); + dictionaryInfoOut.close(); } diff --git a/src/com/hughes/android/dictionary/engine/DictionaryBuilder.java b/src/com/hughes/android/dictionary/engine/DictionaryBuilder.java index 7aee442..45a6cee 100644 --- a/src/com/hughes/android/dictionary/engine/DictionaryBuilder.java +++ b/src/com/hughes/android/dictionary/engine/DictionaryBuilder.java @@ -166,8 +166,14 @@ public class DictionaryBuilder { } parser.parse(file, entrySource, pageLimit); } else if (EnTranslationToTranslationParser.NAME.equals(inputFormat)) { - final Pattern codePattern1 = Pattern.compile(keyValueArgs.remove(prefix + "LangPattern1"), Pattern.CASE_INSENSITIVE); - final Pattern codePattern2 = Pattern.compile(keyValueArgs.remove(prefix + "LangPattern2"), Pattern.CASE_INSENSITIVE); + final String code1 = keyValueArgs.remove(prefix + "LangPattern1"); + final String code2 = keyValueArgs.remove(prefix + "LangPattern2"); + if (code1 == null || code2 == null) { + fatalError("Must specify LangPattern1 and LangPattern2."); + return; + } + final Pattern codePattern1 = Pattern.compile(code1, Pattern.CASE_INSENSITIVE); + final Pattern codePattern2 = Pattern.compile(code2, Pattern.CASE_INSENSITIVE); new EnTranslationToTranslationParser(dictionaryBuilder.indexBuilders, new Pattern[] {codePattern1, codePattern2}).parse(file, entrySource, pageLimit); } else { fatalError("Invalid or missing input format: " + inputFormat); diff --git a/src/com/hughes/android/dictionary/engine/DictionaryBuilderMain.java b/src/com/hughes/android/dictionary/engine/DictionaryBuilderMain.java index 08656da..159513d 100644 --- a/src/com/hughes/android/dictionary/engine/DictionaryBuilderMain.java +++ b/src/com/hughes/android/dictionary/engine/DictionaryBuilderMain.java @@ -14,11 +14,16 @@ package com.hughes.android.dictionary.engine; +import java.util.Arrays; import java.util.LinkedHashMap; +import java.util.LinkedHashSet; +import java.util.List; import java.util.Map; +import java.util.Set; import junit.framework.TestCase; +import com.hughes.android.dictionary.parser.wiktionary.EnTranslationToTranslationParser; import com.hughes.android.dictionary.parser.wiktionary.WiktionaryLangs; public class DictionaryBuilderMain extends TestCase { @@ -27,7 +32,7 @@ public class DictionaryBuilderMain extends TestCase { static final String STOPLISTS = "data/inputs/stoplists/"; static final String OUTPUTS = "data/outputs/"; - public static void main(final String[] args) throws Exception { + public static void main(final String[] args) throws Exception { // Builds all the dictionaries it can, outputs list to a text file. @@ -61,6 +66,122 @@ public class DictionaryBuilderMain extends TestCase { // HACK: The missing "e" prevents a full match, causing "Cantonese" to be appended to the entries. isoToRegex.put("ZH", "Chinese|Mandarin|Cantones"); + + // Build the non EN ones. + + final String[][] nonEnPairs = new String[][] { + {"AR", "DE" }, + {"AR", "ES" }, + {"AR", "FR" }, + {"AR", "IT" }, + {"AR", "JA" }, + {"AR", "RU" }, + {"AR", "ZH" }, + + {"DE", "FR" }, + {"DE", "CA" }, // Catalan + {"DE", "CS" }, // Czech + {"DE", "EO" }, // Esperanto + {"DE", "ES" }, + {"DE", "FR" }, + {"DE", "HU" }, // Hungarian + {"DE", "IT" }, + {"DE", "JA" }, + {"DE", "LA" }, // Latin + {"DE", "PL" }, // Polish + {"DE", "RU" }, + {"DE", "SV" }, // Swedish + {"DE", "ZH" }, + + + {"FR", "BG" }, // Bulgarian + {"FR", "CS" }, // Czech + {"FR", "DE" }, + {"FR", "ES" }, + {"FR", "IT" }, + {"FR", "JA" }, + {"FR", "LA" }, + {"FR", "NL" }, // Dutch + {"FR", "RU" }, + {"FR", "ZH" }, + + {"IT", "DE" }, + {"IT", "EL" }, // Greek + {"IT", "ES" }, + {"IT", "FR" }, + {"IT", "HU" }, + {"IT", "JA" }, + {"IT", "LA" }, // Latin + {"IT", "LV" }, // Latvian + {"IT", "NL" }, + {"IT", "PL" }, + {"IT", "RU" }, + {"IT", "SV" }, + {"IT", "ZH" }, + + {"JA", "ZH" }, + {"JA", "AR" }, + + {"ZH", "AR" }, + {"ZH", "DE" }, + {"ZH", "ES" }, + {"ZH", "FR" }, + {"ZH", "IT" }, + + + {"NO", "SV" }, + {"NO", "FI" }, + {"FI", "SV" }, + {"AR", "HE" }, + {"KO", "JA" }, + {"KO", "ZH" }, + }; + + final Set> done = new LinkedHashSet>(); + for (final String[] pair : nonEnPairs) { + Arrays.sort(pair); + final List pairList = Arrays.asList(pair); + if (done.contains(pairList)) { + continue; + } + done.add(pairList); + + final String lang1 = pair[0]; + final String lang2 = pair[1]; + + final String dictFile = String.format("%s/%s-%s_enwiktionary_BETA.quickdic", + OUTPUTS, lang1, lang2); + System.out.println("building dictFile: " + dictFile); + + if (!isoToStoplist.containsKey(lang1)) { + isoToStoplist.put(lang1, "empty.txt"); + } + if (!isoToStoplist.containsKey(lang2)) { + isoToStoplist.put(lang2, "empty.txt"); + } + + DictionaryBuilder.main(new String[] { + String.format("--dictOut=%s", dictFile), + String.format("--lang1=%s", lang1), + String.format("--lang2=%s", lang2), + String.format("--lang1Stoplist=%s", STOPLISTS + isoToStoplist.get(lang1)), + String.format("--lang2Stoplist=%s", STOPLISTS + isoToStoplist.get(lang2)), + String.format("--dictInfo=(EN)Wikitionary-based %s-%s dictionary.", lang1, lang2), + + String.format("--input2=%swikiSplit/en/EN.data", INPUTS), + String.format("--input2Name=BETA!enwiktionary.%s-%s", lang1, lang2), + String.format("--input2Format=%s", EnTranslationToTranslationParser.NAME), + String.format("--input2LangPattern1=%s", lang1), + String.format("--input2LangPattern2=%s", lang2), + }); + } + if (1==1) { + return; + } + + + // Now build the EN ones. + // isoToWikiName.keySet().retainAll(Arrays.asList("UK", "HR", "FI")); //isoToWikiName.clear(); boolean go = true; @@ -113,6 +234,8 @@ public class DictionaryBuilderMain extends TestCase { } // foreignIso + // Now special case German-English. + final String dictFile = String.format("%s/DE-EN_chemnitz_enwiktionary.quickdic", OUTPUTS); DictionaryBuilder.main(new String[] { "--dictOut=" + dictFile, diff --git a/src/com/hughes/android/dictionary/parser/wiktionary/WiktionaryLangs.java b/src/com/hughes/android/dictionary/parser/wiktionary/WiktionaryLangs.java index 0fbe21e..83fa0a1 100644 --- a/src/com/hughes/android/dictionary/parser/wiktionary/WiktionaryLangs.java +++ b/src/com/hughes/android/dictionary/parser/wiktionary/WiktionaryLangs.java @@ -158,9 +158,8 @@ public class WiktionaryLangs { isoCodeToWikiName.put("PL", Pattern.quote("{{-pl-}}")); isoCodeToWikiName.put("NL", Pattern.quote("{{-nl-}}")); isoCodeToWikiName.put("LV", Pattern.quote("{{-lv-}}")); - isoCodeToWikiName.put("LV", Pattern.quote("{{-la-}}")); + isoCodeToWikiName.put("LA", Pattern.quote("{{-la-}}")); isoCodeToWikiName.put("HU", Pattern.quote("{{-hu-}}")); - isoCodeToWikiName.put("PL", Pattern.quote("{{-pl-}}")); isoCodeToWikiName.put("EL", Pattern.quote("{{-grc-}}")); isoCodeToWikiName.put("SV", Pattern.quote("{{-sv-}}")); diff --git a/todo.txt b/todo.txt index 1695d1a..6593b37 100644 --- a/todo.txt +++ b/todo.txt @@ -1,3 +1,6 @@ +change sorting to put your locale first +fix update screen. + download latest wiktionaries split DictionaryBuilderTest