From: Thad Hughes Date: Sat, 19 Feb 2011 21:00:52 +0000 (-0800) Subject: go X-Git-Url: http://gitweb.fperrin.net/?p=DictionaryPC.git;a=commitdiff_plain;h=9a0850fd39f27b5cf08dbf63510f523d8bceff5d go --- diff --git a/src/com/hughes/android/dictionary/engine/DictionaryBuilder.java b/src/com/hughes/android/dictionary/engine/DictionaryBuilder.java index 7ea0d91..a3a08c9 100644 --- a/src/com/hughes/android/dictionary/engine/DictionaryBuilder.java +++ b/src/com/hughes/android/dictionary/engine/DictionaryBuilder.java @@ -117,6 +117,9 @@ public class DictionaryBuilder { fatalError("Must specify human readable name for: " + prefix + "Name"); } + final EntrySource entrySource = new EntrySource(dictionaryBuilder.dictionary.sources.size(), inputName, dictionaryBuilder.dictionary.pairEntries.size()); + System.out.println(""); + String inputFormat = keyValueArgs.remove(prefix + "Format"); if ("dictcc".equals(inputFormat)) { new DictFileParser(charset, false, DictFileParser.TAB, null, dictionaryBuilder, dictionaryBuilder.indexBuilders.toArray(new IndexBuilder[0]), null).parseFile(file); @@ -135,7 +138,6 @@ public class DictionaryBuilder { fatalError("Invalid or missing input format: " + inputFormat); } - final EntrySource entrySource = new EntrySource(dictionaryBuilder.dictionary.sources.size(), inputName); dictionaryBuilder.dictionary.sources.add(entrySource); System.out.println("Done: " + file + "\n\n"); } diff --git a/src/com/hughes/android/dictionary/engine/DictionaryBuilderMain.java b/src/com/hughes/android/dictionary/engine/DictionaryBuilderMain.java index 9a60d37..7438dc2 100644 --- a/src/com/hughes/android/dictionary/engine/DictionaryBuilderMain.java +++ b/src/com/hughes/android/dictionary/engine/DictionaryBuilderMain.java @@ -22,7 +22,19 @@ public class DictionaryBuilderMain extends TestCase { public static void main(final String[] args) throws Exception { - + + DictionaryBuilder.main(new String[] { + "--dictOut=dictOutputs/DE-EN_chemnitz.quickdic", + "--lang1=DE", + "--lang2=EN", + "--dictInfo=@dictInputs/de-en_chemnitz.info", + + "--input1=dictInputs/de-en_chemnitz.txt", + "--input1Name=chemnitz", + "--input1Charset=UTF8", + "--input1Format=chemnitz", + }); + Lang[] langs1 = new Lang[] { new Lang("^English$", "EN"), new Lang("^German$", "DE"), @@ -118,7 +130,7 @@ public class DictionaryBuilderMain extends TestCase { } // langs2 } // langs1 - + DictionaryBuilder.main(new String[] { "--dictOut=dictOutputs/de-en_all.quickdic", "--lang1=DE", @@ -144,18 +156,6 @@ public class DictionaryBuilderMain extends TestCase { }); - DictionaryBuilder.main(new String[] { - "--dictOut=dictOutputs/de-en_chemnitz.quickdic", - "--lang1=DE", - "--lang2=EN", - "--dictInfo=@dictInputs/de-en_chemnitz.info", - - "--input1=dictInputs/de-en_chemnitz.txt", - "--input1Name=dictcc", - "--input1Charset=UTF8", - "--input1Format=chemnitz", - }); - } } diff --git a/src/com/hughes/android/dictionary/engine/DictionaryBuilderTest.java b/src/com/hughes/android/dictionary/engine/DictionaryBuilderTest.java index 0a1b479..c515294 100644 --- a/src/com/hughes/android/dictionary/engine/DictionaryBuilderTest.java +++ b/src/com/hughes/android/dictionary/engine/DictionaryBuilderTest.java @@ -54,7 +54,7 @@ public class DictionaryBuilderTest extends TestCase { "--dictInfo=@testdata/de-en_dictInfo.txt", "--input1=testdata/de-en_chemnitz_100", - "--input1Name=dictcc", + "--input1Name=chemnitz", "--input1Charset=UTF8", "--input1Format=chemnitz", diff --git a/src/com/hughes/android/dictionary/engine/DictionaryTest.java b/src/com/hughes/android/dictionary/engine/DictionaryTest.java index 87103eb..a462897 100644 --- a/src/com/hughes/android/dictionary/engine/DictionaryTest.java +++ b/src/com/hughes/android/dictionary/engine/DictionaryTest.java @@ -2,20 +2,38 @@ package com.hughes.android.dictionary.engine; import java.io.IOException; import java.io.RandomAccessFile; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.Collections; -import java.util.List; import java.util.concurrent.atomic.AtomicBoolean; import junit.framework.TestCase; import com.hughes.android.dictionary.engine.Index.IndexEntry; -import com.ibm.icu.text.Transliterator; +import com.hughes.android.dictionary.engine.PairEntry.Row; public class DictionaryTest extends TestCase { + + @Override + protected void setUp() { + while (!TransliteratorManager.init(null)) { + try { + Thread.sleep(10); + } catch (InterruptedException e) { + e.printStackTrace(); + } + } + } + + public void testEnItWiktionary() throws IOException { + final RandomAccessFile raf = new RandomAccessFile("dictOutputs/EN-IT_enwiktionary.quickdic", "r"); + final Dictionary dict = new Dictionary(raf); + final Index enIndex = dict.indices.get(0); + final PairEntry.Row row = (Row) enIndex.rows.get(2); + assertEquals("z", row.getRawText(false)); + + raf.close(); + } + public void testGermanMetadata() throws IOException { final RandomAccessFile raf = new RandomAccessFile("testdata/de-en.quickdic", "r"); final Dictionary dict = new Dictionary(raf); @@ -24,6 +42,12 @@ public class DictionaryTest extends TestCase { assertEquals("de", deIndex.shortName); assertEquals("de->en", deIndex.longName); + assertEquals(2, dict.sources.size()); + assertEquals("chemnitz", dict.sources.get(0).name); + assertEquals(0, dict.sources.get(0).pairEntryStart); + assertEquals("dictcc", dict.sources.get(1).name); + assertEquals(113, dict.sources.get(1).pairEntryStart); + raf.close(); } @@ -36,7 +60,7 @@ public class DictionaryTest extends TestCase { System.out.println("testing: " + indexEntry.token); final IndexEntry searchResult = deIndex.findInsertionPoint(indexEntry.token, new AtomicBoolean( false)); - assertEquals(indexEntry.token.toLowerCase(), searchResult.token.toLowerCase()); + assertEquals("Looked up: " + indexEntry.token, indexEntry.token.toLowerCase(), searchResult.token.toLowerCase()); } // TODO: maybe if user types capitalization, use it. @@ -108,102 +132,6 @@ public class DictionaryTest extends TestCase { raf.close(); } - public void testGermanSort() { - final Transliterator normalizer = Transliterator.createFromRules("", Language.de.getDefaultNormalizerRules(), Transliterator.FORWARD); - assertEquals("aüääss", normalizer.transform("aueAeAEß")); - final List words = Arrays.asList( - "er-ben", - "erben", - "Erben", - "Erbse", - "Erbsen", - "essen", - "Essen", - "Grosformat", - "Grosformats", - "Grossformat", - "Großformat", - "Grossformats", - "Großformats", - "Großpoo", - "Großpoos", - "Hörvermögen", - "Hörweite", - "hos", - "Höschen", - "Hostel", - "hulle", - "Hulle", - "huelle", - "Huelle", - "hülle", - "Hülle", - "Huellen", - "Hüllen", - "Hum" - ); - final NormalizeComparator comparator = new NormalizeComparator(normalizer, Language.de.getCollator()); - assertEquals(1, comparator.compare("hülle", "huelle")); - assertEquals(-1, comparator.compare("huelle", "hülle")); - - assertEquals(-1, comparator.compare("hülle", "Hülle")); - - assertEquals("hülle", normalizer.transform("Hülle")); - assertEquals("hulle", normalizer.transform("Hulle")); - - - final List sorted = new ArrayList(words); -// Collections.shuffle(shuffled, new Random(0)); - Collections.sort(sorted, comparator); - System.out.println(sorted.toString()); - for (int i = 0; i < words.size(); ++i) { - System.out.println(words.get(i) + "\t" + sorted.get(i)); - assertEquals(words.get(i), sorted.get(i)); - } - } - - public void testEnglishSort() { - final Transliterator normalizer = Transliterator.createFromRules("", Language.en.getDefaultNormalizerRules(), Transliterator.FORWARD); - - final List words = Arrays.asList( - "pre-print", - "preppie", - "preppy", - "preprocess"); - - final List sorted = new ArrayList(words); - final NormalizeComparator comparator = new NormalizeComparator(normalizer, Language.en.getCollator()); - Collections.sort(sorted, comparator); - for (int i = 0; i < words.size(); ++i) { - if (i > 0) { - assertTrue(comparator.compare(words.get(i-1), words.get(i)) < 0); - } - System.out.println(words.get(i) + "\t" + sorted.get(i)); - assertEquals(words.get(i), sorted.get(i)); - } - - assertTrue(comparator.compare("pre-print", "preppy") < 0); - - } - - public void testLanguage() { - assertEquals(Language.de, Language.lookup("de")); - assertEquals(Language.en, Language.lookup("en")); - assertEquals("es", Language.lookup("es").getSymbol()); - } - - public void testTextNorm() { - //final Transliterator transliterator = Transliterator.getInstance("Any-Latin; Upper; Lower; 'oe' > 'o'; NFD; [:Nonspacing Mark:] Remove; NFC", Transliterator.FORWARD); - final Transliterator transliterator = Transliterator.createFromRules("", ":: Any-Latin; :: Upper; :: Lower; 'oe' > 'o'; :: NFD; :: [:Nonspacing Mark:] Remove; :: NFC ;", Transliterator.FORWARD); - assertEquals("hoschen", transliterator.transliterate("Höschen")); - assertEquals("hoschen", transliterator.transliterate("Hoeschen")); - assertEquals("grosspoo", transliterator.transliterate("Großpoo")); - - assertEquals("kyanpasu", transliterator.transliterate("キャンパス")); - assertEquals("alphabetikos katalogos", transliterator.transliterate("Αλφαβητικός Κατάλογος")); - assertEquals("biologiceskom", transliterator.transliterate("биологическом")); - } - public void testChemnitz() throws IOException { final RandomAccessFile raf = new RandomAccessFile("dictOutputs/de-en_chemnitz.quickdic", "r"); final Dictionary dict = new Dictionary(raf); @@ -215,4 +143,5 @@ public class DictionaryTest extends TestCase { raf.close(); } + } diff --git a/src/com/hughes/android/dictionary/engine/IndexBuilder.java b/src/com/hughes/android/dictionary/engine/IndexBuilder.java index 9721fd3..4f64fa2 100644 --- a/src/com/hughes/android/dictionary/engine/IndexBuilder.java +++ b/src/com/hughes/android/dictionary/engine/IndexBuilder.java @@ -24,7 +24,7 @@ public class IndexBuilder { IndexBuilder(final DictionaryBuilder dictionaryBuilder, final String shortName, final String longName, final Language language, final String normalizerRules, final boolean swapPairEntries) { this.dictionaryBuilder = dictionaryBuilder; index = new Index(dictionaryBuilder.dictionary, shortName, longName, language, normalizerRules, swapPairEntries); - tokenToData = new TreeMap(new NormalizeComparator(index.normalizer, language.collator)); + tokenToData = new TreeMap(new NormalizeComparator(index.normalizer(), language.getCollator())); } public void build() { @@ -50,7 +50,8 @@ public class IndexBuilder { } } } - index.sortedIndexEntries.add(new Index.IndexEntry(tokenData.token, startRow, numRows)); + index.sortedIndexEntries.add(new Index.IndexEntry(tokenData.token, index + .normalizer().transliterate(tokenData.token), startRow, numRows)); } final List sortedEntries = new ArrayList(index.sortedIndexEntries); diff --git a/src/com/hughes/android/dictionary/parser/DictFileParser.java b/src/com/hughes/android/dictionary/parser/DictFileParser.java index 2f53fdc..52a6567 100644 --- a/src/com/hughes/android/dictionary/parser/DictFileParser.java +++ b/src/com/hughes/android/dictionary/parser/DictFileParser.java @@ -102,7 +102,7 @@ public class DictFileParser { fields[0] = fields[1]; fields[1] = temp; } - + final String[][] subfields = new String[2][]; if (subfieldSplit != null) { subfields[0] = subfieldSplit.split(fields[0]); @@ -116,13 +116,12 @@ public class DictFileParser { subfields[1] = new String[] { fields[1] }; } - final Pair[] pairs = new Pair[subfields[0].length]; - for (int i = 0; i < pairs.length; ++i) { + final PairEntry pairEntry = new PairEntry(); + for (int i = 0; i < subfields[0].length; ++i) { subfields[0][i] = subfields[0][i].trim(); subfields[1][i] = subfields[1][i].trim(); - pairs[i] = new Pair(subfields[0][i], subfields[1][i]); + pairEntry.pairs.add(new Pair(subfields[0][i], subfields[1][i])); } - final PairEntry pairEntry = new PairEntry(pairs); final EntryData entryData = new EntryData(dictBuilder.dictionary.pairEntries.size(), pairEntry); dictBuilder.dictionary.pairEntries.add(pairEntry); diff --git a/src/com/hughes/android/dictionary/parser/EnWiktionaryXmlParser.java b/src/com/hughes/android/dictionary/parser/EnWiktionaryXmlParser.java index d5b9006..761c5dc 100644 --- a/src/com/hughes/android/dictionary/parser/EnWiktionaryXmlParser.java +++ b/src/com/hughes/android/dictionary/parser/EnWiktionaryXmlParser.java @@ -201,6 +201,7 @@ public class EnWiktionaryXmlParser extends org.xml.sax.helpers.DefaultHandler im namedArgs.remove("lang"); namedArgs.remove("nocat"); + namedArgs.remove("nocap"); namedArgs.remove("sc"); // Pronunciation @@ -212,7 +213,7 @@ public class EnWiktionaryXmlParser extends org.xml.sax.helpers.DefaultHandler im return; } - if (name.equals("IPA") || name.equals("SAMPA") || name.equals("X-SAMPA")|| name.equals("enPR")) { + if (name.equals("IPA") || name.equals("SAMPA") || name.equals("X-SAMPA") || name.equals("enPR")) { namedArgs.remove("lang"); for (int i = 0; i < 100 && !namedArgs.isEmpty(); ++i) { final String pron = namedArgs.remove("" + i); @@ -323,7 +324,7 @@ public class EnWiktionaryXmlParser extends org.xml.sax.helpers.DefaultHandler im assert positionalArgs.size() >= 1 && namedArgs.isEmpty() : positionalArgs.toString() + namedArgs + title; if (currentPartOfSpeech == null) { - assert !currentWord.partsOfSpeech.isEmpty() : title; + assert currentWord != null && !currentWord.partsOfSpeech.isEmpty() : title; System.err.println("Assuming last part of speech for non-nested translation section: " + title); currentPartOfSpeech = ListUtil.getLast(currentWord.partsOfSpeech); } diff --git a/src/com/hughes/android/dictionary/parser/WikiWord.java b/src/com/hughes/android/dictionary/parser/WikiWord.java index c677d13..5285c0e 100644 --- a/src/com/hughes/android/dictionary/parser/WikiWord.java +++ b/src/com/hughes/android/dictionary/parser/WikiWord.java @@ -125,6 +125,24 @@ public class WikiWord { for (final WikiWord.PartOfSpeech partOfSpeech : partsOfSpeech) { partOfSpeechToQuickDic(dictBuilder, enIndexBuilder, partOfSpeech); } // PartOfSpeech + + // Pronunciation. + if (index != -1) { + final PairEntry pronEntry = new PairEntry(); + for (final Map.Entry accentToPron : accentToPronunciation.entrySet()) { + String accent = accentToPron.getKey(); + if (accent.length() > 0) { + accent = accent + ": "; + } + pronEntry.pairs.add(new Pair(accent + accentToPron.getValue(), "", index != 0)); + } + if (pronEntry.pairs.size() > 0) { + final EntryData entryData = new EntryData(dictBuilder.dictionary.pairEntries.size(), pronEntry); + dictBuilder.dictionary.pairEntries.add(pronEntry); + final Set tokens = DictFileParser.tokenize(title, DictFileParser.NON_CHAR); + dictBuilder.indexBuilders.get(index).addEntryWithTokens(entryData, tokens, EntryTypeName.WIKTIONARY_PRONUNCIATION); + } + } } @@ -145,12 +163,12 @@ public class WikiWord { } } - if (index != -1) { final boolean formOfSwap = index != 0; for (final FormOf formOf : partOfSpeech.formOfs) { final Pair pair = new Pair(title + ": " + formOf.grammarForm + ": " + formOf.target, "", formOfSwap); - final PairEntry pairEntry = new PairEntry(new Pair[] {pair}); + final PairEntry pairEntry = new PairEntry(); + pairEntry.pairs.add(pair); final EntryData entryData = new EntryData(dictBuilder.dictionary.pairEntries.size(), pairEntry); dictBuilder.dictionary.pairEntries.add(pairEntry); @@ -167,8 +185,9 @@ public class WikiWord { // Meanings. for (final Meaning meaning : partOfSpeech.meanings) { - final List pairs = new ArrayList(); - + final PairEntry pairEntry = new PairEntry(); + final List pairs = pairEntry.pairs; + final List> exampleTokens = new ArrayList>(); exampleTokens.add(new LinkedHashSet()); exampleTokens.add(new LinkedHashSet()); @@ -200,7 +219,6 @@ public class WikiWord { } // Create EntryData with the PairEntry. - final PairEntry pairEntry = new PairEntry(pairs.toArray(new Pair[0])); final EntryData entryData = new EntryData(dictBuilder.dictionary.pairEntries.size(), pairEntry); dictBuilder.dictionary.pairEntries.add(pairEntry); @@ -241,22 +259,12 @@ public class WikiWord { final String englishBase = title + " (" + partOfSpeech.name.toLowerCase() + "%s)"; - final StringBuilder englishPron = new StringBuilder(); - for (final Map.Entry accentToPron : accentToPronunciation.entrySet()) { - englishPron.append("\n"); - if (accentToPron.getKey().length() > 0) { - englishPron.append(accentToPron.getKey()).append(": "); - } - englishPron.append(accentToPron.getValue()); - } - for (final TranslationSense translationSense : partOfSpeech.translationSenses) { //System.out.println(" sense: " + translationSense.sense); if (translationSense.sense == null) { //System.err.println(" null sense: " + title); } String englishSense = String.format(englishBase, translationSense.sense != null ? (": " + translationSense.sense) : ""); - englishSense += englishPron.toString(); final StringBuilder[] sideBuilders = new StringBuilder[2]; final List>> sideTokens = new ArrayList>>(); @@ -290,7 +298,8 @@ public class WikiWord { // Construct the Translations-based QuickDic entry for this TranslationSense. if (sideBuilders[0].length() > 0 && sideBuilders[1].length() > 0) { final Pair pair = new Pair(sideBuilders[0].toString(), sideBuilders[1].toString()); - final PairEntry pairEntry = new PairEntry(new Pair[] { pair }); + final PairEntry pairEntry = new PairEntry(); + pairEntry.pairs.add(pair); final EntryData entryData = new EntryData(dictBuilder.dictionary.pairEntries.size(), pairEntry); dictBuilder.dictionary.pairEntries.add(pairEntry);