X-Git-Url: http://gitweb.fperrin.net/?a=blobdiff_plain;f=src%2Fcom%2Fhughes%2Fandroid%2Fdictionary%2Fengine%2FDictionaryTest.java;h=3de220b0db7de98609172b8aa543904142fdad54;hb=481738c6610def8dcd0378f3c419fa8c0a0f6cf3;hp=388e71d5a069ec066b4ea0c9e5b50c8321e76dc2;hpb=a91fcd717450fa7cbbc2b260c265d0a66a232dc6;p=DictionaryPC.git

diff --git a/src/com/hughes/android/dictionary/engine/DictionaryTest.java b/src/com/hughes/android/dictionary/engine/DictionaryTest.java
index 388e71d..3de220b 100644
--- a/src/com/hughes/android/dictionary/engine/DictionaryTest.java
+++ b/src/com/hughes/android/dictionary/engine/DictionaryTest.java
@@ -1,34 +1,129 @@
+// Copyright 2011 Google Inc. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
 package com.hughes.android.dictionary.engine;
 
 import java.io.IOException;
 import java.io.RandomAccessFile;
-import java.util.ArrayList;
 import java.util.Arrays;
-import java.util.Collections;
 import java.util.List;
+import java.util.Random;
 import java.util.concurrent.atomic.AtomicBoolean;
 
 import junit.framework.TestCase;
 
 import com.hughes.android.dictionary.engine.Index.IndexEntry;
-import com.ibm.icu.text.Transliterator;
+import com.hughes.util.CollectionUtil;
 
 
 public class DictionaryTest extends TestCase {
+  
+  static final String TEST_OUTPUTS = com.hughes.android.dictionary.engine.DictionaryBuilderTest.TEST_OUTPUTS;
+  public static final String OUTPUTS = "data/outputs/";
+
+  @Override
+  protected void setUp() {
+    while (!TransliteratorManager.init(null)) {
+      try {
+        Thread.sleep(10);
+      } catch (InterruptedException e) {
+        e.printStackTrace();
+      }
+    }
+  }
+  
+  public void testURLFormatting() {
+  }
+
+  public void testEnItWiktionary() throws IOException {
+    final RandomAccessFile raf = new RandomAccessFile(OUTPUTS + "EN-IT.quickdic", "r");
+    final Dictionary dict = new Dictionary(raf);
+    final Index enIndex = dict.indices.get(0);
     
+    final RowBase row = enIndex.rows.get(4);
+    assertEquals("-ical", row.getRawText(false));
+    
+    final Index itIndex = dict.indices.get(1);
+    {
+    final List<RowBase> rows = itIndex.multiWordSearch("come mai", Arrays.asList("come", "mai"), new AtomicBoolean(false));
+    System.out.println(CollectionUtil.join(rows, "\n  "));
+    assertTrue(rows.toString(), rows.size() > 0);
+    assertTrue(rows.get(0).toString().startsWith("come mai@"));
+    assertTrue(rows.get(0) instanceof TokenRow);
+    assertTrue(!((TokenRow)rows.get(0)).getIndexEntry().htmlEntries.isEmpty());
+    }
+
+    {
+        final IndexEntry searchResult = itIndex.findInsertionPoint("azzurro", new AtomicBoolean(
+                false));
+        HtmlEntry htmlEntry = searchResult.htmlEntries.get(0);
+        System.out.println("azzurro:\n" + htmlEntry.getHtml());
+    }
+
+    raf.close();
+  }
+
+  public void testFr() throws IOException {
+      final RandomAccessFile raf = new RandomAccessFile(OUTPUTS + "FR.quickdic", "r");
+      final Dictionary dict = new Dictionary(raf);
+      final Index frIndex = dict.indices.get(0);
+      
+      // Now they're all cached, we shouldn't have to search.
+//      for (final IndexEntry indexEntry : frIndex.sortedIndexEntries) {
+//          System.out.println(indexEntry.token);
+//      }
+
+      raf.close();
+  }
+
+  
+  public void testDeEnWiktionary() throws IOException {
+      final RandomAccessFile raf = new RandomAccessFile(OUTPUTS + "DE-EN.quickdic", "r");
+      final Dictionary dict = new Dictionary(raf);
+            
+      final Index deIndex = dict.indices.get(0);
+
+      {
+          final IndexEntry searchResult = deIndex.findInsertionPoint("rot", new AtomicBoolean(
+                  false));
+          HtmlEntry htmlEntry = searchResult.htmlEntries.get(0);
+          System.out.println("rot:\n" + htmlEntry.getHtml());
+      }
+
+      raf.close();
+    }
+
   public void testGermanMetadata() throws IOException {
-    final RandomAccessFile raf = new RandomAccessFile("testdata/de-en.quickdic", "r");
+    final RandomAccessFile raf = new RandomAccessFile(TEST_OUTPUTS + "de-en.quickdic", "r");
     final Dictionary dict = new Dictionary(raf);
     final Index deIndex = dict.indices.get(0);
     
-    assertEquals("de", deIndex.shortName);
-    assertEquals("de->en", deIndex.longName);
+    assertEquals("DE", deIndex.shortName);
+    assertEquals("DE->EN", deIndex.longName);
+    
+    assertEquals(2, dict.sources.size());
+    assertEquals("chemnitz", dict.sources.get(0).name);
+    assertEquals("dictcc", dict.sources.get(1).name);
+    
+    assertEquals("dictcc", dict.pairEntries.get(0).entrySource.name);
+    assertEquals("chemnitz", dict.pairEntries.get(1).entrySource.name);
     
     raf.close();
   }
   
   public void testGermanIndex() throws IOException {
-    final RandomAccessFile raf = new RandomAccessFile("testdata/de-en.dict", "r");
+    final RandomAccessFile raf = new RandomAccessFile(TEST_OUTPUTS + "de-en.quickdic", "r");
     final Dictionary dict = new Dictionary(raf);
     final Index deIndex = dict.indices.get(0);
     
@@ -36,7 +131,7 @@ public class DictionaryTest extends TestCase {
       System.out.println("testing: " + indexEntry.token);
       final IndexEntry searchResult = deIndex.findInsertionPoint(indexEntry.token, new AtomicBoolean(
           false));
-      assertEquals(indexEntry.token.toLowerCase(), searchResult.token.toLowerCase());
+      assertEquals("Looked up: " + indexEntry.token, indexEntry.token.toLowerCase(), searchResult.token.toLowerCase());
     }
 
     // TODO: maybe if user types capitalization, use it.
@@ -78,7 +173,7 @@ public class DictionaryTest extends TestCase {
   }
 
   public void testGermanTokenRows() throws IOException {
-    final RandomAccessFile raf = new RandomAccessFile("testdata/de-en.quickdic", "r");
+    final RandomAccessFile raf = new RandomAccessFile(TEST_OUTPUTS + "de-en.quickdic", "r");
     final Dictionary dict = new Dictionary(raf);
     final Index deIndex = dict.indices.get(0);
     
@@ -108,111 +203,186 @@ public class DictionaryTest extends TestCase {
     raf.close();
   }
   
-  public void testGermanSort() {
-    final Transliterator normalizer = Transliterator.createFromRules("", Language.de.getDefaultNormalizerRules(), Transliterator.FORWARD);
-    assertEquals("aüääss", normalizer.transform("aueAeAEß"));
-    final List<String> words = Arrays.asList(
-        "er-ben",
-        "erben",
-        "Erben",
-        "Erbse",
-        "Erbsen",
-        "essen",
-        "Essen",
-        "Grosformat",
-        "Grosformats",
-        "Grossformat",
-        "Großformat",
-        "Grossformats",
-        "Großformats",
-        "Großpoo",
-        "Großpoos",
-        "Hörvermögen",
-        "Hörweite",
-        "hos",
-        "Höschen",
-        "Hostel",
-        "hulle",
-        "Hulle",
-        "huelle",
-        "Huelle",
-        "hülle",
-        "Hülle",
-        "Huellen",
-        "Hüllen",
-        "Hum"
-        );
-    final NormalizeComparator comparator = new NormalizeComparator(normalizer, Language.de.getCollator());
-    assertEquals(1, comparator.compare("hülle", "huelle"));
-    assertEquals(-1, comparator.compare("huelle", "hülle"));
-    
-    assertEquals(-1, comparator.compare("hülle", "Hülle"));
-    
-    assertEquals("hülle", normalizer.transform("Hülle"));
-    assertEquals("hulle", normalizer.transform("Hulle"));
-
-    
-    final List<String> sorted = new ArrayList<String>(words);
-//    Collections.shuffle(shuffled, new Random(0));
-    Collections.sort(sorted, comparator);
-    System.out.println(sorted.toString());
-    for (int i = 0; i < words.size(); ++i) {
-      System.out.println(words.get(i) + "\t" + sorted.get(i));
-      assertEquals(words.get(i), sorted.get(i));
-    }
-  }
-
-  public void testEnglishSort() {
-    final Transliterator normalizer = Transliterator.createFromRules("", Language.en.getDefaultNormalizerRules(), Transliterator.FORWARD);
-
-    final List<String> words = Arrays.asList(
-        "pre-print", 
-        "preppie", 
-        "preppy",
-        "preprocess");
-    
-    final List<String> sorted = new ArrayList<String>(words);
-    final NormalizeComparator comparator = new NormalizeComparator(normalizer, Language.en.getCollator());
-    Collections.sort(sorted, comparator);
-    for (int i = 0; i < words.size(); ++i) {
-      if (i > 0) {
-        assertTrue(comparator.compare(words.get(i-1), words.get(i)) < 0);
-      }
-      System.out.println(words.get(i) + "\t" + sorted.get(i));
-      assertEquals(words.get(i), sorted.get(i));
-    }
+  public void testChemnitz() throws IOException {
+    final RandomAccessFile raf = new RandomAccessFile(TEST_OUTPUTS + "de-en.quickdic", "r");
+    final Dictionary dict = new Dictionary(raf);
+    final Index deIndex = dict.indices.get(0);
+    
+    assertSearchResult("Höschen", "Hos", deIndex.findInsertionPoint("Hos", new AtomicBoolean(false)));
+    assertSearchResult("Höschen", "hos", deIndex.findInsertionPoint("hos", new AtomicBoolean(false)));
     
-    assertTrue(comparator.compare("pre-print", "preppy") < 0);
+    raf.close();
+  }
+
+  public void testMultiSearch() throws IOException {
+    final RandomAccessFile raf = new RandomAccessFile(TEST_OUTPUTS + "de-en.quickdic", "r");
+    final Dictionary dict = new Dictionary(raf);
+    final Index deIndex = dict.indices.get(0);
 
+    {
+    final List<RowBase> rows = deIndex.multiWordSearch("aaa aaab", Arrays.asList("aaa", "aaab"), new AtomicBoolean(false));
+    System.out.println(CollectionUtil.join(rows, "\n  "));
+    assertTrue(rows.toString(), rows.size() > 0);
+    }
+    
+    raf.close();
   }
   
-  public void testLanguage() {
-    assertEquals(Language.de, Language.lookup("de"));
-    assertEquals(Language.en, Language.lookup("en"));
-    assertEquals("es", Language.lookup("es").getSymbol());
+  public void testMultiSearchIt() throws IOException {
+      final RandomAccessFile raf = new RandomAccessFile(OUTPUTS + "IT.quickdic", "r");
+      final Dictionary dict = new Dictionary(raf);
+      final Index index = dict.indices.get(0);
+
+      {
+      final List<RowBase> rows = index.multiWordSearch("fare centro", 
+              Arrays.asList("fare", "centro"), new AtomicBoolean(false));
+      System.out.println(CollectionUtil.join(rows, "\n  "));
+      assertTrue(rows.toString(), rows.size() > 0);
+      assertTrue(rows.get(0).toString().startsWith("fare centro@"));
+      }
   }
 
-  public void testTextNorm() {
-    //final Transliterator transliterator = Transliterator.getInstance("Any-Latin; Upper; Lower; 'oe' > 'o'; NFD; [:Nonspacing Mark:] Remove; NFC", Transliterator.FORWARD);
-    final Transliterator transliterator = Transliterator.createFromRules("", ":: Any-Latin; :: Upper; :: Lower; 'oe' > 'o'; :: NFD; :: [:Nonspacing Mark:] Remove; :: NFC ;", Transliterator.FORWARD);
-    assertEquals("hoschen", transliterator.transliterate("Höschen"));
-    assertEquals("hoschen", transliterator.transliterate("Hoeschen"));
-    assertEquals("grosspoo", transliterator.transliterate("Großpoo"));
+  public void testMultiSearchDeBig() throws IOException {
+    final RandomAccessFile raf = new RandomAccessFile(OUTPUTS + "DE-EN.quickdic", "r");
+    final Dictionary dict = new Dictionary(raf);
+    final Index enIndex = dict.indices.get(1);
 
-    assertEquals("kyanpasu", transliterator.transliterate("キャンパス"));
-    assertEquals("alphabetikos katalogos", transliterator.transliterate("Αλφαβητικός Κατάλογος"));
-    assertEquals("biologiceskom", transliterator.transliterate("биологическом"));
+    {
+    final List<RowBase> rows = enIndex.multiWordSearch("train station", Arrays.asList("train", "station"), new AtomicBoolean(false));
+    System.out.println(CollectionUtil.join(rows, "\n  "));
+    assertTrue(rows.toString(), rows.size() > 0);
+    assertTrue(rows.get(0).toString().startsWith("train station@"));
+    }
+
+    {
+    final List<RowBase> rows = enIndex.multiWordSearch("a train station", Arrays.asList("a", "train", "station"), new AtomicBoolean(false));
+    System.out.println(CollectionUtil.join(rows, "\n  "));
+    assertTrue(rows.toString(), rows.size() > 0);
+    assertEquals("Bahnhofsuhr {{de-noun|g=f|plural=Bahnhofsuhren}}\tstation clock (at a train station)", rows.get(0).toString());
+    }
+
+    {
+    final List<RowBase> rows = enIndex.multiWordSearch("a station", Arrays.asList("a", "station"), new AtomicBoolean(false));
+    System.out.println(CollectionUtil.join(rows, "\n  "));
+    assertTrue(rows.toString(), rows.size() > 0);
+    assertEquals("Abfahrthalle {en-noun}\tDeparture room of a station.", rows.get(0).toString());
+    }
+
+    {
+    // Should print: Giving up, too many words with prefix: p
+    final List<RowBase> rows = enIndex.multiWordSearch("p eat", Arrays.asList("p", "eat"), new AtomicBoolean(false));
+    System.out.println(CollectionUtil.join(rows, "\n  "));
+    assertTrue(rows.toString(), rows.size() > 0);
+    assertTrue(rows.toString().contains("verschlingen; verputzen\tto dispatch (eat)"));
+    }
+
+    {
+    // Should print: Giving up, too many words with prefix: p
+    final List<RowBase> rows = enIndex.multiWordSearch("p p", Arrays.asList("p", "p"), new AtomicBoolean(false));
+    assertTrue(rows.size() >= 1000);
+    }
+
+    {
+    // Should print: Giving up, too many words with prefix: a
+    final List<RowBase> rows = enIndex.multiWordSearch("a a", Arrays.asList("a", "a"), new AtomicBoolean(false));
+    assertTrue(rows.size() >= 1000);
+    }
+
+    {
+    // Should print: Giving up, too many words with prefix: a
+    final List<RowBase> rows = enIndex.multiWordSearch("b ba", Arrays.asList("b", "ba"), new AtomicBoolean(false));
+    assertTrue(rows.size() >= 1000);
+    }
+
+    {
+    // Should print: Giving up, too many words with prefix: a
+    final List<RowBase> rows = enIndex.multiWordSearch("b ba", Arrays.asList("b", "ba"), new AtomicBoolean(false));
+    assertTrue(rows.size() >= 1000);
+    }
+
+    raf.close();
   }
 
-  public void testChemnitz() throws IOException {
-    final RandomAccessFile raf = new RandomAccessFile("dictOutputs/de-en_chemnitz.quickdic", "r");
+  public void testMultiSearchBigAF() throws IOException {
+    final RandomAccessFile raf = new RandomAccessFile(OUTPUTS + "EN-AF.quickdic", "r");
     final Dictionary dict = new Dictionary(raf);
-    final Index deIndex = dict.indices.get(0);
+    final Index enIndex = dict.indices.get(0);
+
+    {
+    final List<RowBase> rows = enIndex.multiWordSearch("pig eats", Arrays.asList("pig", "eats"), new AtomicBoolean(false));
+    System.out.println(CollectionUtil.join(rows, "\n  "));
+    assertTrue(rows.toString(), rows.size() > 0);
+    assertEquals("pig (someone who overeats or eats rapidly) (noun)\tvark", rows.get(0).toString());
+    }
+
+    {
+    final List<RowBase> rows = enIndex.multiWordSearch("pig eat", Arrays.asList("pig", "eat"), new AtomicBoolean(false));
+    System.out.println(CollectionUtil.join(rows, "\n  "));
+    assertTrue(rows.toString(), rows.size() > 0);
+    assertEquals("pig (someone who overeats or eats rapidly) (noun)\tvark", rows.get(0).toString());
+    }
+
+    {
+    final List<RowBase> rows = enIndex.multiWordSearch("pi ea", Arrays.asList("pi", "ea"), new AtomicBoolean(false));
+    System.out.println(CollectionUtil.join(rows, "\n  "));
+    assertTrue(rows.toString(), rows.size() > 0);
+    assertTrue(rows.toString().contains("pig (someone who overeats or eats rapidly) (noun)\tvark"));
+    }
+
+    {
+    final List<RowBase> rows = enIndex.multiWordSearch("p eat", Arrays.asList("p", "eat"), new AtomicBoolean(false));
+    System.out.println(CollectionUtil.join(rows, "\n  "));
+    assertTrue(rows.toString(), rows.size() > 0);
+    assertTrue(rows.toString().contains("pig (someone who overeats or eats rapidly) (noun)\tvark"));
+    }
+
     
-    assertSearchResult("Höschen", "Hos", deIndex.findInsertionPoint("Hos", new AtomicBoolean(false)));
-    assertSearchResult("Höschen", "hos", deIndex.findInsertionPoint("hos", new AtomicBoolean(false)));
+    raf.close();
+  }
+
+
+  public void testExactSearch() throws IOException {
+    final RandomAccessFile raf = new RandomAccessFile(OUTPUTS + "EN-cmn.quickdic", "r");
+    final Dictionary dict = new Dictionary(raf);
+    final Index cmnIndex = dict.indices.get(1);
+
+    final Random random = new Random(10);
+    
+    for (int i = 0; i < 1000; ++i) {
+      final int ii = random.nextInt(cmnIndex.sortedIndexEntries.size());
+      final IndexEntry indexEntry = cmnIndex.sortedIndexEntries.get(ii);
+      final IndexEntry found = cmnIndex.findExact(indexEntry.token);
+      assertNotNull(found);
+      assertEquals(indexEntry.token, found.token);
+      assertEquals(indexEntry, found);  // Test of caching....
+    }
+    
+    raf.close();
+  }
+
+  public void testThai() throws IOException {
+    final RandomAccessFile raf = new RandomAccessFile(OUTPUTS + "EN-TH.quickdic", "r");
+    final Dictionary dict = new Dictionary(raf);
+    final Index thIndex = dict.indices.get(1);
 
+    final IndexEntry entry = thIndex.findInsertionPoint("ดี", new AtomicBoolean(false));
+    assertEquals("di", entry.token);
+    
     raf.close();
   }
 
+  public void testNorwegian() throws IOException {
+      final RandomAccessFile raf = new RandomAccessFile(OUTPUTS + "EN-NL.quickdic", "r");
+      final Dictionary dict = new Dictionary(raf);
+      final Index nlIndex = dict.indices.get(1);
+
+      IndexEntry entry = nlIndex.findInsertionPoint("Xhosa", new AtomicBoolean(false));
+      assertEquals("Xhosa", entry.token);
+
+      entry = nlIndex.findInsertionPoint("Zyne", new AtomicBoolean(false));
+      assertEquals("Zyne", entry.token);
+
+      raf.close();
+  }
+
 }