src/com/hughes/android/dictionary/engine/DictionaryTest.java

   1 // Copyright 2011 Google Inc. All Rights Reserved.
   2 //
   3 // Licensed under the Apache License, Version 2.0 (the "License");
   4 // you may not use this file except in compliance with the License.
   5 // You may obtain a copy of the License at
   6 //
   7 //     http://www.apache.org/licenses/LICENSE-2.0
   8 //
   9 // Unless required by applicable law or agreed to in writing, software
  10 // distributed under the License is distributed on an "AS IS" BASIS,
  11 // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  12 // See the License for the specific language governing permissions and
  13 // limitations under the License.
  14
  15 package com.hughes.android.dictionary.engine;
  16
  17 import java.io.IOException;
  18 import java.io.RandomAccessFile;
  19 import java.util.Arrays;
  20 import java.util.List;
  21 import java.util.Random;
  22 import java.util.concurrent.atomic.AtomicBoolean;
  23
  24 import com.hughes.android.dictionary.engine.Index.IndexEntry;
  25 import com.hughes.util.CollectionUtil;
  26
  27 import junit.framework.TestCase;
  28
  29
  30 public class DictionaryTest extends TestCase {
  31
  32     static final String TEST_OUTPUTS = com.hughes.android.dictionary.engine.DictionaryBuilderTest.TEST_OUTPUTS;
  33     public static final String OUTPUTS = "data/outputs/";
  34
  35     @Override
  36     protected void setUp() {
  37         while (!TransliteratorManager.init(null, null)) {
  38             try {
  39                 Thread.sleep(10);
  40             } catch (InterruptedException e) {
  41                 e.printStackTrace();
  42             }
  43         }
  44     }
  45
  46     public void testURLFormatting() {
  47     }
  48
  49     public void testEnItWiktionary() throws IOException {
  50         final RandomAccessFile raf = new RandomAccessFile(OUTPUTS + "EN-IT.quickdic", "r");
  51         final Dictionary dict = new Dictionary(raf.getChannel());
  52         final Index enIndex = dict.indices.get(0);
  53
  54         final RowBase row = enIndex.rows.get(4);
  55         assertEquals("-ical", row.getRawText(false));
  56
  57         final Index itIndex = dict.indices.get(1);
  58         {
  59             final List<RowBase> rows = itIndex.multiWordSearch("come mai", Arrays.asList("come", "mai"), new AtomicBoolean(false));
  60             System.out.println(CollectionUtil.join(rows, "\n  "));
  61             assertTrue(rows.toString(), rows.size() > 0);
  62             assertTrue(rows.get(0).toString().startsWith("come mai@"));
  63             assertTrue(rows.get(0) instanceof TokenRow);
  64             assertFalse(((TokenRow) rows.get(0)).getIndexEntry().htmlEntries.isEmpty());
  65         }
  66
  67         {
  68             final List<RowBase> rows = itIndex.multiWordSearch("buon g", Arrays.asList("buon", "g"), new AtomicBoolean(false));
  69             System.out.println(CollectionUtil.join(rows, "\n  "));
  70             assertTrue(rows.toString(), rows.size() > 0);
  71             assertTrue(rows.get(0).toString().startsWith("buon giorno@"));
  72             assertTrue(rows.get(0) instanceof TokenRow);
  73             assertFalse(((TokenRow) rows.get(0)).getIndexEntry().htmlEntries.isEmpty());
  74         }
  75
  76         {
  77             final IndexEntry searchResult = itIndex.findInsertionPoint("azzurro", new AtomicBoolean(
  78                                                 false));
  79             HtmlEntry htmlEntry = searchResult.htmlEntries.get(0);
  80             System.out.println("azzurro:\n" + htmlEntry.getHtml());
  81         }
  82
  83         raf.close();
  84     }
  85
  86 //  public void testFr() throws IOException {
  87 //      final RandomAccessFile raf = new RandomAccessFile(OUTPUTS + "FR.quickdic", "r");
  88 //      final Dictionary dict = new Dictionary(raf);
  89 //      final Index frIndex = dict.indices.get(0);
  90 //
  91 //      // Now they're all cached, we shouldn't have to search.
  92 //      for (final IndexEntry indexEntry : frIndex.sortedIndexEntries) {
  93 //          System.out.println(indexEntry.token);
  94 //      }
  95 //
  96 //      raf.close();
  97 //  }
  98
  99
 100     public void testDeEnWiktionary() throws IOException {
 101         final RandomAccessFile raf = new RandomAccessFile(OUTPUTS + "DE-EN.quickdic", "r");
 102         final Dictionary dict = new Dictionary(raf.getChannel());
 103
 104         final Index deIndex = dict.indices.get(0);
 105
 106         {
 107             final IndexEntry searchResult = deIndex.findInsertionPoint("rot", new AtomicBoolean(
 108                                                 false));
 109             HtmlEntry htmlEntry = searchResult.htmlEntries.get(0);
 110             System.out.println("rot:\n" + htmlEntry.getHtml());
 111         }
 112
 113         raf.close();
 114     }
 115
 116     public void testGermanMetadata() throws IOException {
 117         final RandomAccessFile raf = new RandomAccessFile(TEST_OUTPUTS + "de-en.quickdic", "r");
 118         final Dictionary dict = new Dictionary(raf.getChannel());
 119         final Index deIndex = dict.indices.get(0);
 120
 121         assertEquals("DE", deIndex.shortName);
 122         assertEquals("DE->EN", deIndex.longName);
 123
 124         assertEquals(2, dict.sources.size());
 125         assertEquals("chemnitz", dict.sources.get(0).name);
 126         assertEquals("dictcc", dict.sources.get(1).name);
 127
 128         assertEquals("dictcc", dict.pairEntries.get(0).entrySource.name);
 129         assertEquals("chemnitz", dict.pairEntries.get(1).entrySource.name);
 130
 131         raf.close();
 132     }
 133
 134     public void testGermanIndex() throws IOException {
 135         final RandomAccessFile raf = new RandomAccessFile(TEST_OUTPUTS + "de-en.quickdic", "r");
 136         final Dictionary dict = new Dictionary(raf.getChannel());
 137         final Index deIndex = dict.indices.get(0);
 138
 139         for (final Index.IndexEntry indexEntry : deIndex.sortedIndexEntries) {
 140             System.out.println("testing: " + indexEntry.token);
 141             final IndexEntry searchResult = deIndex.findInsertionPoint(indexEntry.token, new AtomicBoolean(
 142                                                 false));
 143             assertEquals("Looked up: " + indexEntry.token, indexEntry.token.toLowerCase(), searchResult.token.toLowerCase());
 144         }
 145
 146         // TODO: maybe if user types capitalization, use it.
 147         assertSearchResult("aaac", "aaac", deIndex.findInsertionPoint("aaac", new AtomicBoolean(false)));
 148         assertSearchResult("aaac", "aaac", deIndex.findInsertionPoint("AAAC", new AtomicBoolean(false)));
 149         assertSearchResult("aaac", "aaac", deIndex.findInsertionPoint("AAAc", new AtomicBoolean(false)));
 150         assertSearchResult("aaac", "aaac", deIndex.findInsertionPoint("aAac", new AtomicBoolean(false)));
 151
 152         // Before the beginning.
 153         assertSearchResult("40", "40" /* special case */, deIndex.findInsertionPoint("", new AtomicBoolean(false)));
 154         assertSearchResult("40", "40" /* special case */, deIndex.findInsertionPoint("__", new AtomicBoolean(false)));
 155
 156         // After the end.
 157         assertSearchResult("Zweckorientiertheit", "zählen", deIndex.findInsertionPoint("ZZZZZ", new AtomicBoolean(false)));
 158
 159         assertSearchResult("ab", "aaac", deIndex.findInsertionPoint("aaaca", new AtomicBoolean(false)));
 160         assertSearchResult("machen", "machen", deIndex.findInsertionPoint("m", new AtomicBoolean(false)));
 161         assertSearchResult("machen", "machen", deIndex.findInsertionPoint("macdddd", new AtomicBoolean(false)));
 162
 163
 164         assertSearchResult("überprüfe", "überprüfe", deIndex.findInsertionPoint("ueberprüfe", new AtomicBoolean(false)));
 165         assertSearchResult("überprüfe", "überprüfe", deIndex.findInsertionPoint("ueberpruefe", new AtomicBoolean(false)));
 166
 167         assertSearchResult("überprüfe", "überprüfe", deIndex.findInsertionPoint("ueberpBLEH", new AtomicBoolean(false)));
 168         assertSearchResult("überprüfe", "überprüfe", deIndex.findInsertionPoint("überprBLEH", new AtomicBoolean(false)));
 169
 170         assertSearchResult("überprüfen", "überprüfe", deIndex.findInsertionPoint("überprüfeBLEH", new AtomicBoolean(false)));
 171
 172         // Check that search in lowercase works.
 173         assertSearchResult("Alibi", "Alibi", deIndex.findInsertionPoint("alib", new AtomicBoolean(false)));
 174         System.out.println(deIndex.findInsertionPoint("alib", new AtomicBoolean(false)));
 175
 176         raf.close();
 177     }
 178
 179     private void assertSearchResult(final String insertionPoint, final String longestPrefix,
 180                                     final IndexEntry actual) {
 181         assertEquals(insertionPoint, actual.token);
 182     }
 183
 184     public void testGermanTokenRows() throws IOException {
 185         final RandomAccessFile raf = new RandomAccessFile(TEST_OUTPUTS + "de-en.quickdic", "r");
 186         final Dictionary dict = new Dictionary(raf.getChannel());
 187         final Index deIndex = dict.indices.get(0);
 188
 189         // Pre-cache a few of these, just to make sure that's working.
 190         for (int i = 0; i < deIndex.rows.size(); i += 7) {
 191             deIndex.rows.get(i).getTokenRow(true);
 192         }
 193
 194         // Do the exhaustive searching.
 195         TokenRow lastTokenRow = null;
 196         for (final RowBase row : deIndex.rows) {
 197             if (row instanceof TokenRow) {
 198                 lastTokenRow = (TokenRow) row;
 199             }
 200             assertEquals(lastTokenRow, row.getTokenRow(true));
 201         }
 202
 203         // Now they're all cached, we shouldn't have to search.
 204         for (final RowBase row : deIndex.rows) {
 205             if (row instanceof TokenRow) {
 206                 lastTokenRow = (TokenRow) row;
 207             }
 208             // This will break if the Row cache isn't big enough.
 209             assertEquals(lastTokenRow, row.getTokenRow(false));
 210         }
 211
 212         raf.close();
 213     }
 214
 215     public void testChemnitz() throws IOException {
 216         final RandomAccessFile raf = new RandomAccessFile(TEST_OUTPUTS + "de-en.quickdic", "r");
 217         final Dictionary dict = new Dictionary(raf.getChannel());
 218         final Index deIndex = dict.indices.get(0);
 219
 220         assertSearchResult("Höschen", "Hos", deIndex.findInsertionPoint("Hos", new AtomicBoolean(false)));
 221         assertSearchResult("Höschen", "hos", deIndex.findInsertionPoint("hos", new AtomicBoolean(false)));
 222
 223         raf.close();
 224     }
 225
 226     public void testMultiSearch() throws IOException {
 227         final RandomAccessFile raf = new RandomAccessFile(TEST_OUTPUTS + "de-en.quickdic", "r");
 228         final Dictionary dict = new Dictionary(raf.getChannel());
 229         final Index deIndex = dict.indices.get(0);
 230
 231         {
 232             final List<RowBase> rows = deIndex.multiWordSearch("aaa aaab", Arrays.asList("aaa", "aaab"), new AtomicBoolean(false));
 233             System.out.println(CollectionUtil.join(rows, "\n  "));
 234             assertTrue(rows.toString(), rows.size() > 0);
 235         }
 236
 237         raf.close();
 238     }
 239
 240     public void testMultiSearchIt() throws IOException {
 241         final RandomAccessFile raf = new RandomAccessFile(OUTPUTS + "IT.quickdic", "r");
 242         final Dictionary dict = new Dictionary(raf.getChannel());
 243         final Index index = dict.indices.get(0);
 244
 245         {
 246             final List<RowBase> rows = index.multiWordSearch("fare centro",
 247                                        Arrays.asList("fare", "centro"), new AtomicBoolean(false));
 248             System.out.println(CollectionUtil.join(rows, "\n  "));
 249             assertTrue(rows.toString(), rows.size() > 0);
 250             assertTrue(rows.get(0).toString().startsWith("fare centro@"));
 251         }
 252     }
 253
 254     public void testMultiSearchDeBig() throws IOException {
 255         final RandomAccessFile raf = new RandomAccessFile(OUTPUTS + "DE-EN.quickdic", "r");
 256         final Dictionary dict = new Dictionary(raf.getChannel());
 257         final Index enIndex = dict.indices.get(1);
 258
 259         {
 260             final List<RowBase> rows = enIndex.multiWordSearch("train station", Arrays.asList("train", "station"), new AtomicBoolean(false));
 261             System.out.println(CollectionUtil.join(rows, "\n  "));
 262             assertTrue(rows.toString(), rows.size() > 0);
 263             assertTrue(rows.get(0).toString().startsWith("train station@"));
 264         }
 265
 266         {
 267             final List<RowBase> rows = enIndex.multiWordSearch("a train station", Arrays.asList("a", "train", "station"), new AtomicBoolean(false));
 268             System.out.println(CollectionUtil.join(rows, "\n  "));
 269             assertTrue(rows.toString(), rows.size() > 0);
 270             assertEquals("Bahnhofsuhr {{de-noun|g=f|plural=Bahnhofsuhren}}\tstation clock (at a train station)", rows.get(0).toString());
 271         }
 272
 273         {
 274             final List<RowBase> rows = enIndex.multiWordSearch("a station", Arrays.asList("a", "station"), new AtomicBoolean(false));
 275             System.out.println(CollectionUtil.join(rows, "\n  "));
 276             assertTrue(rows.toString(), rows.size() > 0);
 277             assertEquals("Abfahrthalle {en-noun}\tDeparture room of a station.", rows.get(0).toString());
 278         }
 279
 280         {
 281             // Should print: Giving up, too many words with prefix: p
 282             final List<RowBase> rows = enIndex.multiWordSearch("p eat", Arrays.asList("p", "eat"), new AtomicBoolean(false));
 283             System.out.println(CollectionUtil.join(rows, "\n  "));
 284             assertTrue(rows.toString(), rows.size() > 0);
 285             assertTrue(rows.toString().contains("verschlingen; verputzen\tto dispatch (eat)"));
 286         }
 287
 288         {
 289             // Should print: Giving up, too many words with prefix: p
 290             final List<RowBase> rows = enIndex.multiWordSearch("p p", Arrays.asList("p", "p"), new AtomicBoolean(false));
 291             assertTrue(rows.size() >= 1000);
 292         }
 293
 294         {
 295             // Should print: Giving up, too many words with prefix: a
 296             final List<RowBase> rows = enIndex.multiWordSearch("a a", Arrays.asList("a", "a"), new AtomicBoolean(false));
 297             assertTrue(rows.size() >= 1000);
 298         }
 299
 300         {
 301             // Should print: Giving up, too many words with prefix: a
 302             final List<RowBase> rows = enIndex.multiWordSearch("b ba", Arrays.asList("b", "ba"), new AtomicBoolean(false));
 303             assertTrue(rows.size() >= 1000);
 304         }
 305
 306         {
 307             // Should print: Giving up, too many words with prefix: a
 308             final List<RowBase> rows = enIndex.multiWordSearch("b ba", Arrays.asList("b", "ba"), new AtomicBoolean(false));
 309             assertTrue(rows.size() >= 1000);
 310         }
 311
 312         raf.close();
 313     }
 314
 315     public void testMultiSearchBigAF() throws IOException {
 316         final RandomAccessFile raf = new RandomAccessFile(OUTPUTS + "AF-EN.quickdic", "r");
 317         final Dictionary dict = new Dictionary(raf.getChannel());
 318         final Index enIndex = dict.indices.get(1);
 319
 320         {
 321             final List<RowBase> rows = enIndex.multiWordSearch("pig eats", Arrays.asList("pig", "eats"), new AtomicBoolean(false));
 322             System.out.println(CollectionUtil.join(rows, "\n  "));
 323             assertTrue(rows.toString(), rows.size() > 0);
 324             assertEquals("vark\tpig (someone who overeats or eats rapidly) (noun)", rows.get(0).toString());
 325         }
 326
 327         {
 328             final List<RowBase> rows = enIndex.multiWordSearch("pig eat", Arrays.asList("pig", "eat"), new AtomicBoolean(false));
 329             System.out.println(CollectionUtil.join(rows, "\n  "));
 330             assertTrue(rows.toString(), rows.size() > 0);
 331             assertEquals("vark\tpig (someone who overeats or eats rapidly) (noun)", rows.get(0).toString());
 332         }
 333
 334         {
 335             final List<RowBase> rows = enIndex.multiWordSearch("pi ea", Arrays.asList("pi", "ea"), new AtomicBoolean(false));
 336             System.out.println(CollectionUtil.join(rows, "\n  "));
 337             assertTrue(rows.toString(), rows.size() > 0);
 338             assertTrue(rows.toString().contains("vark\tpig (someone who overeats or eats rapidly) (noun)"));
 339         }
 340
 341         {
 342             final List<RowBase> rows = enIndex.multiWordSearch("p eat", Arrays.asList("p", "eat"), new AtomicBoolean(false));
 343             System.out.println(CollectionUtil.join(rows, "\n  "));
 344             assertTrue(rows.toString(), rows.size() > 0);
 345             assertTrue(rows.toString().contains("vark\tpig (someone who overeats or eats rapidly) (noun)"));
 346         }
 347
 348
 349         raf.close();
 350     }
 351
 352
 353     public void testExactSearch() throws IOException {
 354         final RandomAccessFile raf = new RandomAccessFile(OUTPUTS + "EN-cmn.quickdic", "r");
 355         final Dictionary dict = new Dictionary(raf.getChannel());
 356         final Index cmnIndex = dict.indices.get(1);
 357
 358         final Random random = new Random(10);
 359
 360         for (int i = 0; i < 1000; ++i) {
 361             final int ii = random.nextInt(cmnIndex.sortedIndexEntries.size());
 362             final IndexEntry indexEntry = cmnIndex.sortedIndexEntries.get(ii);
 363             final IndexEntry found = cmnIndex.findExact(indexEntry.token);
 364             assertNotNull(found);
 365             assertEquals(indexEntry.token, found.token);
 366             assertEquals(indexEntry, found);  // Test of caching....
 367         }
 368
 369         raf.close();
 370     }
 371
 372     public void testThai() throws IOException {
 373         final RandomAccessFile raf = new RandomAccessFile(OUTPUTS + "EN-TH.quickdic", "r");
 374         final Dictionary dict = new Dictionary(raf.getChannel());
 375         final Index thIndex = dict.indices.get(1);
 376
 377         final IndexEntry entry = thIndex.findInsertionPoint("ดี", new AtomicBoolean(false));
 378         assertEquals("di", entry.token);
 379
 380         raf.close();
 381     }
 382
 383     public void testNorwegian() throws IOException {
 384         final RandomAccessFile raf = new RandomAccessFile(OUTPUTS + "EN-NL.quickdic", "r");
 385         final Dictionary dict = new Dictionary(raf.getChannel());
 386         final Index nlIndex = dict.indices.get(1);
 387
 388         IndexEntry entry = nlIndex.findInsertionPoint("Xhosa", new AtomicBoolean(false));
 389         assertEquals("Xhosa", entry.token);
 390
 391         entry = nlIndex.findInsertionPoint("Zyne", new AtomicBoolean(false));
 392         assertEquals("Zyne", entry.token);
 393
 394         raf.close();
 395     }
 396
 397 }