1 // Copyright 2011 Google Inc. All Rights Reserved.
3 // Licensed under the Apache License, Version 2.0 (the "License");
4 // you may not use this file except in compliance with the License.
5 // You may obtain a copy of the License at
7 // http://www.apache.org/licenses/LICENSE-2.0
9 // Unless required by applicable law or agreed to in writing, software
10 // distributed under the License is distributed on an "AS IS" BASIS,
11 // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 // See the License for the specific language governing permissions and
13 // limitations under the License.
15 package com.hughes.android.dictionary.engine;
17 import java.io.IOException;
18 import java.io.RandomAccessFile;
19 import java.util.Arrays;
20 import java.util.List;
21 import java.util.Random;
22 import java.util.concurrent.atomic.AtomicBoolean;
24 import junit.framework.TestCase;
26 import com.hughes.android.dictionary.engine.Index.IndexEntry;
27 import com.hughes.util.CollectionUtil;
30 public class DictionaryTest extends TestCase {
32 static final String TEST_OUTPUTS = com.hughes.android.dictionary.engine.DictionaryBuilderTest.TEST_OUTPUTS;
33 public static final String OUTPUTS = "data/outputs/";
36 protected void setUp() {
37 while (!TransliteratorManager.init(null)) {
40 } catch (InterruptedException e) {
46 public void testURLFormatting() {
49 public void testEnItWiktionary() throws IOException {
50 final RandomAccessFile raf = new RandomAccessFile(OUTPUTS + "EN-IT.quickdic", "r");
51 final Dictionary dict = new Dictionary(raf);
52 final Index enIndex = dict.indices.get(0);
54 final RowBase row = enIndex.rows.get(4);
55 assertEquals("-ical", row.getRawText(false));
57 final Index itIndex = dict.indices.get(1);
59 final List<RowBase> rows = itIndex.multiWordSearch("come mai", Arrays.asList("come", "mai"), new AtomicBoolean(false));
60 System.out.println(CollectionUtil.join(rows, "\n "));
61 assertTrue(rows.toString(), rows.size() > 0);
62 assertTrue(rows.get(0).toString().startsWith("come mai@"));
63 assertTrue(rows.get(0) instanceof TokenRow);
64 assertTrue(!((TokenRow)rows.get(0)).getIndexEntry().htmlEntries.isEmpty());
68 final List<RowBase> rows = itIndex.multiWordSearch("buon g", Arrays.asList("buon", "g"), new AtomicBoolean(false));
69 System.out.println(CollectionUtil.join(rows, "\n "));
70 assertTrue(rows.toString(), rows.size() > 0);
71 assertTrue(rows.get(0).toString().startsWith("buon giorno@"));
72 assertTrue(rows.get(0) instanceof TokenRow);
73 assertTrue(!((TokenRow)rows.get(0)).getIndexEntry().htmlEntries.isEmpty());
77 final IndexEntry searchResult = itIndex.findInsertionPoint("azzurro", new AtomicBoolean(
79 HtmlEntry htmlEntry = searchResult.htmlEntries.get(0);
80 System.out.println("azzurro:\n" + htmlEntry.getHtml());
86 // public void testFr() throws IOException {
87 // final RandomAccessFile raf = new RandomAccessFile(OUTPUTS + "FR.quickdic", "r");
88 // final Dictionary dict = new Dictionary(raf);
89 // final Index frIndex = dict.indices.get(0);
91 // // Now they're all cached, we shouldn't have to search.
92 // for (final IndexEntry indexEntry : frIndex.sortedIndexEntries) {
93 // System.out.println(indexEntry.token);
100 public void testDeEnWiktionary() throws IOException {
101 final RandomAccessFile raf = new RandomAccessFile(OUTPUTS + "DE-EN.quickdic", "r");
102 final Dictionary dict = new Dictionary(raf);
104 final Index deIndex = dict.indices.get(0);
107 final IndexEntry searchResult = deIndex.findInsertionPoint("rot", new AtomicBoolean(
109 HtmlEntry htmlEntry = searchResult.htmlEntries.get(0);
110 System.out.println("rot:\n" + htmlEntry.getHtml());
116 public void testGermanMetadata() throws IOException {
117 final RandomAccessFile raf = new RandomAccessFile(TEST_OUTPUTS + "de-en.quickdic", "r");
118 final Dictionary dict = new Dictionary(raf);
119 final Index deIndex = dict.indices.get(0);
121 assertEquals("DE", deIndex.shortName);
122 assertEquals("DE->EN", deIndex.longName);
124 assertEquals(2, dict.sources.size());
125 assertEquals("chemnitz", dict.sources.get(0).name);
126 assertEquals("dictcc", dict.sources.get(1).name);
128 assertEquals("dictcc", dict.pairEntries.get(0).entrySource.name);
129 assertEquals("chemnitz", dict.pairEntries.get(1).entrySource.name);
134 public void testGermanIndex() throws IOException {
135 final RandomAccessFile raf = new RandomAccessFile(TEST_OUTPUTS + "de-en.quickdic", "r");
136 final Dictionary dict = new Dictionary(raf);
137 final Index deIndex = dict.indices.get(0);
139 for (final Index.IndexEntry indexEntry : deIndex.sortedIndexEntries) {
140 System.out.println("testing: " + indexEntry.token);
141 final IndexEntry searchResult = deIndex.findInsertionPoint(indexEntry.token, new AtomicBoolean(
143 assertEquals("Looked up: " + indexEntry.token, indexEntry.token.toLowerCase(), searchResult.token.toLowerCase());
146 // TODO: maybe if user types capitalization, use it.
147 assertSearchResult("aaac", "aaac", deIndex.findInsertionPoint("aaac", new AtomicBoolean(false)));
148 assertSearchResult("aaac", "aaac", deIndex.findInsertionPoint("AAAC", new AtomicBoolean(false)));
149 assertSearchResult("aaac", "aaac", deIndex.findInsertionPoint("AAAc", new AtomicBoolean(false)));
150 assertSearchResult("aaac", "aaac", deIndex.findInsertionPoint("aAac", new AtomicBoolean(false)));
152 // Before the beginning.
153 assertSearchResult("40", "40" /* special case */, deIndex.findInsertionPoint("", new AtomicBoolean(false)));
154 assertSearchResult("40", "40" /* special case */, deIndex.findInsertionPoint("__", new AtomicBoolean(false)));
157 assertSearchResult("Zweckorientiertheit", "zählen", deIndex.findInsertionPoint("ZZZZZ", new AtomicBoolean(false)));
159 assertSearchResult("ab", "aaac", deIndex.findInsertionPoint("aaaca", new AtomicBoolean(false)));
160 assertSearchResult("machen", "machen", deIndex.findInsertionPoint("m", new AtomicBoolean(false)));
161 assertSearchResult("machen", "machen", deIndex.findInsertionPoint("macdddd", new AtomicBoolean(false)));
164 assertSearchResult("überprüfe", "überprüfe", deIndex.findInsertionPoint("ueberprüfe", new AtomicBoolean(false)));
165 assertSearchResult("überprüfe", "überprüfe", deIndex.findInsertionPoint("ueberpruefe", new AtomicBoolean(false)));
167 assertSearchResult("überprüfe", "überprüfe", deIndex.findInsertionPoint("ueberpBLEH", new AtomicBoolean(false)));
168 assertSearchResult("überprüfe", "überprüfe", deIndex.findInsertionPoint("überprBLEH", new AtomicBoolean(false)));
170 assertSearchResult("überprüfen", "überprüfe", deIndex.findInsertionPoint("überprüfeBLEH", new AtomicBoolean(false)));
172 // Check that search in lowercase works.
173 assertSearchResult("Alibi", "Alibi", deIndex.findInsertionPoint("alib", new AtomicBoolean(false)));
174 System.out.println(deIndex.findInsertionPoint("alib", new AtomicBoolean(false)).toString());
179 private void assertSearchResult(final String insertionPoint, final String longestPrefix,
180 final IndexEntry actual) {
181 assertEquals(insertionPoint, actual.token);
184 public void testGermanTokenRows() throws IOException {
185 final RandomAccessFile raf = new RandomAccessFile(TEST_OUTPUTS + "de-en.quickdic", "r");
186 final Dictionary dict = new Dictionary(raf);
187 final Index deIndex = dict.indices.get(0);
189 // Pre-cache a few of these, just to make sure that's working.
190 for (int i = 0; i < deIndex.rows.size(); i += 7) {
191 deIndex.rows.get(i).getTokenRow(true);
194 // Do the exhaustive searching.
195 TokenRow lastTokenRow = null;
196 for (final RowBase row : deIndex.rows) {
197 if (row instanceof TokenRow) {
198 lastTokenRow = (TokenRow) row;
200 assertEquals(lastTokenRow, row.getTokenRow(true));
203 // Now they're all cached, we shouldn't have to search.
204 for (final RowBase row : deIndex.rows) {
205 if (row instanceof TokenRow) {
206 lastTokenRow = (TokenRow) row;
208 // This will break if the Row cache isn't big enough.
209 assertEquals(lastTokenRow, row.getTokenRow(false));
215 public void testChemnitz() throws IOException {
216 final RandomAccessFile raf = new RandomAccessFile(TEST_OUTPUTS + "de-en.quickdic", "r");
217 final Dictionary dict = new Dictionary(raf);
218 final Index deIndex = dict.indices.get(0);
220 assertSearchResult("Höschen", "Hos", deIndex.findInsertionPoint("Hos", new AtomicBoolean(false)));
221 assertSearchResult("Höschen", "hos", deIndex.findInsertionPoint("hos", new AtomicBoolean(false)));
226 public void testMultiSearch() throws IOException {
227 final RandomAccessFile raf = new RandomAccessFile(TEST_OUTPUTS + "de-en.quickdic", "r");
228 final Dictionary dict = new Dictionary(raf);
229 final Index deIndex = dict.indices.get(0);
232 final List<RowBase> rows = deIndex.multiWordSearch("aaa aaab", Arrays.asList("aaa", "aaab"), new AtomicBoolean(false));
233 System.out.println(CollectionUtil.join(rows, "\n "));
234 assertTrue(rows.toString(), rows.size() > 0);
240 public void testMultiSearchIt() throws IOException {
241 final RandomAccessFile raf = new RandomAccessFile(OUTPUTS + "IT.quickdic", "r");
242 final Dictionary dict = new Dictionary(raf);
243 final Index index = dict.indices.get(0);
246 final List<RowBase> rows = index.multiWordSearch("fare centro",
247 Arrays.asList("fare", "centro"), new AtomicBoolean(false));
248 System.out.println(CollectionUtil.join(rows, "\n "));
249 assertTrue(rows.toString(), rows.size() > 0);
250 assertTrue(rows.get(0).toString().startsWith("fare centro@"));
254 public void testMultiSearchDeBig() throws IOException {
255 final RandomAccessFile raf = new RandomAccessFile(OUTPUTS + "DE-EN.quickdic", "r");
256 final Dictionary dict = new Dictionary(raf);
257 final Index enIndex = dict.indices.get(1);
260 final List<RowBase> rows = enIndex.multiWordSearch("train station", Arrays.asList("train", "station"), new AtomicBoolean(false));
261 System.out.println(CollectionUtil.join(rows, "\n "));
262 assertTrue(rows.toString(), rows.size() > 0);
263 assertTrue(rows.get(0).toString().startsWith("train station@"));
267 final List<RowBase> rows = enIndex.multiWordSearch("a train station", Arrays.asList("a", "train", "station"), new AtomicBoolean(false));
268 System.out.println(CollectionUtil.join(rows, "\n "));
269 assertTrue(rows.toString(), rows.size() > 0);
270 assertEquals("Bahnhofsuhr {{de-noun|g=f|plural=Bahnhofsuhren}}\tstation clock (at a train station)", rows.get(0).toString());
274 final List<RowBase> rows = enIndex.multiWordSearch("a station", Arrays.asList("a", "station"), new AtomicBoolean(false));
275 System.out.println(CollectionUtil.join(rows, "\n "));
276 assertTrue(rows.toString(), rows.size() > 0);
277 assertEquals("Abfahrthalle {en-noun}\tDeparture room of a station.", rows.get(0).toString());
281 // Should print: Giving up, too many words with prefix: p
282 final List<RowBase> rows = enIndex.multiWordSearch("p eat", Arrays.asList("p", "eat"), new AtomicBoolean(false));
283 System.out.println(CollectionUtil.join(rows, "\n "));
284 assertTrue(rows.toString(), rows.size() > 0);
285 assertTrue(rows.toString().contains("verschlingen; verputzen\tto dispatch (eat)"));
289 // Should print: Giving up, too many words with prefix: p
290 final List<RowBase> rows = enIndex.multiWordSearch("p p", Arrays.asList("p", "p"), new AtomicBoolean(false));
291 assertTrue(rows.size() >= 1000);
295 // Should print: Giving up, too many words with prefix: a
296 final List<RowBase> rows = enIndex.multiWordSearch("a a", Arrays.asList("a", "a"), new AtomicBoolean(false));
297 assertTrue(rows.size() >= 1000);
301 // Should print: Giving up, too many words with prefix: a
302 final List<RowBase> rows = enIndex.multiWordSearch("b ba", Arrays.asList("b", "ba"), new AtomicBoolean(false));
303 assertTrue(rows.size() >= 1000);
307 // Should print: Giving up, too many words with prefix: a
308 final List<RowBase> rows = enIndex.multiWordSearch("b ba", Arrays.asList("b", "ba"), new AtomicBoolean(false));
309 assertTrue(rows.size() >= 1000);
315 public void testMultiSearchBigAF() throws IOException {
316 final RandomAccessFile raf = new RandomAccessFile(OUTPUTS + "AF-EN.quickdic", "r");
317 final Dictionary dict = new Dictionary(raf);
318 final Index enIndex = dict.indices.get(1);
321 final List<RowBase> rows = enIndex.multiWordSearch("pig eats", Arrays.asList("pig", "eats"), new AtomicBoolean(false));
322 System.out.println(CollectionUtil.join(rows, "\n "));
323 assertTrue(rows.toString(), rows.size() > 0);
324 assertEquals("vark\tpig (someone who overeats or eats rapidly) (noun)", rows.get(0).toString());
328 final List<RowBase> rows = enIndex.multiWordSearch("pig eat", Arrays.asList("pig", "eat"), new AtomicBoolean(false));
329 System.out.println(CollectionUtil.join(rows, "\n "));
330 assertTrue(rows.toString(), rows.size() > 0);
331 assertEquals("vark\tpig (someone who overeats or eats rapidly) (noun)", rows.get(0).toString());
335 final List<RowBase> rows = enIndex.multiWordSearch("pi ea", Arrays.asList("pi", "ea"), new AtomicBoolean(false));
336 System.out.println(CollectionUtil.join(rows, "\n "));
337 assertTrue(rows.toString(), rows.size() > 0);
338 assertTrue(rows.toString().contains("vark\tpig (someone who overeats or eats rapidly) (noun)"));
342 final List<RowBase> rows = enIndex.multiWordSearch("p eat", Arrays.asList("p", "eat"), new AtomicBoolean(false));
343 System.out.println(CollectionUtil.join(rows, "\n "));
344 assertTrue(rows.toString(), rows.size() > 0);
345 assertTrue(rows.toString().contains("vark\tpig (someone who overeats or eats rapidly) (noun)"));
353 public void testExactSearch() throws IOException {
354 final RandomAccessFile raf = new RandomAccessFile(OUTPUTS + "EN-cmn.quickdic", "r");
355 final Dictionary dict = new Dictionary(raf);
356 final Index cmnIndex = dict.indices.get(1);
358 final Random random = new Random(10);
360 for (int i = 0; i < 1000; ++i) {
361 final int ii = random.nextInt(cmnIndex.sortedIndexEntries.size());
362 final IndexEntry indexEntry = cmnIndex.sortedIndexEntries.get(ii);
363 final IndexEntry found = cmnIndex.findExact(indexEntry.token);
364 assertNotNull(found);
365 assertEquals(indexEntry.token, found.token);
366 assertEquals(indexEntry, found); // Test of caching....
372 public void testThai() throws IOException {
373 final RandomAccessFile raf = new RandomAccessFile(OUTPUTS + "EN-TH.quickdic", "r");
374 final Dictionary dict = new Dictionary(raf);
375 final Index thIndex = dict.indices.get(1);
377 final IndexEntry entry = thIndex.findInsertionPoint("ดี", new AtomicBoolean(false));
378 assertEquals("di", entry.token);
383 public void testNorwegian() throws IOException {
384 final RandomAccessFile raf = new RandomAccessFile(OUTPUTS + "EN-NL.quickdic", "r");
385 final Dictionary dict = new Dictionary(raf);
386 final Index nlIndex = dict.indices.get(1);
388 IndexEntry entry = nlIndex.findInsertionPoint("Xhosa", new AtomicBoolean(false));
389 assertEquals("Xhosa", entry.token);
391 entry = nlIndex.findInsertionPoint("Zyne", new AtomicBoolean(false));
392 assertEquals("Zyne", entry.token);