1 // Copyright 2011 Google Inc. All Rights Reserved.
3 // Licensed under the Apache License, Version 2.0 (the "License");
4 // you may not use this file except in compliance with the License.
5 // You may obtain a copy of the License at
7 // http://www.apache.org/licenses/LICENSE-2.0
9 // Unless required by applicable law or agreed to in writing, software
10 // distributed under the License is distributed on an "AS IS" BASIS,
11 // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 // See the License for the specific language governing permissions and
13 // limitations under the License.
15 package com.hughes.android.dictionary.engine;
17 import java.io.IOException;
18 import java.io.RandomAccessFile;
19 import java.util.Arrays;
20 import java.util.List;
21 import java.util.Random;
22 import java.util.concurrent.atomic.AtomicBoolean;
24 import junit.framework.TestCase;
26 import com.hughes.android.dictionary.engine.Index.IndexEntry;
27 import com.hughes.util.CollectionUtil;
30 public class DictionaryTest extends TestCase {
32 static final String TEST_OUTPUTS = com.hughes.android.dictionary.engine.DictionaryBuilderTest.TEST_OUTPUTS;
33 public static final String OUTPUTS = "data/outputs/";
36 protected void setUp() {
37 while (!TransliteratorManager.init(null)) {
40 } catch (InterruptedException e) {
46 public void testEnItWiktionary() throws IOException {
47 final RandomAccessFile raf = new RandomAccessFile(OUTPUTS + "EN-IT_enwiktionary.quickdic", "r");
48 final Dictionary dict = new Dictionary(raf);
49 final Index enIndex = dict.indices.get(0);
51 final RowBase row = enIndex.rows.get(4);
52 assertEquals("The numeral 00\tzeranta (noun) {m|f|inv}", row.getRawText(false));
57 public void testGermanMetadata() throws IOException {
58 final RandomAccessFile raf = new RandomAccessFile(TEST_OUTPUTS + "de-en.quickdic", "r");
59 final Dictionary dict = new Dictionary(raf);
60 final Index deIndex = dict.indices.get(0);
62 assertEquals("DE", deIndex.shortName);
63 assertEquals("DE->EN", deIndex.longName);
65 assertEquals(2, dict.sources.size());
66 assertEquals("chemnitz", dict.sources.get(0).name);
67 assertEquals("dictcc", dict.sources.get(1).name);
69 assertEquals("dictcc", dict.pairEntries.get(0).entrySource.name);
70 assertEquals("chemnitz", dict.pairEntries.get(1).entrySource.name);
75 public void testGermanIndex() throws IOException {
76 final RandomAccessFile raf = new RandomAccessFile(TEST_OUTPUTS + "de-en.quickdic", "r");
77 final Dictionary dict = new Dictionary(raf);
78 final Index deIndex = dict.indices.get(0);
80 for (final Index.IndexEntry indexEntry : deIndex.sortedIndexEntries) {
81 System.out.println("testing: " + indexEntry.token);
82 final IndexEntry searchResult = deIndex.findInsertionPoint(indexEntry.token, new AtomicBoolean(
84 assertEquals("Looked up: " + indexEntry.token, indexEntry.token.toLowerCase(), searchResult.token.toLowerCase());
87 // TODO: maybe if user types capitalization, use it.
88 assertSearchResult("aaac", "aaac", deIndex.findInsertionPoint("aaac", new AtomicBoolean(false)));
89 assertSearchResult("aaac", "aaac", deIndex.findInsertionPoint("AAAC", new AtomicBoolean(false)));
90 assertSearchResult("aaac", "aaac", deIndex.findInsertionPoint("AAAc", new AtomicBoolean(false)));
91 assertSearchResult("aaac", "aaac", deIndex.findInsertionPoint("aAac", new AtomicBoolean(false)));
93 // Before the beginning.
94 assertSearchResult("40", "40" /* special case */, deIndex.findInsertionPoint("", new AtomicBoolean(false)));
95 assertSearchResult("40", "40" /* special case */, deIndex.findInsertionPoint("__", new AtomicBoolean(false)));
98 assertSearchResult("Zweckorientiertheit", "zählen", deIndex.findInsertionPoint("ZZZZZ", new AtomicBoolean(false)));
100 assertSearchResult("ab", "aaac", deIndex.findInsertionPoint("aaaca", new AtomicBoolean(false)));
101 assertSearchResult("machen", "machen", deIndex.findInsertionPoint("m", new AtomicBoolean(false)));
102 assertSearchResult("machen", "machen", deIndex.findInsertionPoint("macdddd", new AtomicBoolean(false)));
105 assertSearchResult("überprüfe", "überprüfe", deIndex.findInsertionPoint("ueberprüfe", new AtomicBoolean(false)));
106 assertSearchResult("überprüfe", "überprüfe", deIndex.findInsertionPoint("ueberpruefe", new AtomicBoolean(false)));
108 assertSearchResult("überprüfe", "überprüfe", deIndex.findInsertionPoint("ueberpBLEH", new AtomicBoolean(false)));
109 assertSearchResult("überprüfe", "überprüfe", deIndex.findInsertionPoint("überprBLEH", new AtomicBoolean(false)));
111 assertSearchResult("überprüfen", "überprüfe", deIndex.findInsertionPoint("überprüfeBLEH", new AtomicBoolean(false)));
113 // Check that search in lowercase works.
114 assertSearchResult("Alibi", "Alibi", deIndex.findInsertionPoint("alib", new AtomicBoolean(false)));
115 System.out.println(deIndex.findInsertionPoint("alib", new AtomicBoolean(false)).toString());
120 private void assertSearchResult(final String insertionPoint, final String longestPrefix,
121 final IndexEntry actual) {
122 assertEquals(insertionPoint, actual.token);
125 public void testGermanTokenRows() throws IOException {
126 final RandomAccessFile raf = new RandomAccessFile(TEST_OUTPUTS + "de-en.quickdic", "r");
127 final Dictionary dict = new Dictionary(raf);
128 final Index deIndex = dict.indices.get(0);
130 // Pre-cache a few of these, just to make sure that's working.
131 for (int i = 0; i < deIndex.rows.size(); i += 7) {
132 deIndex.rows.get(i).getTokenRow(true);
135 // Do the exhaustive searching.
136 TokenRow lastTokenRow = null;
137 for (final RowBase row : deIndex.rows) {
138 if (row instanceof TokenRow) {
139 lastTokenRow = (TokenRow) row;
141 assertEquals(lastTokenRow, row.getTokenRow(true));
144 // Now they're all cached, we shouldn't have to search.
145 for (final RowBase row : deIndex.rows) {
146 if (row instanceof TokenRow) {
147 lastTokenRow = (TokenRow) row;
149 // This will break if the Row cache isn't big enough.
150 assertEquals(lastTokenRow, row.getTokenRow(false));
156 public void testChemnitz() throws IOException {
157 final RandomAccessFile raf = new RandomAccessFile(TEST_OUTPUTS + "de-en.quickdic", "r");
158 final Dictionary dict = new Dictionary(raf);
159 final Index deIndex = dict.indices.get(0);
161 assertSearchResult("Höschen", "Hos", deIndex.findInsertionPoint("Hos", new AtomicBoolean(false)));
162 assertSearchResult("Höschen", "hos", deIndex.findInsertionPoint("hos", new AtomicBoolean(false)));
167 public void testMultiSearch() throws IOException {
168 final RandomAccessFile raf = new RandomAccessFile(TEST_OUTPUTS + "de-en.quickdic", "r");
169 final Dictionary dict = new Dictionary(raf);
170 final Index deIndex = dict.indices.get(0);
173 final List<RowBase> rows = deIndex.multiWordSearch(Arrays.asList("aaa", "aaab"), new AtomicBoolean(false));
174 System.out.println(CollectionUtil.join(rows, "\n "));
175 assertTrue(rows.toString(), rows.size() > 0);
181 public void testMultiSearchBig() throws IOException {
182 final RandomAccessFile raf = new RandomAccessFile(OUTPUTS + "DE-EN_chemnitz_enwiktionary.quickdic", "r");
183 final Dictionary dict = new Dictionary(raf);
184 final Index enIndex = dict.indices.get(1);
187 final List<RowBase> rows = enIndex.multiWordSearch(Arrays.asList("train", "station"), new AtomicBoolean(false));
188 System.out.println(CollectionUtil.join(rows, "\n "));
189 assertTrue(rows.toString(), rows.size() > 0);
190 assertEquals("Bahnhof {{de-noun|g=m|genitive=Bahnhofs|genitive2=Bahnhofes|plural=Bahnhöfe}}\ttrain station", rows.get(0).toString());
194 final List<RowBase> rows = enIndex.multiWordSearch(Arrays.asList("a", "train", "station"), new AtomicBoolean(false));
195 System.out.println(CollectionUtil.join(rows, "\n "));
196 assertTrue(rows.toString(), rows.size() > 0);
197 assertEquals("Bahnhofsuhr {{de-noun|g=f|plural=Bahnhofsuhren}}\tstation clock (at a train station)", rows.get(0).toString());
201 final List<RowBase> rows = enIndex.multiWordSearch(Arrays.asList("a", "station"), new AtomicBoolean(false));
202 System.out.println(CollectionUtil.join(rows, "\n "));
203 assertTrue(rows.toString(), rows.size() > 0);
204 assertEquals("Kraftwerk {n}\tpower plant (a station built for the production of electric power) (noun)", rows.get(0).toString());
208 // Should print: Giving up, too many words with prefix: p
209 final List<RowBase> rows = enIndex.multiWordSearch(Arrays.asList("p", "eat"), new AtomicBoolean(false));
210 System.out.println(CollectionUtil.join(rows, "\n "));
211 assertTrue(rows.toString(), rows.size() > 0);
212 assertTrue(rows.toString().contains("verschlingen; verputzen\tto dispatch (eat)"));
216 // Should print: Giving up, too many words with prefix: p
217 final List<RowBase> rows = enIndex.multiWordSearch(Arrays.asList("p", "p"), new AtomicBoolean(false));
218 assertTrue(rows.size() >= 1000);
222 // Should print: Giving up, too many words with prefix: a
223 final List<RowBase> rows = enIndex.multiWordSearch(Arrays.asList("a", "a"), new AtomicBoolean(false));
224 assertTrue(rows.size() >= 1000);
228 // Should print: Giving up, too many words with prefix: a
229 final List<RowBase> rows = enIndex.multiWordSearch(Arrays.asList("b", "ba"), new AtomicBoolean(false));
230 assertTrue(rows.size() >= 1000);
234 // Should print: Giving up, too many words with prefix: a
235 final List<RowBase> rows = enIndex.multiWordSearch(Arrays.asList("b", "ba"), new AtomicBoolean(false));
236 assertTrue(rows.size() >= 1000);
242 public void testMultiSearchBigAF() throws IOException {
243 final RandomAccessFile raf = new RandomAccessFile(OUTPUTS + "EN-AF_enwiktionary.quickdic", "r");
244 final Dictionary dict = new Dictionary(raf);
245 final Index enIndex = dict.indices.get(0);
248 final List<RowBase> rows = enIndex.multiWordSearch(Arrays.asList("pig", "eats"), new AtomicBoolean(false));
249 System.out.println(CollectionUtil.join(rows, "\n "));
250 assertTrue(rows.toString(), rows.size() > 0);
251 assertEquals("pig (someone who overeats or eats rapidly) (noun)\tvark", rows.get(0).toString());
255 final List<RowBase> rows = enIndex.multiWordSearch(Arrays.asList("pig", "eat"), new AtomicBoolean(false));
256 System.out.println(CollectionUtil.join(rows, "\n "));
257 assertTrue(rows.toString(), rows.size() > 0);
258 assertEquals("pig (someone who overeats or eats rapidly) (noun)\tvark", rows.get(0).toString());
262 final List<RowBase> rows = enIndex.multiWordSearch(Arrays.asList("pi", "ea"), new AtomicBoolean(false));
263 System.out.println(CollectionUtil.join(rows, "\n "));
264 assertTrue(rows.toString(), rows.size() > 0);
265 assertTrue(rows.toString().contains("pig (someone who overeats or eats rapidly) (noun)\tvark"));
269 final List<RowBase> rows = enIndex.multiWordSearch(Arrays.asList("p", "eat"), new AtomicBoolean(false));
270 System.out.println(CollectionUtil.join(rows, "\n "));
271 assertTrue(rows.toString(), rows.size() > 0);
272 assertTrue(rows.toString().contains("pig (someone who overeats or eats rapidly) (noun)\tvark"));
280 public void testExactSearch() throws IOException {
281 final RandomAccessFile raf = new RandomAccessFile(OUTPUTS + "EN-ZH_enwiktionary.quickdic", "r");
282 final Dictionary dict = new Dictionary(raf);
283 final Index zhIndex = dict.indices.get(1);
285 final Random random = new Random(10);
287 for (int i = 0; i < 1000; ++i) {
288 final int ii = random.nextInt(zhIndex.sortedIndexEntries.size());
289 final IndexEntry indexEntry = zhIndex.sortedIndexEntries.get(ii);
290 final IndexEntry found = zhIndex.findExact(indexEntry.token);
291 assertNotNull(found);
292 assertEquals(indexEntry.token, found.token);
293 assertEquals(indexEntry, found); // Test of caching....
299 public void testThai() throws IOException {
300 final RandomAccessFile raf = new RandomAccessFile(OUTPUTS + "EN-TH_enwiktionary.quickdic", "r");
301 final Dictionary dict = new Dictionary(raf);
302 final Index thIndex = dict.indices.get(1);
304 final IndexEntry entry = thIndex.findInsertionPoint("ดี", new AtomicBoolean(false));
305 assertEquals("di", entry.token);
310 public void testNorwegiani() throws IOException {
311 final RandomAccessFile raf = new RandomAccessFile(OUTPUTS + "EN-NL_enwiktionary.quickdic", "r");
312 final Dictionary dict = new Dictionary(raf);
313 final Index nlIndex = dict.indices.get(1);
315 IndexEntry entry = nlIndex.findInsertionPoint("Xhosa", new AtomicBoolean(false));
316 assertEquals("Xhosa", entry.token);
318 entry = nlIndex.findInsertionPoint("Zyne", new AtomicBoolean(false));
319 assertEquals("Zyne", entry.token);