1 // Copyright 2011 Google Inc. All Rights Reserved.
3 // Licensed under the Apache License, Version 2.0 (the "License");
4 // you may not use this file except in compliance with the License.
5 // You may obtain a copy of the License at
7 // http://www.apache.org/licenses/LICENSE-2.0
9 // Unless required by applicable law or agreed to in writing, software
10 // distributed under the License is distributed on an "AS IS" BASIS,
11 // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 // See the License for the specific language governing permissions and
13 // limitations under the License.
15 package com.hughes.android.dictionary.engine;
17 import java.util.ArrayList;
18 import java.util.Arrays;
19 import java.util.Collections;
20 import java.util.LinkedHashSet;
21 import java.util.List;
24 import junit.framework.TestCase;
26 import com.hughes.android.dictionary.parser.DictFileParser;
27 import com.hughes.android.dictionary.parser.wiktionary.WiktionaryLangs;
28 import com.ibm.icu.text.Transliterator;
30 public class LanguageTest extends TestCase {
32 public void testGermanSort() {
33 final Transliterator normalizer = Transliterator.createFromRules("", Language.de.getDefaultNormalizerRules(), Transliterator.FORWARD);
34 assertEquals("aüääss", normalizer.transform("aueAeAEß"));
35 final List<String> words = Arrays.asList(
66 final NormalizeComparator comparator = new NormalizeComparator(normalizer, Language.de.getCollator());
67 assertEquals(1, comparator.compare("hülle", "huelle"));
68 assertEquals(-1, comparator.compare("huelle", "hülle"));
70 assertEquals(-1, comparator.compare("hülle", "Hülle"));
72 assertEquals("hülle", normalizer.transform("Hülle"));
73 assertEquals("hulle", normalizer.transform("Hulle"));
76 final List<String> sorted = new ArrayList<String>(words);
77 // Collections.shuffle(shuffled, new Random(0));
78 Collections.sort(sorted, comparator);
79 System.out.println(sorted.toString());
80 for (int i = 0; i < words.size(); ++i) {
81 System.out.println(words.get(i) + "\t" + sorted.get(i));
82 assertEquals(words.get(i), sorted.get(i));
86 public void testEnglishSort() {
87 final Transliterator normalizer = Transliterator.createFromRules("", Language.en.getDefaultNormalizerRules(), Transliterator.FORWARD);
89 final List<String> words = Arrays.asList(
95 final List<String> sorted = new ArrayList<String>(words);
96 final NormalizeComparator comparator = new NormalizeComparator(normalizer, Language.en.getCollator());
97 Collections.sort(sorted, comparator);
98 for (int i = 0; i < words.size(); ++i) {
100 assertTrue(comparator.compare(words.get(i-1), words.get(i)) < 0);
102 System.out.println(words.get(i) + "\t" + sorted.get(i));
103 assertEquals(words.get(i), sorted.get(i));
106 assertTrue(comparator.compare("pre-print", "preppy") < 0);
110 public void testLanguage() {
111 assertEquals(Language.de, Language.lookup("de"));
112 assertEquals(Language.en, Language.lookup("en"));
113 assertEquals("es", Language.lookup("es").getIsoCode());
116 public void testTextNorm() {
117 //final Transliterator transliterator = Transliterator.getInstance("Any-Latin; Upper; Lower; 'oe' > 'o'; NFD; [:Nonspacing Mark:] Remove; NFC", Transliterator.FORWARD);
118 final Transliterator transliterator = Transliterator.createFromRules("", ":: Any-Latin; :: Upper; :: Lower; 'oe' > 'o'; :: NFD; :: [:Nonspacing Mark:] Remove; :: NFC ;", Transliterator.FORWARD);
119 assertEquals("hoschen", transliterator.transliterate("Höschen"));
120 assertEquals("hoschen", transliterator.transliterate("Hoeschen"));
121 assertEquals("grosspoo", transliterator.transliterate("Großpoo"));
123 assertEquals("kyanpasu", transliterator.transliterate("キャンパス"));
124 assertEquals("alphabetikos katalogos", transliterator.transliterate("Αλφαβητικός Κατάλογος"));
125 assertEquals("biologiceskom", transliterator.transliterate("биологическом"));
127 public void testHalfTextNorm() {
128 final Transliterator transliterator = Transliterator.createFromRules("", ":: Any-Latin; ' ' > ; :: Lower; ", Transliterator.FORWARD);
129 assertEquals("kyanpasu", transliterator.transliterate("キャンパス"));
130 assertEquals("alphabētikóskatálogos", transliterator.transliterate("Αλφαβητικός Κατάλογος"));
131 assertEquals("biologičeskom", transliterator.transliterate("биологическом"));
133 assertEquals("xièxiè", transliterator.transliterate("謝謝"));
134 assertEquals("xièxiè", transliterator.transliterate("谢谢"));
136 assertEquals("diànnǎo", transliterator.transliterate("電腦"));
137 assertEquals("diànnǎo", transliterator.transliterate("电脑"));
138 assertEquals("jìsuànjī", transliterator.transliterate("計算機"));
139 assertEquals("jìsuànjī", transliterator.transliterate("计算机"));
143 public void testChinese() {
144 final Language cmn = Language.lookup("cmn");
145 final Transliterator transliterator = Transliterator.createFromRules("", cmn.getDefaultNormalizerRules(), Transliterator.FORWARD);
147 assertEquals("xiexie", transliterator.transliterate("謝謝"));
148 assertEquals("xiexie", transliterator.transliterate("谢谢"));
150 assertEquals("diannao", transliterator.transliterate("電腦"));
151 assertEquals("diannao", transliterator.transliterate("电脑"));
152 assertEquals("jisuanji", transliterator.transliterate("計算機"));
153 assertEquals("jisuanji", transliterator.transliterate("计算机"));
155 assertEquals("chengjiu", transliterator.transliterate("成就"));
159 public void testArabic() {
160 final Language ar = Language.lookup("ar");
161 final Transliterator transliterator = Transliterator.createFromRules("", ar.getDefaultNormalizerRules(), Transliterator.FORWARD);
162 // These don't seem quite right....
163 assertEquals("haswb", transliterator.transliterate("حاسوب"));
164 assertEquals("kmbywtr", transliterator.transliterate("كمبيوتر"));
166 assertEquals("{\u200e كمبيوتر \u200e}", Language.fixBidiText("{كمبيوتر}"));
167 assertEquals("{a=\u200e كمبيوتر \u200e}", Language.fixBidiText("{a=كمبيوتر}"));
168 assertEquals("(\u200e كمبيوتر \u200e)", Language.fixBidiText("(كمبيوتر)"));
169 assertEquals("أنثى أنْثَى (’únθā) {f}, إناث (’ināθ) {p}, اناثى (’anāθā) {p}", Language.fixBidiText("أنثى أنْثَى (’únθā) {f}, إناث (’ināθ) {p}, اناثى (’anāθā) {p}"));
173 public void testThai() {
174 final Language th = Language.lookup("TH");
175 final Transliterator transliterator = Transliterator.createFromRules("", th.getDefaultNormalizerRules(), Transliterator.FORWARD);
176 // Not sure these are right, just to know...
177 assertEquals("d", transliterator.transliterate("ด"));
178 assertEquals("di", transliterator.transliterate("ด ี"));
179 assertEquals("dii", transliterator.transliterate("ดีี"));
181 assertEquals(Collections.singleton("ดีี"), DictFileParser.tokenize("ดีี", DictFileParser.NON_CHAR));
185 public void testEnWiktionaryNames() {
186 final Set<String> enLangs = new LinkedHashSet<String>(WiktionaryLangs.isoCodeToEnWikiName.keySet());
187 final List<String> names = new ArrayList<String>();
188 for (final String code : WiktionaryLangs.isoCodeToEnWikiName.keySet()) {
189 names.add(WiktionaryLangs.isoCodeToEnWikiName.get(code));
190 enLangs.add(code.toLowerCase());
192 Collections.sort(names);
193 System.out.println(names);
194 //assertEquals(enLangs, Language.isoCodeToResources.keySet());