src/com/hughes/android/dictionary/engine/LanguageTest.java

   1 // Copyright 2011 Google Inc. All Rights Reserved.
   2 //
   3 // Licensed under the Apache License, Version 2.0 (the "License");
   4 // you may not use this file except in compliance with the License.
   5 // You may obtain a copy of the License at
   6 //
   7 //     http://www.apache.org/licenses/LICENSE-2.0
   8 //
   9 // Unless required by applicable law or agreed to in writing, software
  10 // distributed under the License is distributed on an "AS IS" BASIS,
  11 // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  12 // See the License for the specific language governing permissions and
  13 // limitations under the License.
  14
  15 package com.hughes.android.dictionary.engine;
  16
  17 import java.util.ArrayList;
  18 import java.util.Arrays;
  19 import java.util.Collections;
  20 import java.util.LinkedHashSet;
  21 import java.util.List;
  22 import java.util.Set;
  23
  24 import com.hughes.android.dictionary.parser.DictFileParser;
  25 import com.hughes.android.dictionary.parser.wiktionary.WiktionaryLangs;
  26 import com.ibm.icu.text.Transliterator;
  27
  28 import junit.framework.TestCase;
  29
  30 public class LanguageTest extends TestCase {
  31
  32     public void testGermanSort() {
  33         final Transliterator normalizer = Transliterator.createFromRules("", Language.de.getDefaultNormalizerRules(), Transliterator.FORWARD);
  34         assertEquals("aüääss", normalizer.transform("aueAeAEß"));
  35         final List<String> words = Arrays.asList(
  36                                        "er-ben",
  37                                        "erben",
  38                                        "Erben",
  39                                        "Erbse",
  40                                        "Erbsen",
  41                                        "essen",
  42                                        "Essen",
  43                                        "Grosformat",
  44                                        "Grosformats",
  45                                        "Grossformat",
  46                                        "Großformat",
  47                                        "Grossformats",
  48                                        "Großformats",
  49                                        "Großpoo",
  50                                        "Großpoos",
  51                                        "Hörvermögen",
  52                                        "Hörweite",
  53                                        "hos",
  54                                        "Höschen",
  55                                        "Hostel",
  56                                        "hulle",
  57                                        "Hulle",
  58                                        "huelle",
  59                                        "Huelle",
  60                                        "hülle",
  61                                        "Hülle",
  62                                        "Huellen",
  63                                        "Hüllen",
  64                                        "Hum"
  65                                    );
  66         final NormalizeComparator comparator = new NormalizeComparator(normalizer, Language.de.getCollator(), 7);
  67         assertEquals(1, comparator.compare("hülle", "huelle"));
  68         assertEquals(-1, comparator.compare("huelle", "hülle"));
  69
  70         assertEquals(-1, comparator.compare("hülle", "Hülle"));
  71
  72         assertEquals("hülle", normalizer.transform("Hülle"));
  73         assertEquals("hulle", normalizer.transform("Hulle"));
  74
  75
  76         final List<String> sorted = new ArrayList<>(words);
  77 //    Collections.shuffle(shuffled, new Random(0));
  78         sorted.sort(comparator);
  79         System.out.println(sorted);
  80         for (int i = 0; i < words.size(); ++i) {
  81             System.out.println(words.get(i) + "\t" + sorted.get(i));
  82             assertEquals(words.get(i), sorted.get(i));
  83         }
  84     }
  85
  86     public void testEnglishSort() {
  87         final Transliterator normalizer = Transliterator.createFromRules("", Language.en.getDefaultNormalizerRules(), Transliterator.FORWARD);
  88
  89         final List<String> words = Arrays.asList(
  90                                        "pre-print",
  91                                        "preppie",
  92                                        "preppy",
  93                                        "preprocess");
  94
  95         final List<String> sorted = new ArrayList<>(words);
  96         final NormalizeComparator comparator = new NormalizeComparator(normalizer, Language.en.getCollator(), 7);
  97         sorted.sort(comparator);
  98         for (int i = 0; i < words.size(); ++i) {
  99             if (i > 0) {
 100                 assertTrue(comparator.compare(words.get(i-1), words.get(i)) < 0);
 101             }
 102             System.out.println(words.get(i) + "\t" + sorted.get(i));
 103             assertEquals(words.get(i), sorted.get(i));
 104         }
 105
 106         assertTrue(comparator.compare("pre-print", "preppy") < 0);
 107
 108     }
 109
 110     public void testLanguage() {
 111         assertEquals(Language.de, Language.lookup("de"));
 112         assertEquals(Language.en, Language.lookup("en"));
 113         assertEquals("es", Language.lookup("es").getIsoCode());
 114     }
 115
 116     public void testTextNorm() {
 117         //final Transliterator transliterator = Transliterator.getInstance("Any-Latin; Upper; Lower; 'oe' > 'o'; NFD; [:Nonspacing Mark:] Remove; NFC", Transliterator.FORWARD);
 118         final Transliterator transliterator = Transliterator.createFromRules("", ":: Any-Latin; :: Upper; :: Lower; 'oe' > 'o'; :: NFD; :: [:Nonspacing Mark:] Remove; :: NFC ;", Transliterator.FORWARD);
 119         assertEquals("hoschen", transliterator.transliterate("Höschen"));
 120         assertEquals("hoschen", transliterator.transliterate("Hoeschen"));
 121         assertEquals("grosspoo", transliterator.transliterate("Großpoo"));
 122
 123         assertEquals("kyanpasu", transliterator.transliterate("キャンパス"));
 124         assertEquals("alphabetikos katalogos", transliterator.transliterate("Αλφαβητικός Κατάλογος"));
 125         assertEquals("biologiceskom", transliterator.transliterate("биологическом"));
 126     }
 127     public void testHalfTextNorm() {
 128         final Transliterator transliterator = Transliterator.createFromRules("", ":: Any-Latin; ' ' > ; :: Lower; ", Transliterator.FORWARD);
 129         assertEquals("kyanpasu", transliterator.transliterate("キャンパス"));
 130         assertEquals("alphabētikóskatálogos", transliterator.transliterate("Αλφαβητικός Κατάλογος"));
 131         assertEquals("biologičeskom", transliterator.transliterate("биологическом"));
 132
 133         assertEquals("xièxiè", transliterator.transliterate("謝謝"));
 134         assertEquals("xièxiè", transliterator.transliterate("谢谢"));
 135
 136         assertEquals("diànnǎo", transliterator.transliterate("電腦"));
 137         assertEquals("diànnǎo", transliterator.transliterate("电脑"));
 138         assertEquals("jìsuànjī", transliterator.transliterate("計算機"));
 139         assertEquals("jìsuànjī", transliterator.transliterate("计算机"));
 140     }
 141
 142
 143     public void testChinese() {
 144         final Language cmn = Language.lookup("cmn");
 145         final Transliterator transliterator = Transliterator.createFromRules("", cmn.getDefaultNormalizerRules(), Transliterator.FORWARD);
 146
 147         assertEquals("xiexie", transliterator.transliterate("謝謝"));
 148         assertEquals("xiexie", transliterator.transliterate("谢谢"));
 149
 150         assertEquals("diannao", transliterator.transliterate("電腦"));
 151         assertEquals("diannao", transliterator.transliterate("电脑"));
 152         assertEquals("jisuanji", transliterator.transliterate("計算機"));
 153         assertEquals("jisuanji", transliterator.transliterate("计算机"));
 154
 155         assertEquals("chengjiu", transliterator.transliterate("成就"));
 156
 157     }
 158
 159     public void testArabic() {
 160         final Language ar = Language.lookup("ar");
 161         final Transliterator transliterator = Transliterator.createFromRules("", ar.getDefaultNormalizerRules(), Transliterator.FORWARD);
 162         // These don't seem quite right....
 163         assertEquals("haswb", transliterator.transliterate("حاسوب"));
 164         assertEquals("kmbywtr", transliterator.transliterate("كمبيوتر"));
 165
 166         assertEquals("{\u200e كمبيوتر \u200e}", Language.fixBidiText("{كمبيوتر}"));
 167         assertEquals("{a=\u200e كمبيوتر \u200e}", Language.fixBidiText("{a=كمبيوتر}"));
 168         assertEquals("(\u200e كمبيوتر \u200e)", Language.fixBidiText("(كمبيوتر)"));
 169         assertEquals("أنثى أنْثَى (’únθā) {f}, إناث (’ināθ) {p}, اناثى (’anāθā) {p}", Language.fixBidiText("أنثى أنْثَى (’únθā) {f}, إناث (’ināθ) {p}, اناثى (’anāθā) {p}"));
 170
 171     }
 172
 173     public void testThai() {
 174         final Language th = Language.lookup("TH");
 175         final Transliterator transliterator = Transliterator.createFromRules("", th.getDefaultNormalizerRules(), Transliterator.FORWARD);
 176         // Not sure these are right, just to know...
 177         assertEquals("d", transliterator.transliterate("ด"));
 178         assertEquals("di", transliterator.transliterate("ด ี"));
 179         assertEquals("dii", transliterator.transliterate("ดีี"));
 180
 181         assertEquals(Collections.singleton("ดีี"), DictFileParser.tokenize("ดีี", DictFileParser.NON_CHAR));
 182     }
 183
 184
 185     public void testEnWiktionaryNames() {
 186         final Set<String> enLangs = new LinkedHashSet<>(WiktionaryLangs.isoCodeToEnWikiName.keySet());
 187         final List<String> names = new ArrayList<>();
 188         for (final String code : WiktionaryLangs.isoCodeToEnWikiName.keySet()) {
 189             names.add(WiktionaryLangs.isoCodeToEnWikiName.get(code));
 190             enLangs.add(code.toLowerCase());
 191         }
 192         Collections.sort(names);
 193         System.out.println(names);
 194         //assertEquals(enLangs, Language.isoCodeToResources.keySet());
 195     }
 196
 197 }