src/com/hughes/android/dictionary/engine/LanguageTest.java

   1 // Copyright 2011 Google Inc. All Rights Reserved.
   2 //
   3 // Licensed under the Apache License, Version 2.0 (the "License");
   4 // you may not use this file except in compliance with the License.
   5 // You may obtain a copy of the License at
   6 //
   7 //     http://www.apache.org/licenses/LICENSE-2.0
   8 //
   9 // Unless required by applicable law or agreed to in writing, software
  10 // distributed under the License is distributed on an "AS IS" BASIS,
  11 // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  12 // See the License for the specific language governing permissions and
  13 // limitations under the License.
  14
  15 package com.hughes.android.dictionary.engine;
  16
  17 import java.util.ArrayList;
  18 import java.util.Arrays;
  19 import java.util.Collections;
  20 import java.util.LinkedHashSet;
  21 import java.util.List;
  22 import java.util.Set;
  23
  24 import junit.framework.TestCase;
  25
  26 import com.hughes.android.dictionary.parser.DictFileParser;
  27 import com.hughes.android.dictionary.parser.wiktionary.WiktionaryLangs;
  28 import com.ibm.icu.text.Transliterator;
  29
  30 public class LanguageTest extends TestCase {
  31
  32   public void testGermanSort() {
  33     final Transliterator normalizer = Transliterator.createFromRules("", Language.de.getDefaultNormalizerRules(), Transliterator.FORWARD);
  34     assertEquals("aüääss", normalizer.transform("aueAeAEß"));
  35     final List<String> words = Arrays.asList(
  36         "er-ben",
  37         "erben",
  38         "Erben",
  39         "Erbse",
  40         "Erbsen",
  41         "essen",
  42         "Essen",
  43         "Grosformat",
  44         "Grosformats",
  45         "Grossformat",
  46         "Großformat",
  47         "Grossformats",
  48         "Großformats",
  49         "Großpoo",
  50         "Großpoos",
  51         "Hörvermögen",
  52         "Hörweite",
  53         "hos",
  54         "Höschen",
  55         "Hostel",
  56         "hulle",
  57         "Hulle",
  58         "huelle",
  59         "Huelle",
  60         "hülle",
  61         "Hülle",
  62         "Huellen",
  63         "Hüllen",
  64         "Hum"
  65         );
  66     final NormalizeComparator comparator = new NormalizeComparator(normalizer, Language.de.getCollator(), 7);
  67     assertEquals(1, comparator.compare("hülle", "huelle"));
  68     assertEquals(-1, comparator.compare("huelle", "hülle"));
  69
  70     assertEquals(-1, comparator.compare("hülle", "Hülle"));
  71
  72     assertEquals("hülle", normalizer.transform("Hülle"));
  73     assertEquals("hulle", normalizer.transform("Hulle"));
  74
  75
  76     final List<String> sorted = new ArrayList<String>(words);
  77 //    Collections.shuffle(shuffled, new Random(0));
  78     Collections.sort(sorted, comparator);
  79     System.out.println(sorted.toString());
  80     for (int i = 0; i < words.size(); ++i) {
  81       System.out.println(words.get(i) + "\t" + sorted.get(i));
  82       assertEquals(words.get(i), sorted.get(i));
  83     }
  84   }
  85
  86   public void testEnglishSort() {
  87     final Transliterator normalizer = Transliterator.createFromRules("", Language.en.getDefaultNormalizerRules(), Transliterator.FORWARD);
  88
  89     final List<String> words = Arrays.asList(
  90         "pre-print",
  91         "preppie",
  92         "preppy",
  93         "preprocess");
  94
  95     final List<String> sorted = new ArrayList<String>(words);
  96     final NormalizeComparator comparator = new NormalizeComparator(normalizer, Language.en.getCollator(), 7);
  97     Collections.sort(sorted, comparator);
  98     for (int i = 0; i < words.size(); ++i) {
  99       if (i > 0) {
 100         assertTrue(comparator.compare(words.get(i-1), words.get(i)) < 0);
 101       }
 102       System.out.println(words.get(i) + "\t" + sorted.get(i));
 103       assertEquals(words.get(i), sorted.get(i));
 104     }
 105
 106     assertTrue(comparator.compare("pre-print", "preppy") < 0);
 107
 108   }
 109
 110   public void testLanguage() {
 111     assertEquals(Language.de, Language.lookup("de"));
 112     assertEquals(Language.en, Language.lookup("en"));
 113     assertEquals("es", Language.lookup("es").getIsoCode());
 114   }
 115
 116   public void testTextNorm() {
 117     //final Transliterator transliterator = Transliterator.getInstance("Any-Latin; Upper; Lower; 'oe' > 'o'; NFD; [:Nonspacing Mark:] Remove; NFC", Transliterator.FORWARD);
 118     final Transliterator transliterator = Transliterator.createFromRules("", ":: Any-Latin; :: Upper; :: Lower; 'oe' > 'o'; :: NFD; :: [:Nonspacing Mark:] Remove; :: NFC ;", Transliterator.FORWARD);
 119     assertEquals("hoschen", transliterator.transliterate("Höschen"));
 120     assertEquals("hoschen", transliterator.transliterate("Hoeschen"));
 121     assertEquals("grosspoo", transliterator.transliterate("Großpoo"));
 122
 123     assertEquals("kyanpasu", transliterator.transliterate("キャンパス"));
 124     assertEquals("alphabetikos katalogos", transliterator.transliterate("Αλφαβητικός Κατάλογος"));
 125     assertEquals("biologiceskom", transliterator.transliterate("биологическом"));
 126   }
 127   public void testHalfTextNorm() {
 128     final Transliterator transliterator = Transliterator.createFromRules("", ":: Any-Latin; ' ' > ; :: Lower; ", Transliterator.FORWARD);
 129     assertEquals("kyanpasu", transliterator.transliterate("キャンパス"));
 130     assertEquals("alphabētikóskatálogos", transliterator.transliterate("Αλφαβητικός Κατάλογος"));
 131     assertEquals("biologičeskom", transliterator.transliterate("биологическом"));
 132
 133     assertEquals("xièxiè", transliterator.transliterate("謝謝"));
 134     assertEquals("xièxiè", transliterator.transliterate("谢谢"));
 135
 136     assertEquals("diànnǎo", transliterator.transliterate("電腦"));
 137     assertEquals("diànnǎo", transliterator.transliterate("电脑"));
 138     assertEquals("jìsuànjī", transliterator.transliterate("計算機"));
 139     assertEquals("jìsuànjī", transliterator.transliterate("计算机"));
 140   }
 141
 142
 143   public void testChinese() {
 144     final Language cmn = Language.lookup("cmn");
 145     final Transliterator transliterator = Transliterator.createFromRules("", cmn.getDefaultNormalizerRules(), Transliterator.FORWARD);
 146
 147     assertEquals("xiexie", transliterator.transliterate("謝謝"));
 148     assertEquals("xiexie", transliterator.transliterate("谢谢"));
 149
 150     assertEquals("diannao", transliterator.transliterate("電腦"));
 151     assertEquals("diannao", transliterator.transliterate("电脑"));
 152     assertEquals("jisuanji", transliterator.transliterate("計算機"));
 153     assertEquals("jisuanji", transliterator.transliterate("计算机"));
 154
 155     assertEquals("chengjiu", transliterator.transliterate("成就"));
 156
 157   }
 158
 159   public void testArabic() {
 160     final Language ar = Language.lookup("ar");
 161     final Transliterator transliterator = Transliterator.createFromRules("", ar.getDefaultNormalizerRules(), Transliterator.FORWARD);
 162     // These don't seem quite right....
 163     assertEquals("haswb", transliterator.transliterate("حاسوب"));
 164     assertEquals("kmbywtr", transliterator.transliterate("كمبيوتر"));
 165
 166     assertEquals("{\u200e كمبيوتر \u200e}", Language.fixBidiText("{كمبيوتر}"));
 167     assertEquals("{a=\u200e كمبيوتر \u200e}", Language.fixBidiText("{a=كمبيوتر}"));
 168     assertEquals("(\u200e كمبيوتر \u200e)", Language.fixBidiText("(كمبيوتر)"));
 169     assertEquals("أنثى أنْثَى (’únθā) {f}, إناث (’ināθ) {p}, اناثى (’anāθā) {p}", Language.fixBidiText("أنثى أنْثَى (’únθā) {f}, إناث (’ināθ) {p}, اناثى (’anāθā) {p}"));
 170
 171   }
 172
 173   public void testThai() {
 174     final Language th = Language.lookup("TH");
 175     final Transliterator transliterator = Transliterator.createFromRules("", th.getDefaultNormalizerRules(), Transliterator.FORWARD);
 176     // Not sure these are right, just to know...
 177     assertEquals("d", transliterator.transliterate("ด"));
 178     assertEquals("di", transliterator.transliterate("ด ี"));
 179     assertEquals("dii", transliterator.transliterate("ดีี"));
 180
 181     assertEquals(Collections.singleton("ดีี"), DictFileParser.tokenize("ดีี", DictFileParser.NON_CHAR));
 182   }
 183
 184
 185   public void testEnWiktionaryNames() {
 186     final Set<String> enLangs = new LinkedHashSet<String>(WiktionaryLangs.isoCodeToEnWikiName.keySet());
 187     final List<String> names = new ArrayList<String>();
 188     for (final String code : WiktionaryLangs.isoCodeToEnWikiName.keySet()) {
 189       names.add(WiktionaryLangs.isoCodeToEnWikiName.get(code));
 190       enLangs.add(code.toLowerCase());
 191     }
 192     Collections.sort(names);
 193     System.out.println(names);
 194     //assertEquals(enLangs, Language.isoCodeToResources.keySet());
 195   }
 196
 197 }