go

diff --git a/src/com/hughes/android/dictionary/Dictionary.java b/src/com/hughes/android/dictionary/Dictionary.java
index 8a68d6cf92c868eff450f474fd848e0775d9306a..7fd38f530514997370f09e056ee804b15980b64a 100755 (executable)
--- a/src/com/hughes/android/dictionary/Dictionary.java
+++ b/src/com/hughes/android/dictionary/Dictionary.java
@@ -100,10 +100,10 @@ public final class Dictionary implements RAFSerializable<Dictionary> {
         }\r
         final IndexEntry midEntry = sortedIndex.get(mid);\r
 \r
-        final int comp = language.tokenComparator.compare(word, midEntry.word.toLowerCase());\r
+        final int comp = language.sortComparator.compare(word, midEntry.word.toLowerCase());\r
         if (comp == 0) {\r
           int result = mid;\r
-          while (result > 0 && language.tokenComparator.compare(word, sortedIndex.get(result - 1).word.toLowerCase()) == 0) {\r
+          while (result > 0 && language.findComparator.compare(word, sortedIndex.get(result - 1).word.toLowerCase()) == 0) {\r
             --result;\r
             if (interrupted.get()) {\r
               return result;\r

diff --git a/src/com/hughes/android/dictionary/Entry.java b/src/com/hughes/android/dictionary/Entry.java
index 6143b4948d338c9e7f50b24b2ba04e2075a8e6b2..3facbf5752e5fc8e7c98167bdc8d1f65e62d0243 100755 (executable)
--- a/src/com/hughes/android/dictionary/Entry.java
+++ b/src/com/hughes/android/dictionary/Entry.java
@@ -152,6 +152,11 @@ public final class Entry implements RAFSerializable<Entry> {
     text = text.replaceAll("[\"/\\()<>\\[\\],;?!.]", " ");\r
     text = text.replaceAll("[:] ", " ");\r
     text = text.replaceAll(" [:]", " ");\r
+    \r
+    // Now be really conservative about what we allow inside a token:\r
+    // See: http://unicode.org/Public/UNIDATA/UCD.html#General_Category_Values\r
+    text = text.replaceAll("[^-:\\p{Lu}\\p{Ll}\\p{Lt}\\p{Lm}\\p{Lo}\\p{Nd}\\p{Nl}\\p{No}]", " ");\r
+    \r
     result.addAll(Arrays.asList(WHITESPACE.split(text)));\r
 \r
     text = text.replaceAll("[-]", " ");\r

diff --git a/src/com/hughes/android/dictionary/Language.java b/src/com/hughes/android/dictionary/Language.java
index 122372de6815d99a0230f901ee99a2b9488652d2..9591bc08742a96d7c8d01e092dc03d0d6f8e9233 100755 (executable)
--- a/src/com/hughes/android/dictionary/Language.java
+++ b/src/com/hughes/android/dictionary/Language.java
@@ -1,81 +1,82 @@
 package com.hughes.android.dictionary;\r
 \r
+import java.text.Collator;\r
 import java.util.Comparator;\r
 import java.util.LinkedHashMap;\r
+import java.util.Locale;\r
 import java.util.Map;\r
 \r
-import com.hughes.util.StringUtil;\r
-\r
-public abstract class Language {\r
+public class Language {\r
 \r
   final String symbol;\r
-  final Comparator<String> tokenComparator;\r
+  final Locale locale;\r
+\r
+  final Collator sortCollator;\r
+  final Comparator<String> sortComparator;\r
 \r
-  public Language(final String symbol) {\r
+  final Collator findCollator;\r
+  final Comparator<String> findComparator;\r
+\r
+  public Language(final String symbol, final Locale locale) {\r
     this.symbol = symbol;\r
-    this.tokenComparator = new Comparator<String>() {\r
+    this.locale = locale;\r
+\r
+    this.sortCollator = Collator.getInstance(locale);\r
+    this.sortCollator.setDecomposition(Collator.CANONICAL_DECOMPOSITION);\r
+    this.sortCollator.setStrength(Collator.IDENTICAL);\r
+    this.sortComparator = new Comparator<String>() {\r
       public int compare(final String s1, final String s2) {\r
-        final String norm1 = normalizeTokenForSort(s1);\r
-        final String norm2 = normalizeTokenForSort(s2);\r
-        final int c = norm1.compareTo(norm2);\r
-        if (c != 0) {\r
-          return c;\r
-        }\r
-        return StringUtil.flipCase(StringUtil.reverse(s1)).compareTo(StringUtil.flipCase(StringUtil.reverse(s2)));\r
-      }};\r
+        return sortCollator.compare(textNorm(s1), textNorm(s2));\r
+      }\r
+    };\r
+\r
+    this.findCollator = Collator.getInstance(locale);\r
+    this.findCollator.setDecomposition(Collator.CANONICAL_DECOMPOSITION);\r
+    this.findCollator.setStrength(Collator.SECONDARY);\r
+    this.findComparator = new Comparator<String>() {\r
+      public int compare(final String s1, final String s2) {\r
+        return findCollator.compare(textNorm(s1), textNorm(s2));\r
+      }\r
+    };\r
+\r
   }\r
-  \r
+\r
+  public String textNorm(final String s) {\r
+    return s;\r
+  }\r
+\r
   @Override\r
   public String toString() {\r
     return symbol;\r
   }\r
 \r
-  abstract String normalizeTokenForSort(final String token);\r
+  // ----------------------------------------------------------------\r
 \r
+  public static final Language EN = new Language("EN", Locale.ENGLISH);\r
 \r
-  // ----------------------------------------------------------------\r
-  \r
-  static final String normalizeTokenForSort(final String token, final boolean vowelETranslation) {\r
-    final StringBuilder result = new StringBuilder();\r
-    for (int i = 0; i < token.length(); ++i) {\r
-      Character c = token.charAt(i);\r
-      c = Character.toLowerCase(c);\r
-      // only check for lowercase 'e' in subsequent position means don't treat acronyms as umlauted: SAE.\r
-      if (vowelETranslation && (c == 'a' || c == 'o' || c == 'u') && i + 1 < token.length() && token.charAt(i + 1) == 'e') {\r
-        if (c == 'a') {\r
-          result.append('ä');\r
-        } else if (c == 'o') {\r
-          result.append('ö');\r
-        } else if (c == 'u') {\r
-          result.append('ü');\r
+  public static final Language DE = new Language("DE", Locale.GERMAN) {\r
+    @Override\r
+    public String textNorm(String token) {\r
+      boolean sub = false;\r
+      for (int ePos = token.indexOf('e', 1); ePos != -1; ePos = token.indexOf(\r
+          'e', ePos + 1)) {\r
+        final char pre = Character.toLowerCase(token.charAt(ePos - 1));\r
+        if (pre == 'a' || pre == 'o' || pre == 'u') {\r
+          sub = true;\r
+          break;\r
         }\r
-        ++i;\r
-      } else if (c >= 'a' && c <= 'z' || c >= '0' && c <= '9') {\r
-        result.append(c);\r
-      } else if (c == 'ß') {\r
-        result.append("ss");\r
-      } else if (c == 'ä') {\r
-        result.append(c);\r
-      } else if (c == 'ö') {\r
-        result.append(c);\r
-      } else if (c == 'ü') {\r
-        result.append(c);\r
       }\r
-    }\r
-    return result.toString();\r
-  }\r
+      if (!sub) {\r
+        return token;\r
+      }\r
+      token = token.replaceAll("ae", "ä");\r
+      token = token.replaceAll("oe", "ö");\r
+      token = token.replaceAll("ue", "ü");\r
 \r
-  public static final Language EN = new Language("EN") {\r
-    @Override\r
-    public String normalizeTokenForSort(final String token) {\r
-      return Language.normalizeTokenForSort(token, false);\r
-    }\r
-  };\r
-    \r
-  public static final Language DE = new Language("DE") {\r
-    @Override\r
-    String normalizeTokenForSort(final String token) {\r
-      return Language.normalizeTokenForSort(token, true);\r
+      token = token.replaceAll("Ae", "Ä");\r
+      token = token.replaceAll("Oe", "Ö");\r
+      token = token.replaceAll("Ue", "Ü");\r
+      return token;\r
     }\r
   };\r
 \r
@@ -87,10 +88,9 @@ public abstract class Language {
     symbolToLangauge.put(EN.symbol, EN);\r
     symbolToLangauge.put(DE.symbol, DE);\r
   }\r
-  \r
+\r
   static Language lookup(final String symbol) {\r
     return symbolToLangauge.get(symbol);\r
   }\r
 \r
-\r
 }\r