go

diff --git a/src/com/hughes/android/dictionary/DictionaryBuilder.java b/src/com/hughes/android/dictionary/DictionaryBuilder.java
index 8ff741e6f05053de707ad150a28aede1200cdaf4..bb1301ee1a9789b02a920d59286ea24f90b4a607 100755 (executable)
--- a/src/com/hughes/android/dictionary/DictionaryBuilder.java
+++ b/src/com/hughes/android/dictionary/DictionaryBuilder.java
@@ -36,7 +36,7 @@ public class DictionaryBuilder {
     processInputFile("c:\\de-en-chemnitz.txt", dict, true, Charset.forName("UTF8"));\r
     \r
     // Thad's extra sauce: \r
-    processInputFile("c:\\de-en-dictcc.txt", dict, false, Charset.forName("Cp1252"));\r
+//    processInputFile("c:\\de-en-dictcc.txt", dict, false, Charset.forName("Cp1252"));\r
 \r
     createIndex(dict, Entry.LANG1);\r
     createIndex(dict, Entry.LANG2);\r

diff --git a/src/com/hughes/android/dictionary/DictionaryTest.java b/src/com/hughes/android/dictionary/DictionaryTest.java
index 1578fd2b19553ba2f55af0f28e67c54260d560e7..84bdd5e1b3ec5612176b3e98f0e0a4f70cf61eb4 100755 (executable)
--- a/src/com/hughes/android/dictionary/DictionaryTest.java
+++ b/src/com/hughes/android/dictionary/DictionaryTest.java
@@ -178,4 +178,26 @@ public class DictionaryTest extends TestCase {
     }\r
   }\r
 \r
+  public void testEnglishSort() {\r
+\r
+    final List<String> words = Arrays.asList(\r
+        "preppie", \r
+        "preppy",\r
+        "pre-print", \r
+        "preprocess");\r
+    \r
+    final List<String> sorted = new ArrayList<String>(words);\r
+    Collections.sort(sorted, Language.EN.sortComparator);\r
+    for (int i = 0; i < words.size(); ++i) {\r
+      if (i > 0) {\r
+        assertTrue(Language.EN.sortComparator.compare(words.get(i-1), words.get(i)) < 0);\r
+      }\r
+      System.out.println(words.get(i) + "\t" + sorted.get(i));\r
+      assertEquals(words.get(i), sorted.get(i));\r
+    }\r
+    \r
+    assertTrue(Language.EN.sortCollator.compare("preppy", "pre-print") < 0);\r
+\r
+  }\r
+\r
 }\r

diff --git a/src/com/hughes/android/dictionary/ZIndexBuilder.java b/src/com/hughes/android/dictionary/ZIndexBuilder.java
deleted file mode 100755 (executable)
index 5d4b26a..0000000
--- a/src/com/hughes/android/dictionary/ZIndexBuilder.java
+++ /dev/null
@@ -1,117 +0,0 @@
-package com.hughes.android.dictionary;\r
-\r
-\r
-public class ZIndexBuilder {\r
-\r
-//  static final Pattern WHITESPACE = Pattern.compile("\\s+");\r
-//  static final Pattern NONALPHA = Pattern.compile("[^A-Za-z]+");\r
-//\r
-//  public static void main(String[] args) throws IOException,\r
-//      ClassNotFoundException {\r
-//    if (args.length != 1) {\r
-//      System.err.println("No input file.");\r
-//      return;\r
-//    }\r
-//    final String dictionaryFileName = args[0];\r
-//    createIndex(dictionaryFileName, Entry.LANG1);\r
-//    createIndex(dictionaryFileName, Entry.LANG2);\r
-//  }\r
-//\r
-//  private static void createIndex(final String dictionaryFileName,\r
-//      final byte lang) throws IOException, FileNotFoundException,\r
-//      ClassNotFoundException {\r
-//    Node rootBuilder;\r
-//    rootBuilder = processDictionaryLines(dictionaryFileName, lang);\r
-//    FileUtil.write(rootBuilder, String.format("%s_builder_%d.serialized", dictionaryFileName, lang));\r
-//    rootBuilder = (Node) FileUtil.read(String.format("%s_builder_%d.serialized", dictionaryFileName, lang));\r
-//\r
-//    rootBuilder.forEachNode(new Function<Node>() {\r
-//      @Override\r
-//      public void invoke(final Node node) {\r
-//        for (final List<EntryDescriptor> entryDescriptors : node.entryDescriptorsMap.values()) {\r
-//          Collections.sort(entryDescriptors);\r
-//        }\r
-//      }});\r
-//    \r
-//    // Dump twice to get accurate file locations.\r
-//    for (int i = 0; i < 2; ++i) {\r
-//      final RandomAccessFile raf = new RandomAccessFile(String.format(Dictionary.INDEX_FORMAT, dictionaryFileName, lang), "rw"); \r
-//      rootBuilder.dump(raf);\r
-//      raf.close();\r
-//    }\r
-//  }\r
-//\r
-//  // ----------------------------------------------------------------\r
-//  \r
-//  static final class EntryDescriptor implements Comparable<EntryDescriptor>, Serializable {\r
-//    final int offset;\r
-//    final int numTokens;\r
-//    public EntryDescriptor(int offset, int numTokens) {\r
-//      this.offset = offset;\r
-//      this.numTokens = numTokens;\r
-//    }\r
-//    @Override\r
-//    public boolean equals(Object obj) {\r
-//      final EntryDescriptor that = (EntryDescriptor) obj;\r
-//      return this.offset == that.offset;\r
-//    }\r
-//    @Override\r
-//    public int hashCode() {\r
-//      return offset;\r
-//    }\r
-//    @Override\r
-//    public int compareTo(EntryDescriptor o) {\r
-//      return this.numTokens < o.numTokens ? -1 : this.numTokens == o.numTokens ? 0 : 1;\r
-//    }\r
-//  }\r
-//\r
-//\r
-//  // ----------------------------------------------------------------\r
-//\r
-//  static Node processDictionaryLines(final String dictionaryFileName, final byte lang) throws IOException {\r
-//    final Node root = new Node("");\r
-//    final RandomAccessFile dictionaryFile = new RandomAccessFile(dictionaryFileName, "r");\r
-//    String line;\r
-//    final Entry entry = new Entry();\r
-//    int lineCount = 0;\r
-//    long fileLocation = 0;\r
-//    while ((line = dictionaryFile.readLine()) != null) {\r
-//      assert ((int) fileLocation) == fileLocation;\r
-//\r
-//      line = line.trim();\r
-//      if (line.isEmpty() || line.startsWith("#") || !entry.parseFromLine(line)) {\r
-//        continue;\r
-//      }\r
-//      final String text = entry.getIndexableText(Entry.LANG1);\r
-//      final String[] tokens = WHITESPACE.split(text);\r
-//      final Map<String,String> tokenToNormalizedMap = new LinkedHashMap<String,String>();\r
-//      for (String token : tokens) {\r
-//        if (token.length() <= 1 || !Character.isLetter(token.charAt(0))) {\r
-//          continue;\r
-//        }\r
-//        tokenToNormalizedMap.put(token, EntryFactory.entryFactory.normalizeToken(token));\r
-//      }\r
-//      for (final Map.Entry<String, String> tokenToNormalized : tokenToNormalizedMap.entrySet()) {\r
-//        final String normalizedToken = tokenToNormalized.getValue();\r
-//        final Node node = root.getNode(normalizedToken, 0, true);\r
-//        node.addToken(tokenToNormalized.getKey(), new EntryDescriptor((int) fileLocation, tokens.length));\r
-//        assert node == root.getNode(normalizedToken, 0, false);\r
-//        assert normalizedToken\r
-//            .equals(root.getNode(normalizedToken, 0, false).normalizedToken);\r
-//      }\r
-//\r
-//      if (lineCount % 10000 == 0) {\r
-//        System.out.println("IndexBuilder: " + "lineCount=" + lineCount);\r
-//      }\r
-//      \r
-//      lineCount++;\r
-//      fileLocation = dictionaryFile.getFilePointer();\r
-//    }\r
-//    dictionaryFile.close();\r
-//    \r
-//    root.recursiveSetDescendantCounts();\r
-//    \r
-//    return root;\r
-//  }\r
-\r
-}\r

diff --git a/src/com/hughes/android/dictionary/ZIndexTest.java b/src/com/hughes/android/dictionary/ZIndexTest.java
deleted file mode 100755 (executable)
index 3196093..0000000
--- a/src/com/hughes/android/dictionary/ZIndexTest.java
+++ /dev/null
@@ -1,81 +0,0 @@
-package com.hughes.android.dictionary;\r
-//package com.hughes.android.dictionary;\r
-//\r
-//import java.io.IOException;\r
-//import java.io.RandomAccessFile;\r
-//import java.util.LinkedHashSet;\r
-//import java.util.Set;\r
-//\r
-//import junit.framework.TestCase;\r
-//\r
-//import com.hughes.android.dictionary.Index.Node;\r
-//import com.hughes.util.FileUtil;\r
-//\r
-//public class IndexTest extends TestCase {\r
-//\r
-//  static final String file = "c:\\dict-de-en.txt";\r
-//  static final String file_index = file + "_index_0";\r
-//  \r
-//  public void testRoot() throws IOException {\r
-//    System.out.println("  testRoot");\r
-//    final Index index = new Index(file_index);\r
-//    final Node node = index.lookup("");\r
-//    assertNotNull(node);\r
-//    \r
-//    assertEquals(312220, node.descendantTokenCount);\r
-//    assertEquals(1087063, node.descendantEntryCount);\r
-//    \r
-//    for (final String token : node.tokenToOffsets.keySet()) {\r
-//      System.out.println(token);\r
-//      assertTrue(token.toLowerCase().contains("handhubwagen"));\r
-//    }\r
-//  }\r
-//  \r
-//  public void testLookup() throws IOException {\r
-//    System.out.println("  testLookup");\r
-//    final Index index = new Index(file_index);\r
-//    final Node node = index.lookup("handhubwagen");\r
-//    assertNotNull(node);\r
-//    \r
-//    assertEquals(1, node.descendantTokenCount);\r
-//    assertEquals(2, node.descendantEntryCount);\r
-//    \r
-//    for (final String token : node.tokenToOffsets.keySet()) {\r
-//      System.out.println(token);\r
-//      assertTrue(token.toLowerCase().contains("handhubwagen"));\r
-//    }\r
-//  }\r
-//\r
-//  public void testGetDescendantOffsets() throws IOException {\r
-//    System.out.println("  testGetDescendantOffsets");\r
-//    final Index index = new Index(file_index);\r
-//    \r
-//    final Node node = index.lookup("handhebe");\r
-//    assertNotNull(node);\r
-//    assertEquals("handhebel", node.nodeHandle.normalizedToken);\r
-//    final Set<Integer> offsets = new LinkedHashSet<Integer>();\r
-//    node.getDescendantEntryOffsets(offsets, 10);\r
-//    final RandomAccessFile raf = new RandomAccessFile(file, "r");\r
-//    for (final Integer offset : offsets) {\r
-//      final String entry = FileUtil.readLine(raf, offset);\r
-//      System.out.println(entry);\r
-//      assertTrue(entry.toLowerCase().contains(node.nodeHandle.normalizedToken));\r
-//    }\r
-//  }\r
-//\r
-//  public void testGetDescendants() throws IOException {\r
-//    System.out.println("  testGetDescendant");\r
-//    final Index index = new Index(file_index);\r
-//    final RandomAccessFile raf = new RandomAccessFile(file, "r");\r
-//    for (int i = 1000000; i < 1000050; ++i) {\r
-//      final Object o = index.root.getDescendant(i);\r
-//      if (o instanceof Integer) {\r
-//        System.out.println("  " + FileUtil.readLine(raf, (Integer)o));\r
-//      } else {\r
-//        System.out.println(o);\r
-//      }\r
-//    }\r
-//    raf.close();\r
-//  }\r
-//\r
-//}\r