Align with optimized Dictionary repo code.

[DictionaryPC.git] / src / com / hughes / android / dictionary / engine / IndexBuilder.java

diff --git a/src/com/hughes/android/dictionary/engine/IndexBuilder.java b/src/com/hughes/android/dictionary/engine/IndexBuilder.java
index 680cf4adc5dee305793dced0a21c5b4be27ed0a8..3a47ff947eaea22412b05a368c26fa996d48022d 100644 (file)
--- a/src/com/hughes/android/dictionary/engine/IndexBuilder.java
+++ b/src/com/hughes/android/dictionary/engine/IndexBuilder.java
@@ -14,16 +14,7 @@
 
 package com.hughes.android.dictionary.engine;
 
-import java.util.ArrayList;
-import java.util.Collections;
-import java.util.Comparator;
-import java.util.EnumMap;
-import java.util.HashSet;
-import java.util.List;
-import java.util.Map;
-import java.util.Set;
-import java.util.SortedMap;
-import java.util.TreeMap;
+import java.util.*;
 
 import com.hughes.android.dictionary.engine.Index.IndexEntry;
 import com.hughes.android.dictionary.parser.DictFileParser;
@@ -34,12 +25,14 @@ public class IndexBuilder {
     public final Index index;
     final Set<String> stoplist;
 
-    final SortedMap<String, TokenData> tokenToData;
+    final Map<String, TokenData> fastTokenToData;
+    final SortedMap<FastCompareString, TokenData> tokenToData;
 
     IndexBuilder(final DictionaryBuilder dictionaryBuilder, final String shortName, final String longName, final Language language, final String normalizerRules, final Set<String> stoplist, final boolean swapPairEntries) {
         this.dictionaryBuilder = dictionaryBuilder;
         index = new Index(dictionaryBuilder.dictionary, shortName, longName, language, normalizerRules, swapPairEntries, stoplist);
-        tokenToData = new TreeMap<>(index.getSortComparator());
+        tokenToData = new TreeMap<>(new FastNormalizeComparator(index.getSortComparator()));
+        fastTokenToData = new HashMap<>();
         this.stoplist = stoplist;
     }
 
@@ -95,8 +88,7 @@ public class IndexBuilder {
                 }
 
                 final Index.IndexEntry indexEntry = new Index.IndexEntry(index, tokenData.token, index
-                        .normalizer().transliterate(tokenData.token), startRow, numRows);
-                indexEntry.htmlEntries.addAll(tokenData.htmlEntries);
+                        .normalizer().transliterate(tokenData.token), startRow, numRows, tokenData.htmlEntries);
                 index.sortedIndexEntries.add(indexEntry);
             }
         }
@@ -125,11 +117,16 @@ public class IndexBuilder {
     }
 
     public TokenData getOrCreateTokenData(final String token) {
-        TokenData tokenData = tokenToData.get(token);
-        if (tokenData == null) {
-            tokenData = new TokenData(token);
-            tokenToData.put(token, tokenData);
+        TokenData tokenData = fastTokenToData.get(token);
+        if (tokenData != null) return tokenData;
+        tokenData = new TokenData(token);
+        final FastCompareString c = new FastCompareString(token);
+        if (tokenToData.put(c, tokenData) != null) {
+            // The parallel HashMap assumes that the TreeMap Comparator
+            // is compatible with the equals it uses to compare.
+            throw new RuntimeException("TokenData TreeMap and HashMap out of sync, Comparator may be broken?");
         }
+        fastTokenToData.put(token, tokenData);
         return tokenData;
     }