Align with optimized Dictionary repo code.

[DictionaryPC.git] / src / com / hughes / android / dictionary / engine / IndexBuilder.java

diff --git a/src/com/hughes/android/dictionary/engine/IndexBuilder.java b/src/com/hughes/android/dictionary/engine/IndexBuilder.java
index e7e1b43635627d146d263dbf0c5a6ea5d85b1570..3a47ff947eaea22412b05a368c26fa996d48022d 100644 (file)
--- a/src/com/hughes/android/dictionary/engine/IndexBuilder.java
+++ b/src/com/hughes/android/dictionary/engine/IndexBuilder.java
@@ -14,16 +14,7 @@
 
 package com.hughes.android.dictionary.engine;
 
-import java.util.ArrayList;
-import java.util.Collections;
-import java.util.Comparator;
-import java.util.EnumMap;
-import java.util.HashSet;
-import java.util.List;
-import java.util.Map;
-import java.util.Set;
-import java.util.SortedMap;
-import java.util.TreeMap;
+import java.util.*;
 
 import com.hughes.android.dictionary.engine.Index.IndexEntry;
 import com.hughes.android.dictionary.parser.DictFileParser;
@@ -34,17 +25,19 @@ public class IndexBuilder {
     public final Index index;
     final Set<String> stoplist;
 
-    final SortedMap<String, TokenData> tokenToData;
+    final Map<String, TokenData> fastTokenToData;
+    final SortedMap<FastCompareString, TokenData> tokenToData;
 
     IndexBuilder(final DictionaryBuilder dictionaryBuilder, final String shortName, final String longName, final Language language, final String normalizerRules, final Set<String> stoplist, final boolean swapPairEntries) {
         this.dictionaryBuilder = dictionaryBuilder;
         index = new Index(dictionaryBuilder.dictionary, shortName, longName, language, normalizerRules, swapPairEntries, stoplist);
-        tokenToData = new TreeMap<String, TokenData>(index.getSortComparator());
+        tokenToData = new TreeMap<>(new FastNormalizeComparator(index.getSortComparator()));
+        fastTokenToData = new HashMap<>();
         this.stoplist = stoplist;
     }
 
     public void build() {
-        final Set<IndexedEntry> tokenIndexedEntries = new HashSet<IndexedEntry>();
+        final Set<IndexedEntry> tokenIndexedEntries = new HashSet<>();
         final List<RowBase> rows = index.rows;
         index.mainTokenCount = 0;
         for (final TokenData tokenData : tokenToData.values()) {
@@ -95,19 +88,13 @@ public class IndexBuilder {
                 }
 
                 final Index.IndexEntry indexEntry = new Index.IndexEntry(index, tokenData.token, index
-                        .normalizer().transliterate(tokenData.token), startRow, numRows);
-                indexEntry.htmlEntries.addAll(tokenData.htmlEntries);
+                        .normalizer().transliterate(tokenData.token), startRow, numRows, tokenData.htmlEntries);
                 index.sortedIndexEntries.add(indexEntry);
             }
         }
 
-        final List<IndexEntry> entriesSortedByNumRows = new ArrayList<IndexEntry>(index.sortedIndexEntries);
-        Collections.sort(entriesSortedByNumRows, new Comparator<IndexEntry>() {
-            @Override
-            public int compare(IndexEntry object1, IndexEntry object2) {
-                return object2.numRows - object1.numRows;
-            }
-        });
+        final List<IndexEntry> entriesSortedByNumRows = new ArrayList<>(index.sortedIndexEntries);
+        entriesSortedByNumRows.sort((object1, object2) -> object2.numRows - object1.numRows);
         System.out.println("Most common tokens:");
         for (int i = 0; i < 50 && i < entriesSortedByNumRows.size(); ++i) {
             System.out.println("  " + entriesSortedByNumRows.get(i));
@@ -117,10 +104,10 @@ public class IndexBuilder {
     public static class TokenData {
         final String token;
 
-        final Map<EntryTypeName, List<IndexedEntry>> typeToEntries = new EnumMap<EntryTypeName, List<IndexedEntry>>(EntryTypeName.class);
+        final Map<EntryTypeName, List<IndexedEntry>> typeToEntries = new EnumMap<>(EntryTypeName.class);
         public boolean hasMainEntry = false;
 
-        public List<HtmlEntry> htmlEntries = new ArrayList<HtmlEntry>();
+        public final List<HtmlEntry> htmlEntries = new ArrayList<>();
 
         TokenData(final String token) {
             assert token.equals(token.trim());
@@ -130,11 +117,16 @@ public class IndexBuilder {
     }
 
     public TokenData getOrCreateTokenData(final String token) {
-        TokenData tokenData = tokenToData.get(token);
-        if (tokenData == null) {
-            tokenData = new TokenData(token);
-            tokenToData.put(token, tokenData);
+        TokenData tokenData = fastTokenToData.get(token);
+        if (tokenData != null) return tokenData;
+        tokenData = new TokenData(token);
+        final FastCompareString c = new FastCompareString(token);
+        if (tokenToData.put(c, tokenData) != null) {
+            // The parallel HashMap assumes that the TreeMap Comparator
+            // is compatible with the equals it uses to compare.
+            throw new RuntimeException("TokenData TreeMap and HashMap out of sync, Comparator may be broken?");
         }
+        fastTokenToData.put(token, tokenData);
         return tokenData;
     }
 
@@ -145,7 +137,7 @@ public class IndexBuilder {
             tokenData.hasMainEntry = true;
         }
         if (entries == null) {
-            entries = new ArrayList<IndexedEntry>();
+            entries = new ArrayList<>();
             tokenData.typeToEntries.put(entryTypeName, entries);
         }
         return entries;