X-Git-Url: http://gitweb.fperrin.net/?a=blobdiff_plain;f=src%2Fcom%2Fhughes%2Fandroid%2Fdictionary%2Fengine%2FIndexBuilder.java;h=2db537bcd0c548bef2dabe22d245cae511a4451c;hb=9893c443e12eb0d52b0937de519d1df1c885196c;hp=e7e1b43635627d146d263dbf0c5a6ea5d85b1570;hpb=e479ba38bbcb261951399326623c20ffacc147d4;p=DictionaryPC.git

diff --git a/src/com/hughes/android/dictionary/engine/IndexBuilder.java b/src/com/hughes/android/dictionary/engine/IndexBuilder.java
index e7e1b43..2db537b 100644
--- a/src/com/hughes/android/dictionary/engine/IndexBuilder.java
+++ b/src/com/hughes/android/dictionary/engine/IndexBuilder.java
@@ -14,16 +14,7 @@
 
 package com.hughes.android.dictionary.engine;
 
-import java.util.ArrayList;
-import java.util.Collections;
-import java.util.Comparator;
-import java.util.EnumMap;
-import java.util.HashSet;
-import java.util.List;
-import java.util.Map;
-import java.util.Set;
-import java.util.SortedMap;
-import java.util.TreeMap;
+import java.util.*;
 
 import com.hughes.android.dictionary.engine.Index.IndexEntry;
 import com.hughes.android.dictionary.parser.DictFileParser;
@@ -34,17 +25,19 @@ public class IndexBuilder {
     public final Index index;
     final Set<String> stoplist;
 
-    final SortedMap<String, TokenData> tokenToData;
+    final Map<String, TokenData> fastTokenToData;
+    final SortedMap<FastCompareString, TokenData> tokenToData;
 
     IndexBuilder(final DictionaryBuilder dictionaryBuilder, final String shortName, final String longName, final Language language, final String normalizerRules, final Set<String> stoplist, final boolean swapPairEntries) {
         this.dictionaryBuilder = dictionaryBuilder;
         index = new Index(dictionaryBuilder.dictionary, shortName, longName, language, normalizerRules, swapPairEntries, stoplist);
-        tokenToData = new TreeMap<String, TokenData>(index.getSortComparator());
+        tokenToData = new TreeMap<>(new FastNormalizeComparator(index.getSortComparator()));
+        fastTokenToData = new HashMap<>();
         this.stoplist = stoplist;
     }
 
     public void build() {
-        final Set<IndexedEntry> tokenIndexedEntries = new HashSet<IndexedEntry>();
+        final Set<IndexedEntry> tokenIndexedEntries = new HashSet<>();
         final List<RowBase> rows = index.rows;
         index.mainTokenCount = 0;
         for (final TokenData tokenData : tokenToData.values()) {
@@ -101,13 +94,8 @@ public class IndexBuilder {
             }
         }
 
-        final List<IndexEntry> entriesSortedByNumRows = new ArrayList<IndexEntry>(index.sortedIndexEntries);
-        Collections.sort(entriesSortedByNumRows, new Comparator<IndexEntry>() {
-            @Override
-            public int compare(IndexEntry object1, IndexEntry object2) {
-                return object2.numRows - object1.numRows;
-            }
-        });
+        final List<IndexEntry> entriesSortedByNumRows = new ArrayList<>(index.sortedIndexEntries);
+        entriesSortedByNumRows.sort((object1, object2) -> object2.numRows - object1.numRows);
         System.out.println("Most common tokens:");
         for (int i = 0; i < 50 && i < entriesSortedByNumRows.size(); ++i) {
             System.out.println("  " + entriesSortedByNumRows.get(i));
@@ -117,10 +105,10 @@ public class IndexBuilder {
     public static class TokenData {
         final String token;
 
-        final Map<EntryTypeName, List<IndexedEntry>> typeToEntries = new EnumMap<EntryTypeName, List<IndexedEntry>>(EntryTypeName.class);
+        final Map<EntryTypeName, List<IndexedEntry>> typeToEntries = new EnumMap<>(EntryTypeName.class);
         public boolean hasMainEntry = false;
 
-        public List<HtmlEntry> htmlEntries = new ArrayList<HtmlEntry>();
+        public final List<HtmlEntry> htmlEntries = new ArrayList<>();
 
         TokenData(final String token) {
             assert token.equals(token.trim());
@@ -130,11 +118,16 @@ public class IndexBuilder {
     }
 
     public TokenData getOrCreateTokenData(final String token) {
-        TokenData tokenData = tokenToData.get(token);
-        if (tokenData == null) {
-            tokenData = new TokenData(token);
-            tokenToData.put(token, tokenData);
+        TokenData tokenData = fastTokenToData.get(token);
+        if (tokenData != null) return tokenData;
+        tokenData = new TokenData(token);
+        final FastCompareString c = new FastCompareString(token);
+        if (tokenToData.put(c, tokenData) != null) {
+            // The parallel HashMap assumes that the TreeMap Comparator
+            // is compatible with the equals it uses to compare.
+            throw new RuntimeException("TokenData TreeMap and HashMap out of sync, Comparator may be broken?");
         }
+        fastTokenToData.put(token, tokenData);
         return tokenData;
     }
 
@@ -145,7 +138,7 @@ public class IndexBuilder {
             tokenData.hasMainEntry = true;
         }
         if (entries == null) {
-            entries = new ArrayList<IndexedEntry>();
+            entries = new ArrayList<>();
             tokenData.typeToEntries.put(entryTypeName, entries);
         }
         return entries;