Major refactor in the way wikiText is parsed.

[DictionaryPC.git] / src / com / hughes / android / dictionary / engine / IndexBuilder.java

diff --git a/src/com/hughes/android/dictionary/engine/IndexBuilder.java b/src/com/hughes/android/dictionary/engine/IndexBuilder.java
index cab33187d57f1e9bd6ae60ba6f50c77486d5bfc6..81de5a2901e6d534a4ea63617dda451973613d24 100644 (file)
--- a/src/com/hughes/android/dictionary/engine/IndexBuilder.java
+++ b/src/com/hughes/android/dictionary/engine/IndexBuilder.java
@@ -1,3 +1,17 @@
+// Copyright 2011 Google Inc. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
 package com.hughes.android.dictionary.engine;
 
 import java.util.ArrayList;
@@ -14,18 +28,19 @@ import java.util.TreeMap;
 import com.hughes.android.dictionary.engine.Index.IndexEntry;
 import com.hughes.android.dictionary.parser.DictFileParser;
 
-
 public class IndexBuilder {
   
   final DictionaryBuilder dictionaryBuilder;
   public final Index index;
+  final Set<String> stoplist;
 
   final SortedMap<String, TokenData> tokenToData;
 
-  IndexBuilder(final DictionaryBuilder dictionaryBuilder, final String shortName, final String longName, final Language language, final String normalizerRules, final boolean swapPairEntries) {
+  IndexBuilder(final DictionaryBuilder dictionaryBuilder, final String shortName, final String longName, final Language language, final String normalizerRules, final Set<String> stoplist, final boolean swapPairEntries) {
     this.dictionaryBuilder = dictionaryBuilder;
     index = new Index(dictionaryBuilder.dictionary, shortName, longName, language, normalizerRules, swapPairEntries);
     tokenToData = new TreeMap<String, TokenData>(new NormalizeComparator(index.normalizer(), language.getCollator()));
+    this.stoplist = stoplist;
   }
   
   public void build() {
@@ -59,15 +74,15 @@ public class IndexBuilder {
           .normalizer().transliterate(tokenData.token), startRow, numRows));
     }
     
-    final List<IndexEntry> sortedEntries = new ArrayList<IndexEntry>(index.sortedIndexEntries);
-    Collections.sort(sortedEntries, new Comparator<IndexEntry>() {
+    final List<IndexEntry> entriesSortedByRows = new ArrayList<IndexEntry>(index.sortedIndexEntries);
+    Collections.sort(entriesSortedByRows, new Comparator<IndexEntry>() {
       @Override
       public int compare(IndexEntry object1, IndexEntry object2) {
         return object2.numRows - object1.numRows;
       }});
     System.out.println("Most common tokens:");
-    for (int i = 0; i < 50 && i < sortedEntries.size(); ++i) {
-      System.out.println("  " + sortedEntries.get(i));
+    for (int i = 0; i < 50 && i < entriesSortedByRows.size(); ++i) {
+      System.out.println("  " + entriesSortedByRows.get(i));
     }
   }
   
@@ -83,7 +98,7 @@ public class IndexBuilder {
     }
   }
 
-  public TokenData getOrCreateTokenData(final String token) {
+  private TokenData getOrCreateTokenData(final String token) {
     TokenData tokenData = tokenToData.get(token);
     if (tokenData == null) {
       tokenData = new TokenData(token);
@@ -92,7 +107,7 @@ public class IndexBuilder {
     return tokenData;
   }
 
-  public List<IndexedEntry> getOrCreateEntries(final String token, final EntryTypeName entryTypeName) {
+  private List<IndexedEntry> getOrCreateEntries(final String token, final EntryTypeName entryTypeName) {
     final TokenData tokenData = getOrCreateTokenData(token);
     List<IndexedEntry> entries = tokenData.typeToEntries.get(entryTypeName);
     if (entries == null) {
@@ -105,18 +120,20 @@ public class IndexBuilder {
   public void addEntryWithTokens(final IndexedEntry indexedEntry, final Set<String> tokens,
       final EntryTypeName entryTypeName) {
     for (final String token : tokens) {
+      if (entryTypeName.overridesStopList || !stoplist.contains(token))
       getOrCreateEntries(token, entryTypeName).add(indexedEntry);
     }    
   }
 
   public void addEntryWithString(final IndexedEntry indexedEntry, final String untokenizedString,
-      final EntryTypeName singleTokenEntryTypeName, final EntryTypeName multiTokenEntryTypeName) {
+      final EntryTypeName entryTypeName) {
     final Set<String> tokens = DictFileParser.tokenize(untokenizedString, DictFileParser.NON_CHAR);
-    addEntryWithTokens(indexedEntry, tokens, tokens.size() == 1 ? singleTokenEntryTypeName : multiTokenEntryTypeName);
+    addEntryWithTokens(indexedEntry, tokens, tokens.size() == 1 ? entryTypeName.singleWordInstance : entryTypeName);
   }
 
-  public void addEntryWithString(final IndexedEntry indexedEntry, final String untokenizedString,
+  public void addEntryWithStringNoSingle(final IndexedEntry indexedEntry, final String untokenizedString,
       final EntryTypeName entryTypeName) {
-    addEntryWithString(indexedEntry, untokenizedString, entryTypeName, entryTypeName);
+    final Set<String> tokens = DictFileParser.tokenize(untokenizedString, DictFileParser.NON_CHAR);
+    addEntryWithTokens(indexedEntry, tokens, entryTypeName);
   }
 }