X-Git-Url: http://gitweb.fperrin.net/?a=blobdiff_plain;f=src%2Fcom%2Fhughes%2Fandroid%2Fdictionary%2Fengine%2FIndexBuilder.java;h=32a087f47390d48e1c9ae23d928c07c4449fe735;hb=d06b99b469b18cfa4a8a4bd45d51ee4ebd7efaca;hp=2db62905eb899342e22ba4f6eb84557c45b233ee;hpb=b256bde15d0f6e6cb5325fbe22c76496e40194eb;p=DictionaryPC.git

diff --git a/src/com/hughes/android/dictionary/engine/IndexBuilder.java b/src/com/hughes/android/dictionary/engine/IndexBuilder.java
index 2db6290..32a087f 100644
--- a/src/com/hughes/android/dictionary/engine/IndexBuilder.java
+++ b/src/com/hughes/android/dictionary/engine/IndexBuilder.java
@@ -1,6 +1,22 @@
+// Copyright 2011 Google Inc. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
 package com.hughes.android.dictionary.engine;
 
 import java.util.ArrayList;
+import java.util.Collections;
+import java.util.Comparator;
 import java.util.EnumMap;
 import java.util.HashSet;
 import java.util.List;
@@ -9,51 +25,71 @@ import java.util.Set;
 import java.util.SortedMap;
 import java.util.TreeMap;
 
+import com.hughes.android.dictionary.engine.Index.IndexEntry;
+import com.hughes.android.dictionary.parser.DictFileParser;
 
 public class IndexBuilder {
   
   final DictionaryBuilder dictionaryBuilder;
-  final Index index;
+  public final Index index;
+  final Set<String> stoplist;
 
   final SortedMap<String, TokenData> tokenToData;
 
-  @SuppressWarnings("unchecked")
-  IndexBuilder(final DictionaryBuilder dictionaryBuilder, final String shortName, final String longName, final Language language, final boolean swapPairEntries) {
+  IndexBuilder(final DictionaryBuilder dictionaryBuilder, final String shortName, final String longName, final Language language, final String normalizerRules, final Set<String> stoplist, final boolean swapPairEntries) {
     this.dictionaryBuilder = dictionaryBuilder;
-    index = new Index(dictionaryBuilder.dictionary, shortName, longName, language, swapPairEntries);
-    tokenToData = new TreeMap<String, TokenData>(language.getSortCollator());
+    index = new Index(dictionaryBuilder.dictionary, shortName, longName, language, normalizerRules, swapPairEntries);
+    tokenToData = new TreeMap<String, TokenData>(new NormalizeComparator(index.normalizer(), language.getCollator()));
+    this.stoplist = stoplist;
   }
   
   public void build() {
-    final Set<EntryData> tokenEntryDatas = new HashSet<EntryData>();
+    final Set<IndexedEntry> tokenEntryDatas = new HashSet<IndexedEntry>();
     final List<RowBase> rows = index.rows;
     for (final TokenData tokenData : tokenToData.values()) {
       tokenEntryDatas.clear();
-      final int indexRow = index.sortedIndexEntries.size();
-      index.sortedIndexEntries.add(new Index.IndexEntry(tokenData.token, rows.size()));
-      rows.add(new TokenRow(indexRow, rows.size(), index));
-      System.out.println("Added TokenRow: " + rows.get(rows.size() - 1));
-      int count = 0;
-      System.out.println("TOKEN: " + tokenData.token);
-      for (final Map.Entry<EntryTypeName, List<EntryData>> typeToEntry : tokenData.typeToEntries.entrySet()) {
-        for (final EntryData entryData : typeToEntry.getValue()) {
+      final int indexIndex = index.sortedIndexEntries.size();
+      final int startRow = rows.size();
+      rows.add(new TokenRow(indexIndex, rows.size(), index));
+//      System.out.println("Added TokenRow: " + rows.get(rows.size() - 1));
+      int numRows = 0;
+//      System.out.println("TOKEN: " + tokenData.token);
+      for (final Map.Entry<EntryTypeName, List<IndexedEntry>> typeToEntry : tokenData.typeToEntries.entrySet()) {
+        for (final IndexedEntry entryData : typeToEntry.getValue()) {
+          if (entryData.index() == -1) {
+            entryData.addToDictionary(dictionaryBuilder.dictionary);
+            assert entryData.index() >= 0;
+          }
           if (tokenEntryDatas.add(entryData)) {
             rows.add(new PairEntry.Row(entryData.index(), rows.size(), index));
-            ++count;
+            ++numRows;
             
-            System.out.print("  " + typeToEntry.getKey() + ": ");
-            rows.get(rows.size() - 1).print(System.out);
-            System.out.println();
+//            System.out.print("  " + typeToEntry.getKey() + ": ");
+  //          rows.get(rows.size() - 1).print(System.out);
+//            System.out.println();
           }
         }
       }
+      index.sortedIndexEntries.add(new Index.IndexEntry(tokenData.token, index
+          .normalizer().transliterate(tokenData.token), startRow, numRows));
+    }
+    
+    final List<IndexEntry> entriesSortedByRows = new ArrayList<IndexEntry>(index.sortedIndexEntries);
+    Collections.sort(entriesSortedByRows, new Comparator<IndexEntry>() {
+      @Override
+      public int compare(IndexEntry object1, IndexEntry object2) {
+        return object2.numRows - object1.numRows;
+      }});
+    System.out.println("Most common tokens:");
+    for (int i = 0; i < 50 && i < entriesSortedByRows.size(); ++i) {
+      System.out.println("  " + entriesSortedByRows.get(i));
     }
   }
   
   static class TokenData {
     final String token;
         
-    final Map<EntryTypeName, List<EntryData>> typeToEntries = new EnumMap<EntryTypeName, List<EntryData>>(EntryTypeName.class);
+    final Map<EntryTypeName, List<IndexedEntry>> typeToEntries = new EnumMap<EntryTypeName, List<IndexedEntry>>(EntryTypeName.class);
     
     TokenData(final String token) {
       assert token.equals(token.trim());
@@ -62,7 +98,7 @@ public class IndexBuilder {
     }
   }
 
-  public TokenData getOrCreateTokenData(final String token) {
+  private TokenData getOrCreateTokenData(final String token) {
     TokenData tokenData = tokenToData.get(token);
     if (tokenData == null) {
       tokenData = new TokenData(token);
@@ -71,15 +107,37 @@ public class IndexBuilder {
     return tokenData;
   }
 
-  public List<EntryData> getOrCreateEntries(final String token, final EntryTypeName entryTypeName) {
+  private List<IndexedEntry> getOrCreateEntries(final String token, final EntryTypeName entryTypeName) {
     final TokenData tokenData = getOrCreateTokenData(token);
-    List<EntryData> entries = tokenData.typeToEntries.get(entryTypeName);
+    List<IndexedEntry> entries = tokenData.typeToEntries.get(entryTypeName);
     if (entries == null) {
-      entries = new ArrayList<EntryData>();
+      entries = new ArrayList<IndexedEntry>();
       tokenData.typeToEntries.put(entryTypeName, entries);
     }
     return entries;
   }
-  
 
+  public void addEntryWithTokens(final IndexedEntry indexedEntry, final Set<String> tokens,
+      final EntryTypeName entryTypeName) {
+    if (indexedEntry == null) {
+      System.out.println("asdfasdf");
+    }
+    assert indexedEntry != null;
+    for (final String token : tokens) {
+      if (entryTypeName.overridesStopList || !stoplist.contains(token))
+      getOrCreateEntries(token, entryTypeName).add(indexedEntry);
+    }    
+  }
+
+  public void addEntryWithString(final IndexedEntry indexedEntry, final String untokenizedString,
+      final EntryTypeName entryTypeName) {
+    final Set<String> tokens = DictFileParser.tokenize(untokenizedString, DictFileParser.NON_CHAR);
+    addEntryWithTokens(indexedEntry, tokens, tokens.size() == 1 ? entryTypeName.singleWordInstance : entryTypeName);
+  }
+
+  public void addEntryWithStringNoSingle(final IndexedEntry indexedEntry, final String untokenizedString,
+      final EntryTypeName entryTypeName) {
+    final Set<String> tokens = DictFileParser.tokenize(untokenizedString, DictFileParser.NON_CHAR);
+    addEntryWithTokens(indexedEntry, tokens, entryTypeName);
+  }
 }