1 // Copyright 2011 Google Inc. All Rights Reserved.
3 // Licensed under the Apache License, Version 2.0 (the "License");
4 // you may not use this file except in compliance with the License.
5 // You may obtain a copy of the License at
7 // http://www.apache.org/licenses/LICENSE-2.0
9 // Unless required by applicable law or agreed to in writing, software
10 // distributed under the License is distributed on an "AS IS" BASIS,
11 // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 // See the License for the specific language governing permissions and
13 // limitations under the License.
15 package com.hughes.android.dictionary.engine;
17 import java.util.ArrayList;
18 import java.util.Collections;
19 import java.util.Comparator;
20 import java.util.EnumMap;
21 import java.util.HashSet;
22 import java.util.List;
25 import java.util.SortedMap;
26 import java.util.TreeMap;
28 import com.hughes.android.dictionary.engine.Index.IndexEntry;
29 import com.hughes.android.dictionary.parser.DictFileParser;
31 public class IndexBuilder {
33 final DictionaryBuilder dictionaryBuilder;
34 public final Index index;
35 final Set<String> stoplist;
37 final SortedMap<String, TokenData> tokenToData;
39 IndexBuilder(final DictionaryBuilder dictionaryBuilder, final String shortName, final String longName, final Language language, final String normalizerRules, final Set<String> stoplist, final boolean swapPairEntries) {
40 this.dictionaryBuilder = dictionaryBuilder;
41 index = new Index(dictionaryBuilder.dictionary, shortName, longName, language, normalizerRules, swapPairEntries, stoplist);
42 tokenToData = new TreeMap<String, TokenData>(index.getSortComparator());
43 this.stoplist = stoplist;
47 final Set<IndexedEntry> tokenIndexedEntries = new HashSet<IndexedEntry>();
48 final List<RowBase> rows = index.rows;
49 index.mainTokenCount = 0;
50 for (final TokenData tokenData : tokenToData.values()) {
51 tokenIndexedEntries.clear();
52 final int indexIndex = index.sortedIndexEntries.size();
53 final int startRow = rows.size();
55 TokenRow tokenRow = null;
57 int numRows = 0; // off by one--doesn't count the token row!
58 // System.out.println("TOKEN: " + tokenData.token);
59 for (final Map.Entry<EntryTypeName, List<IndexedEntry>> typeToIndexedEntries : tokenData.typeToEntries.entrySet()) {
60 for (final IndexedEntry indexedEntry : typeToIndexedEntries.getValue()) {
62 if (!indexedEntry.isValid) {
66 if (tokenRow == null) {
67 // System.out.println("Added TokenRow: " + rows.get(rows.size() - 1));
68 tokenRow = new TokenRow(indexIndex, rows.size(), index, tokenData.hasMainEntry);
70 if (tokenRow.hasMainEntry) {
71 index.mainTokenCount++;
75 if (indexedEntry.index() == -1) {
76 indexedEntry.addToDictionary(dictionaryBuilder.dictionary);
77 assert indexedEntry.index() >= 0;
79 if (tokenIndexedEntries.add(indexedEntry)) {
80 rows.add(new PairEntry.Row(indexedEntry.index(), rows.size(), index));
81 ++indexedEntry.entry.entrySource.numEntries;
84 // System.out.print(" " + typeToEntry.getKey() + ": ");
85 // rows.get(rows.size() - 1).print(System.out);
86 // System.out.println();
90 index.sortedIndexEntries.add(new Index.IndexEntry(tokenData.token, index
91 .normalizer().transliterate(tokenData.token), startRow, numRows));
94 final List<IndexEntry> entriesSortedByNumRows = new ArrayList<IndexEntry>(index.sortedIndexEntries);
95 Collections.sort(entriesSortedByNumRows, new Comparator<IndexEntry>() {
97 public int compare(IndexEntry object1, IndexEntry object2) {
98 return object2.numRows - object1.numRows;
100 System.out.println("Most common tokens:");
101 for (int i = 0; i < 50 && i < entriesSortedByNumRows.size(); ++i) {
102 System.out.println(" " + entriesSortedByNumRows.get(i));
106 static class TokenData {
109 final Map<EntryTypeName, List<IndexedEntry>> typeToEntries = new EnumMap<EntryTypeName, List<IndexedEntry>>(EntryTypeName.class);
110 boolean hasMainEntry = false;
112 TokenData(final String token) {
113 assert token.equals(token.trim());
114 assert token.length() > 0;
119 private TokenData getOrCreateTokenData(final String token) {
120 TokenData tokenData = tokenToData.get(token);
121 if (tokenData == null) {
122 tokenData = new TokenData(token);
123 tokenToData.put(token, tokenData);
128 private List<IndexedEntry> getOrCreateEntries(final String token, final EntryTypeName entryTypeName) {
129 final TokenData tokenData = getOrCreateTokenData(token);
130 List<IndexedEntry> entries = tokenData.typeToEntries.get(entryTypeName);
131 if (entryTypeName.mainWord) {
132 tokenData.hasMainEntry = true;
134 if (entries == null) {
135 entries = new ArrayList<IndexedEntry>();
136 tokenData.typeToEntries.put(entryTypeName, entries);
141 public void addEntryWithTokens(final IndexedEntry indexedEntry, final Set<String> tokens,
142 final EntryTypeName entryTypeName) {
143 if (indexedEntry == null) {
144 System.out.println("asdfasdf");
146 assert indexedEntry != null;
147 for (final String token : tokens) {
148 if (entryTypeName.overridesStopList || !stoplist.contains(token)) {
149 getOrCreateEntries(token, entryTypeName).add(indexedEntry);
154 public void addEntryWithString(final IndexedEntry indexedEntry, final String untokenizedString,
155 final EntryTypeName entryTypeName) {
156 final Set<String> tokens = DictFileParser.tokenize(untokenizedString, DictFileParser.NON_CHAR);
157 addEntryWithTokens(indexedEntry, tokens, tokens.size() == 1 ? entryTypeName.singleWordInstance : entryTypeName);
160 public void addEntryWithStringNoSingle(final IndexedEntry indexedEntry, final String untokenizedString,
161 final EntryTypeName entryTypeName) {
162 final Set<String> tokens = DictFileParser.tokenize(untokenizedString, DictFileParser.NON_CHAR);
163 addEntryWithTokens(indexedEntry, tokens, entryTypeName);