1 package com.hughes.android.dictionary.engine;
3 import java.util.ArrayList;
4 import java.util.Collections;
5 import java.util.Comparator;
6 import java.util.EnumMap;
7 import java.util.HashSet;
11 import java.util.SortedMap;
12 import java.util.TreeMap;
14 import com.hughes.android.dictionary.engine.Index.IndexEntry;
15 import com.hughes.android.dictionary.parser.DictFileParser;
18 public class IndexBuilder {
20 final DictionaryBuilder dictionaryBuilder;
21 public final Index index;
23 final SortedMap<String, TokenData> tokenToData;
25 IndexBuilder(final DictionaryBuilder dictionaryBuilder, final String shortName, final String longName, final Language language, final String normalizerRules, final boolean swapPairEntries) {
26 this.dictionaryBuilder = dictionaryBuilder;
27 index = new Index(dictionaryBuilder.dictionary, shortName, longName, language, normalizerRules, swapPairEntries);
28 tokenToData = new TreeMap<String, TokenData>(new NormalizeComparator(index.normalizer(), language.getCollator()));
32 final Set<IndexedEntry> tokenEntryDatas = new HashSet<IndexedEntry>();
33 final List<RowBase> rows = index.rows;
34 for (final TokenData tokenData : tokenToData.values()) {
35 tokenEntryDatas.clear();
36 final int indexIndex = index.sortedIndexEntries.size();
37 final int startRow = rows.size();
38 rows.add(new TokenRow(indexIndex, rows.size(), index));
39 // System.out.println("Added TokenRow: " + rows.get(rows.size() - 1));
41 // System.out.println("TOKEN: " + tokenData.token);
42 for (final Map.Entry<EntryTypeName, List<IndexedEntry>> typeToEntry : tokenData.typeToEntries.entrySet()) {
43 for (final IndexedEntry entryData : typeToEntry.getValue()) {
44 if (entryData.index() == -1) {
45 entryData.addToDictionary(dictionaryBuilder.dictionary);
46 assert entryData.index() >= 0;
48 if (tokenEntryDatas.add(entryData)) {
49 rows.add(new PairEntry.Row(entryData.index(), rows.size(), index));
52 // System.out.print(" " + typeToEntry.getKey() + ": ");
53 // rows.get(rows.size() - 1).print(System.out);
54 // System.out.println();
58 index.sortedIndexEntries.add(new Index.IndexEntry(tokenData.token, index
59 .normalizer().transliterate(tokenData.token), startRow, numRows));
62 final List<IndexEntry> sortedEntries = new ArrayList<IndexEntry>(index.sortedIndexEntries);
63 Collections.sort(sortedEntries, new Comparator<IndexEntry>() {
65 public int compare(IndexEntry object1, IndexEntry object2) {
66 return object2.numRows - object1.numRows;
68 System.out.println("Most common tokens:");
69 for (int i = 0; i < 50 && i < sortedEntries.size(); ++i) {
70 System.out.println(" " + sortedEntries.get(i));
74 static class TokenData {
77 final Map<EntryTypeName, List<IndexedEntry>> typeToEntries = new EnumMap<EntryTypeName, List<IndexedEntry>>(EntryTypeName.class);
79 TokenData(final String token) {
80 assert token.equals(token.trim());
81 assert token.length() > 0;
86 public TokenData getOrCreateTokenData(final String token) {
87 TokenData tokenData = tokenToData.get(token);
88 if (tokenData == null) {
89 tokenData = new TokenData(token);
90 tokenToData.put(token, tokenData);
95 public List<IndexedEntry> getOrCreateEntries(final String token, final EntryTypeName entryTypeName) {
96 final TokenData tokenData = getOrCreateTokenData(token);
97 List<IndexedEntry> entries = tokenData.typeToEntries.get(entryTypeName);
98 if (entries == null) {
99 entries = new ArrayList<IndexedEntry>();
100 tokenData.typeToEntries.put(entryTypeName, entries);
105 public void addEntryWithTokens(final IndexedEntry indexedEntry, final Set<String> tokens,
106 final EntryTypeName entryTypeName) {
107 for (final String token : tokens) {
108 getOrCreateEntries(token, entryTypeName).add(indexedEntry);
112 public void addEntryWithString(final IndexedEntry indexedEntry, final String untokenizedString,
113 final EntryTypeName singleTokenEntryTypeName, final EntryTypeName multiTokenEntryTypeName) {
114 final Set<String> tokens = DictFileParser.tokenize(untokenizedString, DictFileParser.NON_CHAR);
115 addEntryWithTokens(indexedEntry, tokens, tokens.size() == 1 ? singleTokenEntryTypeName : multiTokenEntryTypeName);
118 public void addEntryWithString(final IndexedEntry indexedEntry, final String untokenizedString,
119 final EntryTypeName entryTypeName) {
120 addEntryWithString(indexedEntry, untokenizedString, entryTypeName, entryTypeName);