IndexBuilder(final DictionaryBuilder dictionaryBuilder, final String shortName, final String longName, final Language language, final String normalizerRules, final Set<String> stoplist, final boolean swapPairEntries) {
this.dictionaryBuilder = dictionaryBuilder;
- index = new Index(dictionaryBuilder.dictionary, shortName, longName, language, normalizerRules, swapPairEntries);
- tokenToData = new TreeMap<String, TokenData>(new NormalizeComparator(index.normalizer(), language.getCollator()));
+ index = new Index(dictionaryBuilder.dictionary, shortName, longName, language, normalizerRules, swapPairEntries, stoplist);
+ tokenToData = new TreeMap<String, TokenData>(index.getSortComparator());
this.stoplist = stoplist;
}
public void build() {
- final Set<IndexedEntry> tokenEntryDatas = new HashSet<IndexedEntry>();
+ final Set<IndexedEntry> tokenIndexedEntries = new HashSet<IndexedEntry>();
final List<RowBase> rows = index.rows;
+ index.mainTokenCount = 0;
for (final TokenData tokenData : tokenToData.values()) {
- tokenEntryDatas.clear();
+ tokenIndexedEntries.clear();
final int indexIndex = index.sortedIndexEntries.size();
final int startRow = rows.size();
- rows.add(new TokenRow(indexIndex, rows.size(), index, tokenData.hasMainEntry));
-// System.out.println("Added TokenRow: " + rows.get(rows.size() - 1));
- int numRows = 0;
+ TokenRow tokenRow = null;
+
+ int numRows = 0; // off by one--doesn't count the token row!
// System.out.println("TOKEN: " + tokenData.token);
- for (final Map.Entry<EntryTypeName, List<IndexedEntry>> typeToEntry : tokenData.typeToEntries.entrySet()) {
- for (final IndexedEntry entryData : typeToEntry.getValue()) {
- if (entryData.index() == -1) {
- entryData.addToDictionary(dictionaryBuilder.dictionary);
- assert entryData.index() >= 0;
+ for (final Map.Entry<EntryTypeName, List<IndexedEntry>> typeToIndexedEntries : tokenData.typeToEntries.entrySet()) {
+ for (final IndexedEntry indexedEntry : typeToIndexedEntries.getValue()) {
+
+ if (!indexedEntry.isValid) {
+ continue;
+ }
+
+ if (tokenRow == null) {
+// System.out.println("Added TokenRow: " + rows.get(rows.size() - 1));
+ tokenRow = new TokenRow(indexIndex, rows.size(), index, tokenData.hasMainEntry);
+ rows.add(tokenRow);
+ if (tokenRow.hasMainEntry) {
+ index.mainTokenCount++;
+ }
+ }
+
+ if (indexedEntry.index() == -1) {
+ indexedEntry.addToDictionary(dictionaryBuilder.dictionary);
+ assert indexedEntry.index() >= 0;
}
- if (tokenEntryDatas.add(entryData)) {
- rows.add(new PairEntry.Row(entryData.index(), rows.size(), index));
+ if (tokenIndexedEntries.add(indexedEntry)) {
+ rows.add(indexedEntry.entry.CreateRow(indexedEntry.index(), rows.size(), index));
+ ++indexedEntry.entry.entrySource.numEntries;
++numRows;
// System.out.print(" " + typeToEntry.getKey() + ": ");
private List<IndexedEntry> getOrCreateEntries(final String token, final EntryTypeName entryTypeName) {
final TokenData tokenData = getOrCreateTokenData(token);
List<IndexedEntry> entries = tokenData.typeToEntries.get(entryTypeName);
- if (entryTypeName.overridesStopList) {
+ if (entryTypeName.mainWord) {
tokenData.hasMainEntry = true;
}
if (entries == null) {