-
- final DictionaryBuilder dictionaryBuilder;
- public final Index index;
-
- final SortedMap<String, TokenData> tokenToData;
-
- IndexBuilder(final DictionaryBuilder dictionaryBuilder, final String shortName, final String longName, final Language language, final String normalizerRules, final boolean swapPairEntries) {
- this.dictionaryBuilder = dictionaryBuilder;
- index = new Index(dictionaryBuilder.dictionary, shortName, longName, language, normalizerRules, swapPairEntries);
- tokenToData = new TreeMap<String, TokenData>(new NormalizeComparator(index.normalizer, language.collator));
- }
-
- public void build() {
- final Set<EntryData> tokenEntryDatas = new HashSet<EntryData>();
- final List<RowBase> rows = index.rows;
- for (final TokenData tokenData : tokenToData.values()) {
- tokenEntryDatas.clear();
- final int indexIndex = index.sortedIndexEntries.size();
- final int startRow = rows.size();
- rows.add(new TokenRow(indexIndex, rows.size(), index));
-// System.out.println("Added TokenRow: " + rows.get(rows.size() - 1));
- int numRows = 0;
+
+ final DictionaryBuilder dictionaryBuilder;
+ public final Index index;
+ final Set<String> stoplist;
+
+ final SortedMap<FastCompareString, TokenData> tokenToData;
+
+ IndexBuilder(final DictionaryBuilder dictionaryBuilder, final String shortName, final String longName, final Language language, final String normalizerRules, final Set<String> stoplist, final boolean swapPairEntries) {
+ this.dictionaryBuilder = dictionaryBuilder;
+ index = new Index(dictionaryBuilder.dictionary, shortName, longName, language, normalizerRules, swapPairEntries, stoplist);
+ tokenToData = new TreeMap<>(new FastNormalizeComparator(index.getSortComparator()));
+ this.stoplist = stoplist;
+ }
+
+ public void build() {
+ final Set<IndexedEntry> tokenIndexedEntries = new HashSet<>();
+ final List<RowBase> rows = index.rows;
+ index.mainTokenCount = 0;
+ for (final TokenData tokenData : tokenToData.values()) {
+ tokenIndexedEntries.clear();
+ final int indexIndex = index.sortedIndexEntries.size();
+ final int startRow = rows.size();
+
+ TokenRow tokenRow = null;
+ if (!tokenData.htmlEntries.isEmpty()) {
+ tokenRow = new TokenRow(indexIndex, rows.size(), index, tokenData.hasMainEntry);
+ rows.add(tokenRow);
+ }
+
+// System.out.println("Added TokenRow: " + rows.get(rows.size() - 1));
+
+ int numRows = 0; // off by one--doesn't count the token row!