-
- static final int CACHE_SIZE = 5000;
-
- final Dictionary dict;
-
- public final String shortName; // Typically the ISO code for the language.
- public final String longName;
-
- // persisted: tells how the entries are sorted.
- public final Language sortLanguage;
- final String normalizerRules;
-
- // Built from the two above.
- private Transliterator normalizer;
-
- // persisted
- public final List<IndexEntry> sortedIndexEntries;
-
- // One big list!
- // Various sub-types.
- // persisted
- public final List<RowBase> rows;
- public final boolean swapPairEntries;
-
- // Version 2:
- int mainTokenCount = -1;
-
- // --------------------------------------------------------------------------
-
- public Index(final Dictionary dict, final String shortName, final String longName, final Language sortLanguage, final String normalizerRules, final boolean swapPairEntries) {
- this.dict = dict;
- this.shortName = shortName;
- this.longName = longName;
- this.sortLanguage = sortLanguage;
- this.normalizerRules = normalizerRules;
- this.swapPairEntries = swapPairEntries;
- sortedIndexEntries = new ArrayList<IndexEntry>();
- rows = new ArrayList<RowBase>();
-
- normalizer = null;
- }
-
- public synchronized Transliterator normalizer() {
- if (normalizer == null) {
- normalizer = Transliterator.createFromRules("", normalizerRules, Transliterator.FORWARD);
- }
- return normalizer;
- }
-
- public Index(final Dictionary dict, final RandomAccessFile raf) throws IOException {
- this.dict = dict;
- shortName = raf.readUTF();
- longName = raf.readUTF();
- final String languageCode = raf.readUTF();
- sortLanguage = Language.lookup(languageCode);
- normalizerRules = raf.readUTF();
- swapPairEntries = raf.readBoolean();
- if (sortLanguage == null) {
- throw new IOException("Unsupported language: " + languageCode);
- }
- if (dict.dictFileVersion >= 2) {
- mainTokenCount = raf.readInt();
- }
- sortedIndexEntries = CachingList.create(RAFList.create(raf, IndexEntry.SERIALIZER, raf.getFilePointer()), CACHE_SIZE);
- rows = CachingList.create(UniformRAFList.create(raf, new RowBase.Serializer(this), raf.getFilePointer()), CACHE_SIZE);
- }
-
- @Override
- public void write(final RandomAccessFile raf) throws IOException {
- raf.writeUTF(shortName);
- raf.writeUTF(longName);
- raf.writeUTF(sortLanguage.getIsoCode());
- raf.writeUTF(normalizerRules);
- raf.writeBoolean(swapPairEntries);
- if (dict.dictFileVersion >= 2) {
- raf.writeInt(mainTokenCount);
- }
- RAFList.write(raf, sortedIndexEntries, IndexEntry.SERIALIZER);
- UniformRAFList.write(raf, (Collection<RowBase>) rows, new RowBase.Serializer(this), 5);
- }
-
- public void print(final PrintStream out) {
- for (final RowBase row : rows) {
- row.print(out);
- }
- }
-
- public static final class IndexEntry implements RAFSerializable<Index.IndexEntry> {
- public final String token;
- private final String normalizedToken;
- public final int startRow;
- public final int numRows;
-
-
- static final RAFSerializer<IndexEntry> SERIALIZER = new RAFSerializer<IndexEntry> () {
- @Override
- public IndexEntry read(RandomAccessFile raf) throws IOException {
- return new IndexEntry(raf);
- }
- @Override
- public void write(RandomAccessFile raf, IndexEntry t) throws IOException {
- t.write(raf);
- }};
-
- public IndexEntry(final String token, final String normalizedToken, final int startRow, final int numRows) {
- assert token.equals(token.trim());
- assert token.length() > 0;
- this.token = token;
- this.normalizedToken = normalizedToken;
- this.startRow = startRow;
- this.numRows = numRows;
- }
-
- public IndexEntry(final RandomAccessFile raf) throws IOException {
- token = raf.readUTF();
- startRow = raf.readInt();
- numRows = raf.readInt();
- final boolean hasNormalizedForm = raf.readBoolean();
- normalizedToken = hasNormalizedForm ? raf.readUTF() : token;
- }
-
- public void write(RandomAccessFile raf) throws IOException {
- raf.writeUTF(token);
- raf.writeInt(startRow);
- raf.writeInt(numRows);
- final boolean hasNormalizedForm = !token.equals(normalizedToken);
- raf.writeBoolean(hasNormalizedForm);
- if (hasNormalizedForm) {
- raf.writeUTF(normalizedToken);
- }
- }
-
- public String toString() {
- return String.format("%s@%d(%d)", token, startRow, numRows);
- }
-
- public String normalizedToken() {
- return normalizedToken;
- }
- }
-
- public IndexEntry findInsertionPoint(String token, final AtomicBoolean interrupted) {
- if (TransliteratorManager.init(null)) {
- final Transliterator normalizer = normalizer();
- token = normalizer.transliterate(token);
- } else {
- // Do our best since the Transliterators aren't up yet.
- token = token.toLowerCase();
- }
-
- int start = 0;
- int end = sortedIndexEntries.size();
-
- final Collator sortCollator = sortLanguage.getCollator();
- while (start < end) {
- final int mid = (start + end) / 2;
- if (interrupted.get()) {
+
+ private static final int CACHE_SIZE = 5000;
+
+ public final Dictionary dict;
+
+ public final String shortName; // Typically the ISO code for the language.
+ public final String longName;
+
+ // persisted: tells how the entries are sorted.
+ public final Language sortLanguage;
+ public final String normalizerRules;
+
+ // Built from the two above.
+ private Transliterator normalizer;
+
+ // persisted
+ public final List<IndexEntry> sortedIndexEntries;
+
+ // persisted.
+ public final Set<String> stoplist;
+
+ // One big list!
+ // Various sub-types.
+ // persisted
+ public final List<RowBase> rows;
+ public final boolean swapPairEntries;
+
+ // Version 2:
+ @SuppressWarnings("WeakerAccess")
+ public int mainTokenCount = -1;
+
+ // --------------------------------------------------------------------------
+
+ public Index(final Dictionary dict, final String shortName, final String longName,
+ final Language sortLanguage, final String normalizerRules,
+ final boolean swapPairEntries, final Set<String> stoplist) {
+ this.dict = dict;
+ this.shortName = shortName;
+ this.longName = longName;
+ this.sortLanguage = sortLanguage;
+ this.normalizerRules = normalizerRules;
+ this.swapPairEntries = swapPairEntries;
+ sortedIndexEntries = new ArrayList<>();
+ this.stoplist = stoplist;
+ rows = new ArrayList<>();
+
+ normalizer = null;
+ }
+
+ /**
+ * Deferred initialization because it can be slow.
+ */
+ @SuppressWarnings("WeakerAccess")
+ public synchronized Transliterator normalizer() {
+ if (normalizer == null) {
+ normalizer = TransliteratorManager.get(normalizerRules);
+ }
+ return normalizer;
+ }
+
+ /**
+ * Note that using this comparator probably involves doing too many text
+ * normalizations.
+ */
+ @SuppressWarnings("WeakerAccess")
+ public NormalizeComparator getSortComparator() {
+ return new NormalizeComparator(normalizer(), sortLanguage.getCollator(), dict.dictFileVersion);
+ }
+
+ public Index(final Dictionary dict, final FileChannel inp, final DataInput raf) throws IOException {
+ this.dict = dict;
+ shortName = raf.readUTF();
+ longName = raf.readUTF();
+ final String languageCode = raf.readUTF();
+ sortLanguage = Language.lookup(languageCode);
+ normalizerRules = raf.readUTF();
+ swapPairEntries = raf.readBoolean();
+ if (dict.dictFileVersion >= 2) {
+ mainTokenCount = raf.readInt();
+ }
+ sortedIndexEntries = CachingList.create(
+ RAFList.create(inp, new IndexEntrySerializer(dict.dictFileVersion == 6 ? inp : null), inp.position(),
+ dict.dictFileVersion, dict.dictInfo + " idx " + languageCode + ": "), CACHE_SIZE, true);
+ if (dict.dictFileVersion >= 7) {
+ int count = StringUtil.readVarInt(raf);
+ stoplist = new HashSet<>(count);
+ for (int i = 0; i < count; ++i) {
+ stoplist.add(raf.readUTF());
+ }
+ } else if (dict.dictFileVersion >= 4) {
+ stoplist = new SerializableSerializer<Set<String>>().read(raf);
+ } else {
+ stoplist = Collections.emptySet();
+ }
+ rows = CachingList.create(
+ UniformRAFList.create(inp, new RowBase.Serializer(this), inp.position()),
+ CACHE_SIZE, true);
+ }
+
+ @Override
+ public void write(final DataOutput out) throws IOException {
+ RandomAccessFile raf = (RandomAccessFile)out;
+ raf.writeUTF(shortName);
+ raf.writeUTF(longName);
+ raf.writeUTF(sortLanguage.getIsoCode());
+ raf.writeUTF(normalizerRules);
+ raf.writeBoolean(swapPairEntries);
+ raf.writeInt(mainTokenCount);
+ RAFList.write(raf, sortedIndexEntries, new IndexEntrySerializer(null), 32, true);
+ StringUtil.writeVarInt(raf, stoplist.size());
+ for (String i : stoplist) {
+ raf.writeUTF(i);
+ }
+ UniformRAFList.write(raf, rows, new RowBase.Serializer(this), 3 /* bytes per entry */);
+ }
+
+ public void print(final PrintStream out) {
+ for (final RowBase row : rows) {
+ row.print(out);
+ }
+ }
+
+ private final class IndexEntrySerializer implements RAFSerializer<IndexEntry> {
+ private final FileChannel ch;
+
+ IndexEntrySerializer(FileChannel ch) {
+ this.ch = ch;
+ }
+
+ @Override
+ public IndexEntry read(DataInput raf) throws IOException {
+ return new IndexEntry(Index.this, ch, raf);
+ }
+
+ @Override
+ public void write(DataOutput raf, IndexEntry t) throws IOException {
+ t.write(raf);
+ }
+ }
+
+ public static final class IndexEntry implements RAFSerializable<Index.IndexEntry> {
+ public final String token;
+ private final String normalizedToken;
+ public final int startRow;
+ final int numRows; // doesn't count the token row!
+ public List<HtmlEntry> htmlEntries;
+
+ public IndexEntry(final Index index, final String token, final String normalizedToken,
+ final int startRow, final int numRows) {
+ assert token.equals(token.trim());
+ assert token.length() > 0;
+ this.token = token;
+ this.normalizedToken = normalizedToken;
+ this.startRow = startRow;
+ this.numRows = numRows;
+ this.htmlEntries = new ArrayList<>();
+ }
+
+ IndexEntry(final Index index, final FileChannel ch, final DataInput raf) throws IOException {
+ token = raf.readUTF();
+ if (index.dict.dictFileVersion >= 7) {
+ startRow = StringUtil.readVarInt(raf);
+ numRows = StringUtil.readVarInt(raf);
+ } else {
+ startRow = raf.readInt();
+ numRows = raf.readInt();
+ }
+ final boolean hasNormalizedForm = raf.readBoolean();
+ normalizedToken = hasNormalizedForm ? raf.readUTF() : token;
+ if (index.dict.dictFileVersion >= 7) {
+ int size = StringUtil.readVarInt(raf);
+ if (size == 0) {
+ this.htmlEntries = Collections.emptyList();
+ } else {
+ final int[] htmlEntryIndices = new int[size];
+ for (int i = 0; i < size; ++i) {
+ htmlEntryIndices[i] = StringUtil.readVarInt(raf);
+ }
+ this.htmlEntries = new AbstractList<HtmlEntry>() {
+ @Override
+ public HtmlEntry get(int i) {
+ return index.dict.htmlEntries.get(htmlEntryIndices[i]);
+ }
+ @Override
+ public int size() {
+ return htmlEntryIndices.length;
+ }
+ };
+ }
+ } else if (index.dict.dictFileVersion >= 6) {
+ this.htmlEntries = CachingList.create(
+ RAFList.create(ch, index.dict.htmlEntryIndexSerializer,
+ ch.position(), index.dict.dictFileVersion,
+ index.dict.dictInfo + " htmlEntries: "), 1, false);
+ } else {
+ this.htmlEntries = Collections.emptyList();
+ }
+ }
+
+ public void write(DataOutput raf) throws IOException {
+ raf.writeUTF(token);
+ StringUtil.writeVarInt(raf, startRow);
+ StringUtil.writeVarInt(raf, numRows);
+ final boolean hasNormalizedForm = !token.equals(normalizedToken);
+ raf.writeBoolean(hasNormalizedForm);
+ if (hasNormalizedForm) {
+ raf.writeUTF(normalizedToken);
+ }
+ StringUtil.writeVarInt(raf, htmlEntries.size());
+ for (HtmlEntry e : htmlEntries)
+ StringUtil.writeVarInt(raf, e.index());
+ }
+
+ public String toString() {
+ return String.format("%s@%d(%d)", token, startRow, numRows);
+ }
+
+ String normalizedToken() {
+ return normalizedToken;
+ }
+ }
+
+ private static final TransformingList.Transformer<IndexEntry, String> INDEX_ENTRY_TO_TOKEN = new TransformingList.Transformer<IndexEntry, String>() {
+ @Override
+ public String transform(IndexEntry t1) {
+ return t1.token;
+ }
+ };
+
+ public IndexEntry findExact(final String exactToken) {
+ final int result = Collections.binarySearch(
+ TransformingList.create(sortedIndexEntries, INDEX_ENTRY_TO_TOKEN), exactToken,
+ getSortComparator());
+ if (result >= 0) {
+ return sortedIndexEntries.get(result);
+ }