X-Git-Url: http://gitweb.fperrin.net/?a=blobdiff_plain;f=src%2Fcom%2Fhughes%2Fandroid%2Fdictionary%2Fengine%2FIndex.java;h=364f5b857accdbf8f6ea2224ef99413627137937;hb=2a18ab8b97ba0254a0655d595f05c492eb0eecd4;hp=285024afdb0d0e00b5a2aa38aa87af27b3b2da4b;hpb=2d412ee0753c98d9ad18cc6ffc7ec46e07627532;p=Dictionary.git diff --git a/src/com/hughes/android/dictionary/engine/Index.java b/src/com/hughes/android/dictionary/engine/Index.java index 285024a..364f5b8 100644 --- a/src/com/hughes/android/dictionary/engine/Index.java +++ b/src/com/hughes/android/dictionary/engine/Index.java @@ -12,50 +12,44 @@ // See the License for the specific language governing permissions and // limitations under the License. -/** - * - */ - package com.hughes.android.dictionary.engine; -import com.hughes.android.dictionary.DictionaryInfo; -import com.hughes.android.dictionary.DictionaryInfo.IndexInfo; -import com.hughes.android.dictionary.engine.RowBase.RowKey; -import com.hughes.util.CachingList; -import com.hughes.util.StringUtil; -import com.hughes.util.TransformingList; -import com.hughes.util.raf.RAFList; -import com.hughes.util.raf.RAFSerializable; -import com.hughes.util.raf.RAFSerializer; -import com.hughes.util.raf.SerializableSerializer; -import com.hughes.util.raf.UniformRAFList; -import com.ibm.icu.text.Transliterator; - import java.io.DataInput; -import java.io.DataInputStream; import java.io.DataOutput; import java.io.IOException; import java.io.PrintStream; import java.io.RandomAccessFile; -import java.nio.channels.Channels; -import java.nio.channels.FileChannel; import java.util.AbstractList; import java.util.ArrayList; import java.util.Collection; import java.util.Collections; import java.util.Comparator; import java.util.EnumMap; -import java.util.HashSet; import java.util.HashMap; +import java.util.HashSet; import java.util.List; import java.util.Map; import java.util.Set; import java.util.concurrent.atomic.AtomicBoolean; import java.util.regex.Pattern; +import com.hughes.android.dictionary.DictionaryInfo; +import com.hughes.android.dictionary.DictionaryInfo.IndexInfo; +import com.hughes.android.dictionary.engine.RowBase.RowKey; +import com.hughes.util.CachingList; +import com.hughes.util.DataInputBuffer; +import com.hughes.util.StringUtil; +import com.hughes.util.TransformingList; +import com.hughes.util.raf.RAFList; +import com.hughes.util.raf.RAFSerializable; +import com.hughes.util.raf.RAFSerializer; +import com.hughes.util.raf.SerializableSerializer; +import com.hughes.util.raf.UniformRAFList; +import com.ibm.icu.text.Transliterator; + public final class Index implements RAFSerializable { - static final int CACHE_SIZE = 5000; + private static final int CACHE_SIZE = 5000; public final Dictionary dict; @@ -64,7 +58,7 @@ public final class Index implements RAFSerializable { // persisted: tells how the entries are sorted. public final Language sortLanguage; - final String normalizerRules; + public final String normalizerRules; // Built from the two above. private Transliterator normalizer; @@ -82,7 +76,8 @@ public final class Index implements RAFSerializable { public final boolean swapPairEntries; // Version 2: - int mainTokenCount = -1; + @SuppressWarnings("WeakerAccess") + public int mainTokenCount = -1; // -------------------------------------------------------------------------- @@ -95,9 +90,9 @@ public final class Index implements RAFSerializable { this.sortLanguage = sortLanguage; this.normalizerRules = normalizerRules; this.swapPairEntries = swapPairEntries; - sortedIndexEntries = new ArrayList(); + sortedIndexEntries = new ArrayList<>(); this.stoplist = stoplist; - rows = new ArrayList(); + rows = new ArrayList<>(); normalizer = null; } @@ -105,6 +100,7 @@ public final class Index implements RAFSerializable { /** * Deferred initialization because it can be slow. */ + @SuppressWarnings("WeakerAccess") public synchronized Transliterator normalizer() { if (normalizer == null) { normalizer = TransliteratorManager.get(normalizerRules); @@ -116,11 +112,12 @@ public final class Index implements RAFSerializable { * Note that using this comparator probably involves doing too many text * normalizations. */ + @SuppressWarnings("WeakerAccess") public NormalizeComparator getSortComparator() { return new NormalizeComparator(normalizer(), sortLanguage.getCollator(), dict.dictFileVersion); } - public Index(final Dictionary dict, final FileChannel inp, final DataInput raf) throws IOException { + public Index(final Dictionary dict, final DataInputBuffer raf) throws IOException { this.dict = dict; shortName = raf.readUTF(); longName = raf.readUTF(); @@ -128,18 +125,15 @@ public final class Index implements RAFSerializable { sortLanguage = Language.lookup(languageCode); normalizerRules = raf.readUTF(); swapPairEntries = raf.readBoolean(); - if (sortLanguage == null) { - throw new IOException("Unsupported language: " + languageCode); - } if (dict.dictFileVersion >= 2) { mainTokenCount = raf.readInt(); } sortedIndexEntries = CachingList.create( - RAFList.create(inp, new IndexEntrySerializer(dict.dictFileVersion == 6 ? inp : null), inp.position(), + RAFList.create(raf, new IndexEntrySerializer(), dict.dictFileVersion, dict.dictInfo + " idx " + languageCode + ": "), CACHE_SIZE, true); if (dict.dictFileVersion >= 7) { int count = StringUtil.readVarInt(raf); - stoplist = new HashSet(count); + stoplist = new HashSet<>(count); for (int i = 0; i < count; ++i) { stoplist.add(raf.readUTF()); } @@ -149,7 +143,7 @@ public final class Index implements RAFSerializable { stoplist = Collections.emptySet(); } rows = CachingList.create( - UniformRAFList.create(inp, new RowBase.Serializer(this), inp.position()), + UniformRAFList.create(raf, new RowBase.Serializer(this)), CACHE_SIZE, true); } @@ -161,10 +155,8 @@ public final class Index implements RAFSerializable { raf.writeUTF(sortLanguage.getIsoCode()); raf.writeUTF(normalizerRules); raf.writeBoolean(swapPairEntries); - if (dict.dictFileVersion >= 2) { - raf.writeInt(mainTokenCount); - } - RAFList.write(raf, sortedIndexEntries, new IndexEntrySerializer(null), 32, true); + raf.writeInt(mainTokenCount); + RAFList.write(raf, sortedIndexEntries, new IndexEntrySerializer(), 32, true); StringUtil.writeVarInt(raf, stoplist.size()); for (String i : stoplist) { raf.writeUTF(i); @@ -179,28 +171,22 @@ public final class Index implements RAFSerializable { } private final class IndexEntrySerializer implements RAFSerializer { - private final FileChannel ch; - - public IndexEntrySerializer(FileChannel ch) { - this.ch = ch; - } - @Override public IndexEntry read(DataInput raf) throws IOException { - return new IndexEntry(Index.this, ch, raf); + return new IndexEntry(Index.this, raf); } @Override public void write(DataOutput raf, IndexEntry t) throws IOException { t.write(raf); } - }; + } public static final class IndexEntry implements RAFSerializable { public final String token; private final String normalizedToken; public final int startRow; - public final int numRows; // doesn't count the token row! + final int numRows; // doesn't count the token row! public List htmlEntries; public IndexEntry(final Index index, final String token, final String normalizedToken, @@ -211,10 +197,10 @@ public final class Index implements RAFSerializable { this.normalizedToken = normalizedToken; this.startRow = startRow; this.numRows = numRows; - this.htmlEntries = new ArrayList(); + this.htmlEntries = new ArrayList<>(); } - public IndexEntry(final Index index, final FileChannel ch, final DataInput raf) throws IOException { + IndexEntry(final Index index, final DataInput raf) throws IOException { token = raf.readUTF(); if (index.dict.dictFileVersion >= 7) { startRow = StringUtil.readVarInt(raf); @@ -239,6 +225,7 @@ public final class Index implements RAFSerializable { public HtmlEntry get(int i) { return index.dict.htmlEntries.get(htmlEntryIndices[i]); } + @Override public int size() { return htmlEntryIndices.length; @@ -247,8 +234,8 @@ public final class Index implements RAFSerializable { } } else if (index.dict.dictFileVersion >= 6) { this.htmlEntries = CachingList.create( - RAFList.create(ch, index.dict.htmlEntryIndexSerializer, - ch.position(), index.dict.dictFileVersion, + RAFList.create((DataInputBuffer)raf, index.dict.htmlEntryIndexSerializer, + index.dict.dictFileVersion, index.dict.dictInfo + " htmlEntries: "), 1, false); } else { this.htmlEntries = Collections.emptyList(); @@ -273,12 +260,12 @@ public final class Index implements RAFSerializable { return String.format("%s@%d(%d)", token, startRow, numRows); } - public String normalizedToken() { + String normalizedToken() { return normalizedToken; } } - static final TransformingList.Transformer INDEX_ENTRY_TO_TOKEN = new TransformingList.Transformer() { + private static final TransformingList.Transformer INDEX_ENTRY_TO_TOKEN = new TransformingList.Transformer() { @Override public String transform(IndexEntry t1) { return t1.token; @@ -300,12 +287,12 @@ public final class Index implements RAFSerializable { return index != -1 ? sortedIndexEntries.get(index) : null; } - private int compareIdx(String token, final Comparator sortCollator, int idx) { + private int compareIdx(String token, final Comparator sortCollator, int idx) { final IndexEntry entry = sortedIndexEntries.get(idx); return NormalizeComparator.compareWithoutDash(token, entry.normalizedToken(), sortCollator, dict.dictFileVersion); } - private int findMatchLen(final Comparator sortCollator, String a, String b) { + private int findMatchLen(final Comparator sortCollator, String a, String b) { int start = 0; int end = Math.min(a.length(), b.length()); while (start < end) @@ -319,13 +306,13 @@ public final class Index implements RAFSerializable { return start; } - public int findInsertionPointIndex(String token, final AtomicBoolean interrupted) { + private int findInsertionPointIndex(String token, final AtomicBoolean interrupted) { token = normalizeToken(token); int start = 0; int end = sortedIndexEntries.size(); - final Comparator sortCollator = sortLanguage.getCollator(); + final Comparator sortCollator = sortLanguage.getCollator(); while (start < end) { final int mid = (start + end) / 2; if (interrupted.get()) { @@ -337,8 +324,7 @@ public final class Index implements RAFSerializable { if (comp == 0) comp = sortCollator.compare(token, midEntry.normalizedToken()); if (comp == 0) { - final int result = windBackCase(token, mid, interrupted); - return result; + return windBackCase(token, mid, interrupted); } else if (comp < 0) { // System.out.println("Upper bound: " + midEntry + ", norm=" + // midEntry.normalizedToken() + ", mid=" + mid); @@ -382,7 +368,7 @@ public final class Index implements RAFSerializable { return result; } - private final int windBackCase(final String token, int result, final AtomicBoolean interrupted) { + private int windBackCase(final String token, int result, final AtomicBoolean interrupted) { while (result > 0 && sortedIndexEntries.get(result - 1).normalizedToken().equals(token)) { --result; if (interrupted.get()) { @@ -398,10 +384,10 @@ public final class Index implements RAFSerializable { private static final int MAX_SEARCH_ROWS = 1000; - private final Map prefixToNumRows = new HashMap(); + private final Map prefixToNumRows = new HashMap<>(); - private synchronized final int getUpperBoundOnRowsStartingWith(final String normalizedPrefix, - final int maxRows, final AtomicBoolean interrupted) { + private synchronized int getUpperBoundOnRowsStartingWith(final String normalizedPrefix, + final int maxRows, final AtomicBoolean interrupted) { final Integer numRows = prefixToNumRows.get(normalizedPrefix); if (numRows != null) { return numRows; @@ -414,7 +400,8 @@ public final class Index implements RAFSerializable { return -1; } final IndexEntry indexEntry = sortedIndexEntries.get(index); - if (!indexEntry.normalizedToken.startsWith(normalizedPrefix)) { + if (!indexEntry.normalizedToken.startsWith(normalizedPrefix) && + !NormalizeComparator.withoutDash(indexEntry.normalizedToken).startsWith(normalizedPrefix)) { break; } rowCount += indexEntry.numRows + indexEntry.htmlEntries.size(); @@ -423,7 +410,7 @@ public final class Index implements RAFSerializable { break; } } - prefixToNumRows.put(normalizedPrefix, numRows); + prefixToNumRows.put(normalizedPrefix, rowCount); return rowCount; } @@ -431,9 +418,9 @@ public final class Index implements RAFSerializable { final String searchText, final List searchTokens, final AtomicBoolean interrupted) { final long startMills = System.currentTimeMillis(); - final List result = new ArrayList(); + final List result = new ArrayList<>(); - final Set normalizedNonStoplist = new HashSet(); + final Set normalizedNonStoplist = new HashSet<>(); String bestPrefix = null; int leastRows = Integer.MAX_VALUE; @@ -477,8 +464,8 @@ public final class Index implements RAFSerializable { + ", searchTokens=" + searchTokens); // Place to store the things that match. - final Map> matches = new EnumMap>( - RowMatchType.class); + final Map> matches = new EnumMap<>( + RowMatchType.class); for (final RowMatchType rowMatchType : RowMatchType.values()) { if (rowMatchType != RowMatchType.NO_MATCH) { matches.put(rowMatchType, new ArrayList()); @@ -497,14 +484,15 @@ public final class Index implements RAFSerializable { final String searchToken = bestPrefix; final int insertionPointIndex = findInsertionPointIndex(searchToken, interrupted); - final Set rowsAlreadySeen = new HashSet(); + final Set rowsAlreadySeen = new HashSet<>(); for (int index = insertionPointIndex; index < sortedIndexEntries.size() && matchCount < MAX_SEARCH_ROWS; ++index) { if (interrupted.get()) { return null; } final IndexEntry indexEntry = sortedIndexEntries.get(index); - if (!indexEntry.normalizedToken.startsWith(searchToken)) { + if (!indexEntry.normalizedToken.startsWith(searchToken) && + !NormalizeComparator.withoutDash(indexEntry.normalizedToken).startsWith(searchToken)) { break; } @@ -537,8 +525,8 @@ public final class Index implements RAFSerializable { final RowBase.LengthComparator lengthComparator = new RowBase.LengthComparator( swapPairEntries); for (final Collection rows : matches.values()) { - final List ordered = new ArrayList(rows); - Collections.sort(ordered, lengthComparator); + final List ordered = new ArrayList<>(rows); + ordered.sort(lengthComparator); result.addAll(ordered); }