X-Git-Url: http://gitweb.fperrin.net/?a=blobdiff_plain;f=src%2Fcom%2Fhughes%2Fandroid%2Fdictionary%2Fengine%2FIndex.java;h=ea0de2528d2aa76fe5d951e649b47ecb4db71916;hb=83d497f704ad1f8ba85190255d46a3fbe0e3c353;hp=93dd34bb02d4e9fd21feaf0cc4e6ed9d8ad231fc;hpb=71e3edc548080b5c6abcd043652035397c6a41d8;p=Dictionary.git diff --git a/src/com/hughes/android/dictionary/engine/Index.java b/src/com/hughes/android/dictionary/engine/Index.java index 93dd34b..ea0de25 100644 --- a/src/com/hughes/android/dictionary/engine/Index.java +++ b/src/com/hughes/android/dictionary/engine/Index.java @@ -13,7 +13,7 @@ // limitations under the License. /** - * + * */ package com.hughes.android.dictionary.engine; @@ -29,14 +29,16 @@ import com.hughes.util.raf.RAFSerializable; import com.hughes.util.raf.RAFSerializer; import com.hughes.util.raf.SerializableSerializer; import com.hughes.util.raf.UniformRAFList; -import java.text.Collator; import com.ibm.icu.text.Transliterator; import java.io.DataInput; +import java.io.DataInputStream; import java.io.DataOutput; import java.io.IOException; import java.io.PrintStream; import java.io.RandomAccessFile; +import java.nio.channels.Channels; +import java.nio.channels.FileChannel; import java.util.AbstractList; import java.util.ArrayList; import java.util.Collection; @@ -44,8 +46,7 @@ import java.util.Collections; import java.util.Comparator; import java.util.EnumMap; import java.util.HashSet; -import java.util.LinkedHashMap; -import java.util.LinkedHashSet; +import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.Set; @@ -86,8 +87,8 @@ public final class Index implements RAFSerializable { // -------------------------------------------------------------------------- public Index(final Dictionary dict, final String shortName, final String longName, - final Language sortLanguage, final String normalizerRules, - final boolean swapPairEntries, final Set stoplist) { + final Language sortLanguage, final String normalizerRules, + final boolean swapPairEntries, final Set stoplist) { this.dict = dict; this.shortName = shortName; this.longName = longName; @@ -106,8 +107,7 @@ public final class Index implements RAFSerializable { */ public synchronized Transliterator normalizer() { if (normalizer == null) { - normalizer = Transliterator - .createFromRules("", normalizerRules, Transliterator.FORWARD); + normalizer = TransliteratorManager.get(normalizerRules); } return normalizer; } @@ -120,9 +120,8 @@ public final class Index implements RAFSerializable { return new NormalizeComparator(normalizer(), sortLanguage.getCollator(), dict.dictFileVersion); } - public Index(final Dictionary dict, final DataInput inp) throws IOException { + public Index(final Dictionary dict, final FileChannel inp, final DataInput raf) throws IOException { this.dict = dict; - RandomAccessFile raf = (RandomAccessFile)inp; shortName = raf.readUTF(); longName = raf.readUTF(); final String languageCode = raf.readUTF(); @@ -136,22 +135,22 @@ public final class Index implements RAFSerializable { mainTokenCount = raf.readInt(); } sortedIndexEntries = CachingList.create( - RAFList.create(raf, indexEntrySerializer, raf.getFilePointer(), - dict.dictFileVersion), CACHE_SIZE); + RAFList.create(inp, new IndexEntrySerializer(dict.dictFileVersion == 6 ? inp : null), inp.position(), + dict.dictFileVersion, dict.dictInfo + " idx " + languageCode + ": "), CACHE_SIZE, true); if (dict.dictFileVersion >= 7) { int count = StringUtil.readVarInt(raf); stoplist = new HashSet(count); for (int i = 0; i < count; ++i) { stoplist.add(raf.readUTF()); } - } else if (dict.dictFileVersion >= 4) { + } else if (dict.dictFileVersion >= 4) { stoplist = new SerializableSerializer>().read(raf); } else { stoplist = Collections.emptySet(); } rows = CachingList.create( - UniformRAFList.create(raf, new RowBase.Serializer(this), raf.getFilePointer()), - CACHE_SIZE); + UniformRAFList.create(inp, new RowBase.Serializer(this), inp.position()), + CACHE_SIZE, true); } @Override @@ -165,7 +164,7 @@ public final class Index implements RAFSerializable { if (dict.dictFileVersion >= 2) { raf.writeInt(mainTokenCount); } - RAFList.write(raf, sortedIndexEntries, indexEntrySerializer, 32, true); + RAFList.write(raf, sortedIndexEntries, new IndexEntrySerializer(null), 32, true); StringUtil.writeVarInt(raf, stoplist.size()); for (String i : stoplist) { raf.writeUTF(i); @@ -179,30 +178,33 @@ public final class Index implements RAFSerializable { } } - private final RAFSerializer indexEntrySerializer = new RAFSerializer() { + private final class IndexEntrySerializer implements RAFSerializer { + private final FileChannel ch; + + public IndexEntrySerializer(FileChannel ch) { + this.ch = ch; + } + @Override public IndexEntry read(DataInput raf) throws IOException { - return new IndexEntry(Index.this, raf); + return new IndexEntry(Index.this, ch, raf); } @Override public void write(DataOutput raf, IndexEntry t) throws IOException { t.write(raf); } - }; + } public static final class IndexEntry implements RAFSerializable { - private final Index index; public final String token; private final String normalizedToken; public final int startRow; public final int numRows; // doesn't count the token row! public List htmlEntries; - private int[] htmlEntryIndices; public IndexEntry(final Index index, final String token, final String normalizedToken, - final int startRow, final int numRows) { - this.index = index; + final int startRow, final int numRows) { assert token.equals(token.trim()); assert token.length() > 0; this.token = token; @@ -212,8 +214,7 @@ public final class Index implements RAFSerializable { this.htmlEntries = new ArrayList(); } - public IndexEntry(final Index index, final DataInput raf) throws IOException { - this.index = index; + public IndexEntry(final Index index, final FileChannel ch, final DataInput raf) throws IOException { token = raf.readUTF(); if (index.dict.dictFileVersion >= 7) { startRow = StringUtil.readVarInt(raf); @@ -224,27 +225,31 @@ public final class Index implements RAFSerializable { } final boolean hasNormalizedForm = raf.readBoolean(); normalizedToken = hasNormalizedForm ? raf.readUTF() : token; - htmlEntryIndices = null; if (index.dict.dictFileVersion >= 7) { int size = StringUtil.readVarInt(raf); - htmlEntryIndices = new int[size]; - for (int i = 0; i < size; ++i) { - htmlEntryIndices[i] = StringUtil.readVarInt(raf); - } - this.htmlEntries = CachingList.create(new AbstractList() { - @Override - public HtmlEntry get(int i) { - return index.dict.htmlEntries.get(htmlEntryIndices[i]); + if (size == 0) { + this.htmlEntries = Collections.emptyList(); + } else { + final int[] htmlEntryIndices = new int[size]; + for (int i = 0; i < size; ++i) { + htmlEntryIndices[i] = StringUtil.readVarInt(raf); } - @Override - public int size() { - return htmlEntryIndices.length; - } - }, 1); + this.htmlEntries = new AbstractList() { + @Override + public HtmlEntry get(int i) { + return index.dict.htmlEntries.get(htmlEntryIndices[i]); + } + @Override + public int size() { + return htmlEntryIndices.length; + } + }; + } } else if (index.dict.dictFileVersion >= 6) { this.htmlEntries = CachingList.create( - RAFList.create((RandomAccessFile)raf, index.dict.htmlEntryIndexSerializer, - ((RandomAccessFile)raf).getFilePointer(), index.dict.dictFileVersion), 1); + RAFList.create(ch, index.dict.htmlEntryIndexSerializer, + ch.position(), index.dict.dictFileVersion, + index.dict.dictInfo + " htmlEntries: "), 1, false); } else { this.htmlEntries = Collections.emptyList(); } @@ -282,8 +287,8 @@ public final class Index implements RAFSerializable { public IndexEntry findExact(final String exactToken) { final int result = Collections.binarySearch( - TransformingList.create(sortedIndexEntries, INDEX_ENTRY_TO_TOKEN), exactToken, - getSortComparator()); + TransformingList.create(sortedIndexEntries, INDEX_ENTRY_TO_TOKEN), exactToken, + getSortComparator()); if (result >= 0) { return sortedIndexEntries.get(result); } @@ -295,6 +300,25 @@ public final class Index implements RAFSerializable { return index != -1 ? sortedIndexEntries.get(index) : null; } + private int compareIdx(String token, final Comparator sortCollator, int idx) { + final IndexEntry entry = sortedIndexEntries.get(idx); + return NormalizeComparator.compareWithoutDash(token, entry.normalizedToken(), sortCollator, dict.dictFileVersion); + } + + private int findMatchLen(final Comparator sortCollator, String a, String b) { + int start = 0; + int end = Math.min(a.length(), b.length()); + while (start < end) + { + int mid = (start + end + 1) / 2; + if (sortCollator.compare(a.substring(0, mid), b.substring(0, mid)) == 0) + start = mid; + else + end = mid - 1; + } + return start; + } + public int findInsertionPointIndex(String token, final AtomicBoolean interrupted) { token = normalizeToken(token); @@ -309,7 +333,7 @@ public final class Index implements RAFSerializable { } final IndexEntry midEntry = sortedIndexEntries.get(mid); - int comp = NormalizeComparator.compareWithoutLeadingDash(token, midEntry.normalizedToken(), sortCollator, dict.dictFileVersion); + int comp = NormalizeComparator.compareWithoutDash(token, midEntry.normalizedToken(), sortCollator, dict.dictFileVersion); if (comp == 0) comp = sortCollator.compare(token, midEntry.normalizedToken()); if (comp == 0) { @@ -318,14 +342,39 @@ public final class Index implements RAFSerializable { } else if (comp < 0) { // System.out.println("Upper bound: " + midEntry + ", norm=" + // midEntry.normalizedToken() + ", mid=" + mid); - end = mid; + + // Hack for robustness if sort order is broken + if (mid + 2 < end && + compareIdx(token, sortCollator, mid + 1) > 0 && + compareIdx(token, sortCollator, mid + 2) > 0) { + start = mid; + } else { + end = mid; + } } else { // System.out.println("Lower bound: " + midEntry + ", norm=" + // midEntry.normalizedToken() + ", mid=" + mid); - start = mid + 1; + + // Hack for robustness if sort order is broken + if (mid - 2 >= start && + compareIdx(token, sortCollator, mid - 1) < 0 && + compareIdx(token, sortCollator, mid - 2) < 0) { + end = mid + 1; + } else { + start = mid + 1; + } } } + // if the word before is the better match, move + // our result to it + if (start > 0 && start < sortedIndexEntries.size()) { + String prev = sortedIndexEntries.get(start - 1).normalizedToken(); + String next = sortedIndexEntries.get(start).normalizedToken(); + if (findMatchLen(sortCollator, token, prev) >= findMatchLen(sortCollator, token, next)) + start--; + } + // If we search for a substring of a string that's in there, return // that. int result = Math.min(start, sortedIndexEntries.size() - 1); @@ -349,7 +398,7 @@ public final class Index implements RAFSerializable { private static final int MAX_SEARCH_ROWS = 1000; - private final Map prefixToNumRows = new LinkedHashMap(); + private final Map prefixToNumRows = new HashMap(); private synchronized final int getUpperBoundOnRowsStartingWith(final String normalizedPrefix, final int maxRows, final AtomicBoolean interrupted) { @@ -379,12 +428,12 @@ public final class Index implements RAFSerializable { } public final List multiWordSearch( - final String searchText, final List searchTokens, - final AtomicBoolean interrupted) { + final String searchText, final List searchTokens, + final AtomicBoolean interrupted) { final long startMills = System.currentTimeMillis(); final List result = new ArrayList(); - final Set normalizedNonStoplist = new LinkedHashSet(); + final Set normalizedNonStoplist = new HashSet(); String bestPrefix = null; int leastRows = Integer.MAX_VALUE; @@ -401,7 +450,7 @@ public final class Index implements RAFSerializable { if (!stoplist.contains(searchToken)) { if (normalizedNonStoplist.add(normalized)) { final int numRows = getUpperBoundOnRowsStartingWith(normalized, - MAX_SEARCH_ROWS, interrupted); + MAX_SEARCH_ROWS, interrupted); if (numRows != -1 && numRows < leastRows) { if (numRows == 0) { // We really are done here. @@ -425,11 +474,11 @@ public final class Index implements RAFSerializable { System.out.println("Everything was in the stoplist!"); } System.out.println("Searching using prefix: " + bestPrefix + ", leastRows=" + leastRows - + ", searchTokens=" + searchTokens); + + ", searchTokens=" + searchTokens); // Place to store the things that match. final Map> matches = new EnumMap>( - RowMatchType.class); + RowMatchType.class); for (final RowMatchType rowMatchType : RowMatchType.values()) { if (rowMatchType != RowMatchType.NO_MATCH) { matches.put(rowMatchType, new ArrayList()); @@ -475,7 +524,7 @@ public final class Index implements RAFSerializable { } rowsAlreadySeen.add(rowKey); final RowMatchType matchType = row.matches(searchTokens, pattern, normalizer(), - swapPairEntries); + swapPairEntries); if (matchType != RowMatchType.NO_MATCH) { matches.get(matchType).add(row); ++matchCount; @@ -486,7 +535,7 @@ public final class Index implements RAFSerializable { // Sort them into a reasonable order. final RowBase.LengthComparator lengthComparator = new RowBase.LengthComparator( - swapPairEntries); + swapPairEntries); for (final Collection rows : matches.values()) { final List ordered = new ArrayList(rows); Collections.sort(ordered, lengthComparator); @@ -498,7 +547,7 @@ public final class Index implements RAFSerializable { } private String normalizeToken(final String searchToken) { - if (TransliteratorManager.init(null)) { + if (TransliteratorManager.init(null, null)) { final Transliterator normalizer = normalizer(); return normalizer.transliterate(searchToken); } else {