X-Git-Url: http://gitweb.fperrin.net/?a=blobdiff_plain;f=src%2Fcom%2Fhughes%2Fandroid%2Fdictionary%2Fengine%2FIndex.java;h=ea0de2528d2aa76fe5d951e649b47ecb4db71916;hb=83d497f704ad1f8ba85190255d46a3fbe0e3c353;hp=63867081a6623aa60ec344ce33b8b3fea2d562df;hpb=c03ad90f3c2ed0b550bc8e1724b14aed63f87321;p=Dictionary.git diff --git a/src/com/hughes/android/dictionary/engine/Index.java b/src/com/hughes/android/dictionary/engine/Index.java index 6386708..ea0de25 100644 --- a/src/com/hughes/android/dictionary/engine/Index.java +++ b/src/com/hughes/android/dictionary/engine/Index.java @@ -13,7 +13,7 @@ // limitations under the License. /** - * + * */ package com.hughes.android.dictionary.engine; @@ -29,22 +29,24 @@ import com.hughes.util.raf.RAFSerializable; import com.hughes.util.raf.RAFSerializer; import com.hughes.util.raf.SerializableSerializer; import com.hughes.util.raf.UniformRAFList; -import com.ibm.icu.text.Collator; import com.ibm.icu.text.Transliterator; import java.io.DataInput; +import java.io.DataInputStream; import java.io.DataOutput; import java.io.IOException; import java.io.PrintStream; import java.io.RandomAccessFile; +import java.nio.channels.Channels; +import java.nio.channels.FileChannel; import java.util.AbstractList; import java.util.ArrayList; import java.util.Collection; import java.util.Collections; +import java.util.Comparator; import java.util.EnumMap; import java.util.HashSet; -import java.util.LinkedHashMap; -import java.util.LinkedHashSet; +import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.Set; @@ -85,8 +87,8 @@ public final class Index implements RAFSerializable { // -------------------------------------------------------------------------- public Index(final Dictionary dict, final String shortName, final String longName, - final Language sortLanguage, final String normalizerRules, - final boolean swapPairEntries, final Set stoplist) { + final Language sortLanguage, final String normalizerRules, + final boolean swapPairEntries, final Set stoplist) { this.dict = dict; this.shortName = shortName; this.longName = longName; @@ -105,8 +107,7 @@ public final class Index implements RAFSerializable { */ public synchronized Transliterator normalizer() { if (normalizer == null) { - normalizer = Transliterator - .createFromRules("", normalizerRules, Transliterator.FORWARD); + normalizer = TransliteratorManager.get(normalizerRules); } return normalizer; } @@ -116,12 +117,11 @@ public final class Index implements RAFSerializable { * normalizations. */ public NormalizeComparator getSortComparator() { - return new NormalizeComparator(normalizer(), sortLanguage.getCollator()); + return new NormalizeComparator(normalizer(), sortLanguage.getCollator(), dict.dictFileVersion); } - public Index(final Dictionary dict, final DataInput inp) throws IOException { + public Index(final Dictionary dict, final FileChannel inp, final DataInput raf) throws IOException { this.dict = dict; - RandomAccessFile raf = (RandomAccessFile)inp; shortName = raf.readUTF(); longName = raf.readUTF(); final String languageCode = raf.readUTF(); @@ -135,16 +135,22 @@ public final class Index implements RAFSerializable { mainTokenCount = raf.readInt(); } sortedIndexEntries = CachingList.create( - RAFList.create(raf, indexEntrySerializer, raf.getFilePointer(), dict.dictFileVersion, - dict.dictFileVersion >= 7 ? 16 : 1, dict.dictFileVersion >= 7), CACHE_SIZE); - if (dict.dictFileVersion >= 4) { + RAFList.create(inp, new IndexEntrySerializer(dict.dictFileVersion == 6 ? inp : null), inp.position(), + dict.dictFileVersion, dict.dictInfo + " idx " + languageCode + ": "), CACHE_SIZE, true); + if (dict.dictFileVersion >= 7) { + int count = StringUtil.readVarInt(raf); + stoplist = new HashSet(count); + for (int i = 0; i < count; ++i) { + stoplist.add(raf.readUTF()); + } + } else if (dict.dictFileVersion >= 4) { stoplist = new SerializableSerializer>().read(raf); } else { stoplist = Collections.emptySet(); } rows = CachingList.create( - UniformRAFList.create(raf, new RowBase.Serializer(this), raf.getFilePointer()), - CACHE_SIZE); + UniformRAFList.create(inp, new RowBase.Serializer(this), inp.position()), + CACHE_SIZE, true); } @Override @@ -158,8 +164,11 @@ public final class Index implements RAFSerializable { if (dict.dictFileVersion >= 2) { raf.writeInt(mainTokenCount); } - RAFList.write(raf, sortedIndexEntries, indexEntrySerializer, 16, true); - new SerializableSerializer>().write(raf, stoplist); + RAFList.write(raf, sortedIndexEntries, new IndexEntrySerializer(null), 32, true); + StringUtil.writeVarInt(raf, stoplist.size()); + for (String i : stoplist) { + raf.writeUTF(i); + } UniformRAFList.write(raf, rows, new RowBase.Serializer(this), 3 /* bytes per entry */); } @@ -169,30 +178,33 @@ public final class Index implements RAFSerializable { } } - private final RAFSerializer indexEntrySerializer = new RAFSerializer() { + private final class IndexEntrySerializer implements RAFSerializer { + private final FileChannel ch; + + public IndexEntrySerializer(FileChannel ch) { + this.ch = ch; + } + @Override public IndexEntry read(DataInput raf) throws IOException { - return new IndexEntry(Index.this, raf); + return new IndexEntry(Index.this, ch, raf); } @Override public void write(DataOutput raf, IndexEntry t) throws IOException { t.write(raf); } - }; + } public static final class IndexEntry implements RAFSerializable { - private final Index index; public final String token; private final String normalizedToken; public final int startRow; public final int numRows; // doesn't count the token row! public List htmlEntries; - private int[] htmlEntryIndices; public IndexEntry(final Index index, final String token, final String normalizedToken, - final int startRow, final int numRows) { - this.index = index; + final int startRow, final int numRows) { assert token.equals(token.trim()); assert token.length() > 0; this.token = token; @@ -202,8 +214,7 @@ public final class Index implements RAFSerializable { this.htmlEntries = new ArrayList(); } - public IndexEntry(final Index index, final DataInput raf) throws IOException { - this.index = index; + public IndexEntry(final Index index, final FileChannel ch, final DataInput raf) throws IOException { token = raf.readUTF(); if (index.dict.dictFileVersion >= 7) { startRow = StringUtil.readVarInt(raf); @@ -214,27 +225,31 @@ public final class Index implements RAFSerializable { } final boolean hasNormalizedForm = raf.readBoolean(); normalizedToken = hasNormalizedForm ? raf.readUTF() : token; - htmlEntryIndices = null; if (index.dict.dictFileVersion >= 7) { int size = StringUtil.readVarInt(raf); - htmlEntryIndices = new int[size]; - for (int i = 0; i < size; ++i) { - htmlEntryIndices[i] = StringUtil.readVarInt(raf); - } - this.htmlEntries = CachingList.create(new AbstractList() { - @Override - public HtmlEntry get(int i) { - return index.dict.htmlEntries.get(htmlEntryIndices[i]); - } - @Override - public int size() { - return htmlEntryIndices.length; + if (size == 0) { + this.htmlEntries = Collections.emptyList(); + } else { + final int[] htmlEntryIndices = new int[size]; + for (int i = 0; i < size; ++i) { + htmlEntryIndices[i] = StringUtil.readVarInt(raf); } - }, 1); + this.htmlEntries = new AbstractList() { + @Override + public HtmlEntry get(int i) { + return index.dict.htmlEntries.get(htmlEntryIndices[i]); + } + @Override + public int size() { + return htmlEntryIndices.length; + } + }; + } } else if (index.dict.dictFileVersion >= 6) { this.htmlEntries = CachingList.create( - RAFList.create((RandomAccessFile)raf, index.dict.htmlEntryIndexSerializer, - ((RandomAccessFile)raf).getFilePointer(), index.dict.dictFileVersion), 1); + RAFList.create(ch, index.dict.htmlEntryIndexSerializer, + ch.position(), index.dict.dictFileVersion, + index.dict.dictInfo + " htmlEntries: "), 1, false); } else { this.htmlEntries = Collections.emptyList(); } @@ -272,8 +287,8 @@ public final class Index implements RAFSerializable { public IndexEntry findExact(final String exactToken) { final int result = Collections.binarySearch( - TransformingList.create(sortedIndexEntries, INDEX_ENTRY_TO_TOKEN), exactToken, - getSortComparator()); + TransformingList.create(sortedIndexEntries, INDEX_ENTRY_TO_TOKEN), exactToken, + getSortComparator()); if (result >= 0) { return sortedIndexEntries.get(result); } @@ -285,13 +300,32 @@ public final class Index implements RAFSerializable { return index != -1 ? sortedIndexEntries.get(index) : null; } + private int compareIdx(String token, final Comparator sortCollator, int idx) { + final IndexEntry entry = sortedIndexEntries.get(idx); + return NormalizeComparator.compareWithoutDash(token, entry.normalizedToken(), sortCollator, dict.dictFileVersion); + } + + private int findMatchLen(final Comparator sortCollator, String a, String b) { + int start = 0; + int end = Math.min(a.length(), b.length()); + while (start < end) + { + int mid = (start + end + 1) / 2; + if (sortCollator.compare(a.substring(0, mid), b.substring(0, mid)) == 0) + start = mid; + else + end = mid - 1; + } + return start; + } + public int findInsertionPointIndex(String token, final AtomicBoolean interrupted) { token = normalizeToken(token); int start = 0; int end = sortedIndexEntries.size(); - final Collator sortCollator = sortLanguage.getCollator(); + final Comparator sortCollator = sortLanguage.getCollator(); while (start < end) { final int mid = (start + end) / 2; if (interrupted.get()) { @@ -299,21 +333,48 @@ public final class Index implements RAFSerializable { } final IndexEntry midEntry = sortedIndexEntries.get(mid); - final int comp = sortCollator.compare(token, midEntry.normalizedToken()); + int comp = NormalizeComparator.compareWithoutDash(token, midEntry.normalizedToken(), sortCollator, dict.dictFileVersion); + if (comp == 0) + comp = sortCollator.compare(token, midEntry.normalizedToken()); if (comp == 0) { final int result = windBackCase(token, mid, interrupted); return result; } else if (comp < 0) { // System.out.println("Upper bound: " + midEntry + ", norm=" + // midEntry.normalizedToken() + ", mid=" + mid); - end = mid; + + // Hack for robustness if sort order is broken + if (mid + 2 < end && + compareIdx(token, sortCollator, mid + 1) > 0 && + compareIdx(token, sortCollator, mid + 2) > 0) { + start = mid; + } else { + end = mid; + } } else { // System.out.println("Lower bound: " + midEntry + ", norm=" + // midEntry.normalizedToken() + ", mid=" + mid); - start = mid + 1; + + // Hack for robustness if sort order is broken + if (mid - 2 >= start && + compareIdx(token, sortCollator, mid - 1) < 0 && + compareIdx(token, sortCollator, mid - 2) < 0) { + end = mid + 1; + } else { + start = mid + 1; + } } } + // if the word before is the better match, move + // our result to it + if (start > 0 && start < sortedIndexEntries.size()) { + String prev = sortedIndexEntries.get(start - 1).normalizedToken(); + String next = sortedIndexEntries.get(start).normalizedToken(); + if (findMatchLen(sortCollator, token, prev) >= findMatchLen(sortCollator, token, next)) + start--; + } + // If we search for a substring of a string that's in there, return // that. int result = Math.min(start, sortedIndexEntries.size() - 1); @@ -337,7 +398,7 @@ public final class Index implements RAFSerializable { private static final int MAX_SEARCH_ROWS = 1000; - private final Map prefixToNumRows = new LinkedHashMap(); + private final Map prefixToNumRows = new HashMap(); private synchronized final int getUpperBoundOnRowsStartingWith(final String normalizedPrefix, final int maxRows, final AtomicBoolean interrupted) { @@ -367,12 +428,12 @@ public final class Index implements RAFSerializable { } public final List multiWordSearch( - final String searchText, final List searchTokens, - final AtomicBoolean interrupted) { + final String searchText, final List searchTokens, + final AtomicBoolean interrupted) { final long startMills = System.currentTimeMillis(); final List result = new ArrayList(); - final Set normalizedNonStoplist = new LinkedHashSet(); + final Set normalizedNonStoplist = new HashSet(); String bestPrefix = null; int leastRows = Integer.MAX_VALUE; @@ -389,7 +450,7 @@ public final class Index implements RAFSerializable { if (!stoplist.contains(searchToken)) { if (normalizedNonStoplist.add(normalized)) { final int numRows = getUpperBoundOnRowsStartingWith(normalized, - MAX_SEARCH_ROWS, interrupted); + MAX_SEARCH_ROWS, interrupted); if (numRows != -1 && numRows < leastRows) { if (numRows == 0) { // We really are done here. @@ -413,11 +474,11 @@ public final class Index implements RAFSerializable { System.out.println("Everything was in the stoplist!"); } System.out.println("Searching using prefix: " + bestPrefix + ", leastRows=" + leastRows - + ", searchTokens=" + searchTokens); + + ", searchTokens=" + searchTokens); // Place to store the things that match. final Map> matches = new EnumMap>( - RowMatchType.class); + RowMatchType.class); for (final RowMatchType rowMatchType : RowMatchType.values()) { if (rowMatchType != RowMatchType.NO_MATCH) { matches.put(rowMatchType, new ArrayList()); @@ -463,7 +524,7 @@ public final class Index implements RAFSerializable { } rowsAlreadySeen.add(rowKey); final RowMatchType matchType = row.matches(searchTokens, pattern, normalizer(), - swapPairEntries); + swapPairEntries); if (matchType != RowMatchType.NO_MATCH) { matches.get(matchType).add(row); ++matchCount; @@ -474,7 +535,7 @@ public final class Index implements RAFSerializable { // Sort them into a reasonable order. final RowBase.LengthComparator lengthComparator = new RowBase.LengthComparator( - swapPairEntries); + swapPairEntries); for (final Collection rows : matches.values()) { final List ordered = new ArrayList(rows); Collections.sort(ordered, lengthComparator); @@ -486,7 +547,7 @@ public final class Index implements RAFSerializable { } private String normalizeToken(final String searchToken) { - if (TransliteratorManager.init(null)) { + if (TransliteratorManager.init(null, null)) { final Transliterator normalizer = normalizer(); return normalizer.transliterate(searchToken); } else {