// limitations under the License.
/**
- *
+ *
*/
package com.hughes.android.dictionary.engine;
import java.util.Comparator;
import java.util.EnumMap;
import java.util.HashSet;
-import java.util.LinkedHashMap;
-import java.util.LinkedHashSet;
+import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Set;
// --------------------------------------------------------------------------
public Index(final Dictionary dict, final String shortName, final String longName,
- final Language sortLanguage, final String normalizerRules,
- final boolean swapPairEntries, final Set<String> stoplist) {
+ final Language sortLanguage, final String normalizerRules,
+ final boolean swapPairEntries, final Set<String> stoplist) {
this.dict = dict;
this.shortName = shortName;
this.longName = longName;
*/
public synchronized Transliterator normalizer() {
if (normalizer == null) {
- normalizer = Transliterator
- .createFromRules("", normalizerRules, Transliterator.FORWARD);
+ normalizer = TransliteratorManager.get(normalizerRules);
}
return normalizer;
}
mainTokenCount = raf.readInt();
}
sortedIndexEntries = CachingList.create(
- RAFList.create(raf, indexEntrySerializer, raf.getFilePointer(),
- dict.dictFileVersion), CACHE_SIZE);
+ RAFList.create(raf, indexEntrySerializer, raf.getFilePointer(),
+ dict.dictFileVersion, dict.dictInfo + " idx " + languageCode + ": "), CACHE_SIZE);
if (dict.dictFileVersion >= 7) {
int count = StringUtil.readVarInt(raf);
stoplist = new HashSet<String>(count);
for (int i = 0; i < count; ++i) {
stoplist.add(raf.readUTF());
}
- } else if (dict.dictFileVersion >= 4) {
+ } else if (dict.dictFileVersion >= 4) {
stoplist = new SerializableSerializer<Set<String>>().read(raf);
} else {
stoplist = Collections.emptySet();
}
rows = CachingList.create(
- UniformRAFList.create(raf, new RowBase.Serializer(this), raf.getFilePointer()),
- CACHE_SIZE);
+ UniformRAFList.create(raf, new RowBase.Serializer(this), raf.getFilePointer()),
+ CACHE_SIZE);
}
@Override
};
public static final class IndexEntry implements RAFSerializable<Index.IndexEntry> {
- private final Index index;
public final String token;
private final String normalizedToken;
public final int startRow;
public final int numRows; // doesn't count the token row!
public List<HtmlEntry> htmlEntries;
- private int[] htmlEntryIndices;
public IndexEntry(final Index index, final String token, final String normalizedToken,
- final int startRow, final int numRows) {
- this.index = index;
+ final int startRow, final int numRows) {
assert token.equals(token.trim());
assert token.length() > 0;
this.token = token;
}
public IndexEntry(final Index index, final DataInput raf) throws IOException {
- this.index = index;
token = raf.readUTF();
if (index.dict.dictFileVersion >= 7) {
startRow = StringUtil.readVarInt(raf);
}
final boolean hasNormalizedForm = raf.readBoolean();
normalizedToken = hasNormalizedForm ? raf.readUTF() : token;
- htmlEntryIndices = null;
if (index.dict.dictFileVersion >= 7) {
int size = StringUtil.readVarInt(raf);
- htmlEntryIndices = new int[size];
- for (int i = 0; i < size; ++i) {
- htmlEntryIndices[i] = StringUtil.readVarInt(raf);
- }
- this.htmlEntries = CachingList.create(new AbstractList<HtmlEntry>() {
- @Override
- public HtmlEntry get(int i) {
- return index.dict.htmlEntries.get(htmlEntryIndices[i]);
- }
- @Override
- public int size() {
- return htmlEntryIndices.length;
+ if (size == 0) {
+ this.htmlEntries = Collections.emptyList();
+ } else {
+ final int[] htmlEntryIndices = new int[size];
+ for (int i = 0; i < size; ++i) {
+ htmlEntryIndices[i] = StringUtil.readVarInt(raf);
}
- }, 1);
+ this.htmlEntries = new AbstractList<HtmlEntry>() {
+ @Override
+ public HtmlEntry get(int i) {
+ return index.dict.htmlEntries.get(htmlEntryIndices[i]);
+ }
+ @Override
+ public int size() {
+ return htmlEntryIndices.length;
+ }
+ };
+ }
} else if (index.dict.dictFileVersion >= 6) {
this.htmlEntries = CachingList.create(
- RAFList.create((RandomAccessFile)raf, index.dict.htmlEntryIndexSerializer,
- ((RandomAccessFile)raf).getFilePointer(), index.dict.dictFileVersion), 1);
+ RAFList.create((RandomAccessFile)raf, index.dict.htmlEntryIndexSerializer,
+ ((RandomAccessFile)raf).getFilePointer(), index.dict.dictFileVersion,
+ index.dict.dictInfo + " htmlEntries: "), 1);
} else {
this.htmlEntries = Collections.emptyList();
}
public IndexEntry findExact(final String exactToken) {
final int result = Collections.binarySearch(
- TransformingList.create(sortedIndexEntries, INDEX_ENTRY_TO_TOKEN), exactToken,
- getSortComparator());
+ TransformingList.create(sortedIndexEntries, INDEX_ENTRY_TO_TOKEN), exactToken,
+ getSortComparator());
if (result >= 0) {
return sortedIndexEntries.get(result);
}
return index != -1 ? sortedIndexEntries.get(index) : null;
}
+ private int compareIdx(String token, final Comparator sortCollator, int idx) {
+ final IndexEntry entry = sortedIndexEntries.get(idx);
+ return NormalizeComparator.compareWithoutDash(token, entry.normalizedToken(), sortCollator, dict.dictFileVersion);
+ }
+
public int findInsertionPointIndex(String token, final AtomicBoolean interrupted) {
token = normalizeToken(token);
}
final IndexEntry midEntry = sortedIndexEntries.get(mid);
- int comp = NormalizeComparator.compareWithoutLeadingDash(token, midEntry.normalizedToken(), sortCollator, dict.dictFileVersion);
+ int comp = NormalizeComparator.compareWithoutDash(token, midEntry.normalizedToken(), sortCollator, dict.dictFileVersion);
if (comp == 0)
comp = sortCollator.compare(token, midEntry.normalizedToken());
if (comp == 0) {
} else if (comp < 0) {
// System.out.println("Upper bound: " + midEntry + ", norm=" +
// midEntry.normalizedToken() + ", mid=" + mid);
- end = mid;
+
+ // Hack for robustness if sort order is broken
+ if (mid + 2 < end &&
+ compareIdx(token, sortCollator, mid + 1) > 0 &&
+ compareIdx(token, sortCollator, mid + 2) > 0) {
+ start = mid;
+ } else {
+ end = mid;
+ }
} else {
// System.out.println("Lower bound: " + midEntry + ", norm=" +
// midEntry.normalizedToken() + ", mid=" + mid);
- start = mid + 1;
+
+ // Hack for robustness if sort order is broken
+ if (mid - 2 >= start &&
+ compareIdx(token, sortCollator, mid - 1) < 0 &&
+ compareIdx(token, sortCollator, mid - 2) < 0) {
+ end = mid + 1;
+ } else {
+ start = mid + 1;
+ }
}
}
private static final int MAX_SEARCH_ROWS = 1000;
- private final Map<String, Integer> prefixToNumRows = new LinkedHashMap<String, Integer>();
+ private final Map<String, Integer> prefixToNumRows = new HashMap<String, Integer>();
private synchronized final int getUpperBoundOnRowsStartingWith(final String normalizedPrefix,
final int maxRows, final AtomicBoolean interrupted) {
}
public final List<RowBase> multiWordSearch(
- final String searchText, final List<String> searchTokens,
- final AtomicBoolean interrupted) {
+ final String searchText, final List<String> searchTokens,
+ final AtomicBoolean interrupted) {
final long startMills = System.currentTimeMillis();
final List<RowBase> result = new ArrayList<RowBase>();
- final Set<String> normalizedNonStoplist = new LinkedHashSet<String>();
+ final Set<String> normalizedNonStoplist = new HashSet<String>();
String bestPrefix = null;
int leastRows = Integer.MAX_VALUE;
if (!stoplist.contains(searchToken)) {
if (normalizedNonStoplist.add(normalized)) {
final int numRows = getUpperBoundOnRowsStartingWith(normalized,
- MAX_SEARCH_ROWS, interrupted);
+ MAX_SEARCH_ROWS, interrupted);
if (numRows != -1 && numRows < leastRows) {
if (numRows == 0) {
// We really are done here.
System.out.println("Everything was in the stoplist!");
}
System.out.println("Searching using prefix: " + bestPrefix + ", leastRows=" + leastRows
- + ", searchTokens=" + searchTokens);
+ + ", searchTokens=" + searchTokens);
// Place to store the things that match.
final Map<RowMatchType, List<RowBase>> matches = new EnumMap<RowMatchType, List<RowBase>>(
- RowMatchType.class);
+ RowMatchType.class);
for (final RowMatchType rowMatchType : RowMatchType.values()) {
if (rowMatchType != RowMatchType.NO_MATCH) {
matches.put(rowMatchType, new ArrayList<RowBase>());
}
rowsAlreadySeen.add(rowKey);
final RowMatchType matchType = row.matches(searchTokens, pattern, normalizer(),
- swapPairEntries);
+ swapPairEntries);
if (matchType != RowMatchType.NO_MATCH) {
matches.get(matchType).add(row);
++matchCount;
// Sort them into a reasonable order.
final RowBase.LengthComparator lengthComparator = new RowBase.LengthComparator(
- swapPairEntries);
+ swapPairEntries);
for (final Collection<RowBase> rows : matches.values()) {
final List<RowBase> ordered = new ArrayList<RowBase>(rows);
Collections.sort(ordered, lengthComparator);
}
private String normalizeToken(final String searchToken) {
- if (TransliteratorManager.init(null)) {
+ if (TransliteratorManager.init(null, null)) {
final Transliterator normalizer = normalizer();
return normalizer.transliterate(searchToken);
} else {
projects / Dictionary.git / blobdiff