// See the License for the specific language governing permissions and
// limitations under the License.
-/**
- *
- */
-
package com.hughes.android.dictionary.engine;
import com.hughes.android.dictionary.DictionaryInfo;
import com.ibm.icu.text.Transliterator;
import java.io.DataInput;
-import java.io.DataInputStream;
import java.io.DataOutput;
import java.io.IOException;
import java.io.PrintStream;
import java.io.RandomAccessFile;
-import java.nio.channels.Channels;
import java.nio.channels.FileChannel;
import java.util.AbstractList;
import java.util.ArrayList;
public final class Index implements RAFSerializable<Index> {
- static final int CACHE_SIZE = 5000;
+ private static final int CACHE_SIZE = 5000;
public final Dictionary dict;
// persisted: tells how the entries are sorted.
public final Language sortLanguage;
- final String normalizerRules;
+ public final String normalizerRules;
// Built from the two above.
private Transliterator normalizer;
public final boolean swapPairEntries;
// Version 2:
- int mainTokenCount = -1;
+ @SuppressWarnings("WeakerAccess")
+ public int mainTokenCount = -1;
// --------------------------------------------------------------------------
this.sortLanguage = sortLanguage;
this.normalizerRules = normalizerRules;
this.swapPairEntries = swapPairEntries;
- sortedIndexEntries = new ArrayList<IndexEntry>();
+ sortedIndexEntries = new ArrayList<>();
this.stoplist = stoplist;
- rows = new ArrayList<RowBase>();
+ rows = new ArrayList<>();
normalizer = null;
}
/**
* Deferred initialization because it can be slow.
*/
+ @SuppressWarnings("WeakerAccess")
public synchronized Transliterator normalizer() {
if (normalizer == null) {
normalizer = TransliteratorManager.get(normalizerRules);
* Note that using this comparator probably involves doing too many text
* normalizations.
*/
+ @SuppressWarnings("WeakerAccess")
public NormalizeComparator getSortComparator() {
return new NormalizeComparator(normalizer(), sortLanguage.getCollator(), dict.dictFileVersion);
}
sortLanguage = Language.lookup(languageCode);
normalizerRules = raf.readUTF();
swapPairEntries = raf.readBoolean();
- if (sortLanguage == null) {
- throw new IOException("Unsupported language: " + languageCode);
- }
if (dict.dictFileVersion >= 2) {
mainTokenCount = raf.readInt();
}
dict.dictFileVersion, dict.dictInfo + " idx " + languageCode + ": "), CACHE_SIZE, true);
if (dict.dictFileVersion >= 7) {
int count = StringUtil.readVarInt(raf);
- stoplist = new HashSet<String>(count);
+ stoplist = new HashSet<>(count);
for (int i = 0; i < count; ++i) {
stoplist.add(raf.readUTF());
}
raf.writeUTF(sortLanguage.getIsoCode());
raf.writeUTF(normalizerRules);
raf.writeBoolean(swapPairEntries);
- if (dict.dictFileVersion >= 2) {
- raf.writeInt(mainTokenCount);
- }
+ raf.writeInt(mainTokenCount);
RAFList.write(raf, sortedIndexEntries, new IndexEntrySerializer(null), 32, true);
StringUtil.writeVarInt(raf, stoplist.size());
for (String i : stoplist) {
private final class IndexEntrySerializer implements RAFSerializer<IndexEntry> {
private final FileChannel ch;
- public IndexEntrySerializer(FileChannel ch) {
+ IndexEntrySerializer(FileChannel ch) {
this.ch = ch;
}
public void write(DataOutput raf, IndexEntry t) throws IOException {
t.write(raf);
}
- };
+ }
public static final class IndexEntry implements RAFSerializable<Index.IndexEntry> {
public final String token;
private final String normalizedToken;
public final int startRow;
- public final int numRows; // doesn't count the token row!
+ final int numRows; // doesn't count the token row!
public List<HtmlEntry> htmlEntries;
public IndexEntry(final Index index, final String token, final String normalizedToken,
this.normalizedToken = normalizedToken;
this.startRow = startRow;
this.numRows = numRows;
- this.htmlEntries = new ArrayList<HtmlEntry>();
+ this.htmlEntries = new ArrayList<>();
}
- public IndexEntry(final Index index, final FileChannel ch, final DataInput raf) throws IOException {
+ IndexEntry(final Index index, final FileChannel ch, final DataInput raf) throws IOException {
token = raf.readUTF();
if (index.dict.dictFileVersion >= 7) {
startRow = StringUtil.readVarInt(raf);
return String.format("%s@%d(%d)", token, startRow, numRows);
}
- public String normalizedToken() {
+ String normalizedToken() {
return normalizedToken;
}
}
- static final TransformingList.Transformer<IndexEntry, String> INDEX_ENTRY_TO_TOKEN = new TransformingList.Transformer<IndexEntry, String>() {
+ private static final TransformingList.Transformer<IndexEntry, String> INDEX_ENTRY_TO_TOKEN = new TransformingList.Transformer<IndexEntry, String>() {
@Override
public String transform(IndexEntry t1) {
return t1.token;
return index != -1 ? sortedIndexEntries.get(index) : null;
}
- private int compareIdx(String token, final Comparator sortCollator, int idx) {
+ private int compareIdx(String token, final Comparator<Object> sortCollator, int idx) {
final IndexEntry entry = sortedIndexEntries.get(idx);
return NormalizeComparator.compareWithoutDash(token, entry.normalizedToken(), sortCollator, dict.dictFileVersion);
}
- public int findInsertionPointIndex(String token, final AtomicBoolean interrupted) {
+ private int findMatchLen(final Comparator<Object> sortCollator, String a, String b) {
+ int start = 0;
+ int end = Math.min(a.length(), b.length());
+ while (start < end)
+ {
+ int mid = (start + end + 1) / 2;
+ if (sortCollator.compare(a.substring(0, mid), b.substring(0, mid)) == 0)
+ start = mid;
+ else
+ end = mid - 1;
+ }
+ return start;
+ }
+
+ private int findInsertionPointIndex(String token, final AtomicBoolean interrupted) {
token = normalizeToken(token);
int start = 0;
int end = sortedIndexEntries.size();
- final Comparator sortCollator = sortLanguage.getCollator();
+ final Comparator<Object> sortCollator = sortLanguage.getCollator();
while (start < end) {
final int mid = (start + end) / 2;
if (interrupted.get()) {
if (comp == 0)
comp = sortCollator.compare(token, midEntry.normalizedToken());
if (comp == 0) {
- final int result = windBackCase(token, mid, interrupted);
- return result;
+ return windBackCase(token, mid, interrupted);
} else if (comp < 0) {
// System.out.println("Upper bound: " + midEntry + ", norm=" +
// midEntry.normalizedToken() + ", mid=" + mid);
}
}
+ // if the word before is the better match, move
+ // our result to it
+ if (start > 0 && start < sortedIndexEntries.size()) {
+ String prev = sortedIndexEntries.get(start - 1).normalizedToken();
+ String next = sortedIndexEntries.get(start).normalizedToken();
+ if (findMatchLen(sortCollator, token, prev) >= findMatchLen(sortCollator, token, next))
+ start--;
+ }
+
// If we search for a substring of a string that's in there, return
// that.
int result = Math.min(start, sortedIndexEntries.size() - 1);
private static final int MAX_SEARCH_ROWS = 1000;
- private final Map<String, Integer> prefixToNumRows = new HashMap<String, Integer>();
+ private final Map<String, Integer> prefixToNumRows = new HashMap<>();
private synchronized final int getUpperBoundOnRowsStartingWith(final String normalizedPrefix,
final int maxRows, final AtomicBoolean interrupted) {
return -1;
}
final IndexEntry indexEntry = sortedIndexEntries.get(index);
- if (!indexEntry.normalizedToken.startsWith(normalizedPrefix)) {
+ if (!indexEntry.normalizedToken.startsWith(normalizedPrefix) &&
+ !NormalizeComparator.withoutDash(indexEntry.normalizedToken).startsWith(normalizedPrefix)) {
break;
}
rowCount += indexEntry.numRows + indexEntry.htmlEntries.size();
break;
}
}
- prefixToNumRows.put(normalizedPrefix, numRows);
+ prefixToNumRows.put(normalizedPrefix, rowCount);
return rowCount;
}
final String searchText, final List<String> searchTokens,
final AtomicBoolean interrupted) {
final long startMills = System.currentTimeMillis();
- final List<RowBase> result = new ArrayList<RowBase>();
+ final List<RowBase> result = new ArrayList<>();
- final Set<String> normalizedNonStoplist = new HashSet<String>();
+ final Set<String> normalizedNonStoplist = new HashSet<>();
String bestPrefix = null;
int leastRows = Integer.MAX_VALUE;
+ ", searchTokens=" + searchTokens);
// Place to store the things that match.
- final Map<RowMatchType, List<RowBase>> matches = new EnumMap<RowMatchType, List<RowBase>>(
- RowMatchType.class);
+ final Map<RowMatchType, List<RowBase>> matches = new EnumMap<>(
+ RowMatchType.class);
for (final RowMatchType rowMatchType : RowMatchType.values()) {
if (rowMatchType != RowMatchType.NO_MATCH) {
matches.put(rowMatchType, new ArrayList<RowBase>());
final String searchToken = bestPrefix;
final int insertionPointIndex = findInsertionPointIndex(searchToken, interrupted);
- final Set<RowKey> rowsAlreadySeen = new HashSet<RowBase.RowKey>();
+ final Set<RowKey> rowsAlreadySeen = new HashSet<>();
for (int index = insertionPointIndex; index < sortedIndexEntries.size()
&& matchCount < MAX_SEARCH_ROWS; ++index) {
if (interrupted.get()) {
return null;
}
final IndexEntry indexEntry = sortedIndexEntries.get(index);
- if (!indexEntry.normalizedToken.startsWith(searchToken)) {
+ if (!indexEntry.normalizedToken.startsWith(searchToken) &&
+ !NormalizeComparator.withoutDash(indexEntry.normalizedToken).startsWith(searchToken)) {
break;
}
final RowBase.LengthComparator lengthComparator = new RowBase.LengthComparator(
swapPairEntries);
for (final Collection<RowBase> rows : matches.values()) {
- final List<RowBase> ordered = new ArrayList<RowBase>(rows);
+ final List<RowBase> ordered = new ArrayList<>(rows);
Collections.sort(ordered, lengthComparator);
result.addAll(ordered);
}
projects / Dictionary.git / blobdiff