+// Copyright 2011 Google Inc. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
package com.hughes.android.dictionary.parser;
import java.io.BufferedReader;
import java.io.InputStreamReader;
import java.nio.charset.Charset;
import java.util.Arrays;
+import java.util.Collections;
import java.util.LinkedHashSet;
import java.util.List;
import java.util.Set;
import java.util.regex.Pattern;
import com.hughes.android.dictionary.engine.DictionaryBuilder;
-import com.hughes.android.dictionary.engine.EntryData;
+import com.hughes.android.dictionary.engine.EntrySource;
+import com.hughes.android.dictionary.engine.IndexedEntry;
import com.hughes.android.dictionary.engine.EntryTypeName;
import com.hughes.android.dictionary.engine.IndexBuilder;
import com.hughes.android.dictionary.engine.Language;
import com.hughes.android.dictionary.engine.PairEntry;
import com.hughes.android.dictionary.engine.PairEntry.Pair;
-public class DictFileParser {
+public class DictFileParser implements Parser {
static final Logger logger = Logger.getLogger(DictFileParser.class.getName());
public static final Pattern PIPE = Pattern.compile("\\|");
static final Pattern SPACES = Pattern.compile("\\s+");
- static final Pattern DE_NOUN = Pattern.compile("([^ ]+) *\\{(m|f|n|pl)\\}");
- static final Pattern EN_VERB = Pattern.compile("^to ([^ ]+)");
+// static final Pattern DE_NOUN = Pattern.compile("([^ ]+) *\\{(m|f|n|pl)\\}");
+// static final Pattern EN_VERB = Pattern.compile("^to ([^ ]+)");
static final Pattern BRACKETED = Pattern.compile("\\[([^]]+)\\]");
static final Pattern PARENTHESIZED = Pattern.compile("\\(([^)]+)\\)");
static final Pattern CURLY_BRACED = Pattern.compile("\\{([^}]+)\\}");
static final Pattern NON_CHAR_DASH = Pattern.compile("[^-'\\p{L}0-9]+");
- static final Pattern NON_CHAR = Pattern.compile("[^\\p{L}0-9]+");
+ public static final Pattern NON_CHAR = Pattern.compile("[^\\p{L}0-9]+");
static final Pattern TRIM_PUNC = Pattern.compile("^[^\\p{L}0-9]+|[^\\p{L}0-9]+$");
final IndexBuilder[] langIndexBuilders;
final IndexBuilder bothIndexBuilder;
+ EntrySource entrySource;
+
// final Set<String> alreadyDone = new HashSet<String>();
public DictFileParser(final Charset charset, boolean flipCols,
this.bothIndexBuilder = bothIndexBuilder;
}
- public void parseFile(final File file) throws IOException {
+ @Override
+ public void parse(final File file, final EntrySource entrySouce, final int pageLimit) throws IOException {
+ this.entrySource = entrySouce;
final BufferedReader reader = new BufferedReader(new InputStreamReader(new FileInputStream(file), charset));
String line;
int count = 0;
while ((line = reader.readLine()) != null) {
+ if (pageLimit >= 0 && count >= pageLimit) {
+ return;
+ }
if (count % 10000 == 0) {
logger.info("count=" + count + ", line=" + line);
}
fields[0] = fields[1];
fields[1] = temp;
}
-
+
final String[][] subfields = new String[2][];
if (subfieldSplit != null) {
subfields[0] = subfieldSplit.split(fields[0]);
subfields[1] = new String[] { fields[1] };
}
- final Pair[] pairs = new Pair[subfields[0].length];
- for (int i = 0; i < pairs.length; ++i) {
+ final PairEntry pairEntry = new PairEntry(entrySource);
+ for (int i = 0; i < subfields[0].length; ++i) {
subfields[0][i] = subfields[0][i].trim();
subfields[1][i] = subfields[1][i].trim();
- pairs[i] = new Pair(subfields[0][i], subfields[1][i]);
+ if (subfields[0][i].length() == 0 && subfields[1][i].length() == 0) {
+ logger.warning("Empty pair: " + line);
+ continue;
+ }
+ if (subfields[0][i].length() == 0) {
+ subfields[0][i] = "__";
+ }
+ if (subfields[1][i].length() == 0) {
+ subfields[1][i] = "__";
+ }
+ pairEntry.pairs.add(new Pair(subfields[0][i], subfields[1][i]));
}
- final PairEntry pairEntry = new PairEntry(pairs);
- final EntryData entryData = new EntryData(dictBuilder.dictionary.pairEntries.size(), pairEntry);
- dictBuilder.dictionary.pairEntries.add(pairEntry);
+ final IndexedEntry entryData = new IndexedEntry(pairEntry);
for (int l = 0; l < 2; ++l) {
// alreadyDone.clear();
}
private void parseFieldGeneric(final IndexBuilder indexBuilder, String field,
- final EntryData entryData, final int subfieldIdx, final int numSubFields) {
+ final IndexedEntry entryData, final int subfieldIdx, final int numSubFields) {
// remove bracketed and parenthesized stuff.
final StringBuilder bracketed = new StringBuilder();
final StringBuilder parenthesized = new StringBuilder();
for (String token : tokens) {
token = TRIM_PUNC.matcher(token).replaceAll("");
if (/*!alreadyDone.contains(token) && */token.length() > 0) {
- final List<EntryData> entries = indexBuilder.getOrCreateEntries(token, entryTypeName);
- entries.add(entryData);
+ indexBuilder.addEntryWithTokens(entryData, Collections.singleton(token), entryTypeName);
// alreadyDone.add(token);
// also split words on dashes, do them, too.
final String[] dashed = token.split("-");
for (final String dashedToken : dashed) {
if (/*!alreadyDone.contains(dashedToken) && */dashedToken.length() > 0) {
- final List<EntryData> dashEntries = indexBuilder.getOrCreateEntries(dashedToken, EntryTypeName.PART_OF_HYPHENATED);
- dashEntries.add(entryData);
+ indexBuilder.addEntryWithTokens(entryData, Collections.singleton(dashedToken), EntryTypeName.PART_OF_HYPHENATED);
}
}
}
for (final String token : bracketedTokens) {
assert !token.contains("-");
if (/*!alreadyDone.contains(token) && */token.length() > 0) {
- final List<EntryData> entries = indexBuilder.getOrCreateEntries(token, EntryTypeName.BRACKETED);
- entries.add(entryData);
+ indexBuilder.addEntryWithTokens(entryData, Collections.singleton(token), EntryTypeName.BRACKETED);
}
}
for (final String token : parenTokens) {
assert !token.contains("-");
if (/*!alreadyDone.contains(token) && */token.length() > 0) {
- final List<EntryData> entries = indexBuilder.getOrCreateEntries(token, EntryTypeName.PARENTHESIZED);
- entries.add(entryData);
+ indexBuilder.addEntryWithTokens(entryData, Collections.singleton(token), EntryTypeName.PARENTHESIZED);
}
}
}
private String parseField_DE(final IndexBuilder indexBuilder, String field,
- final EntryData entryData, final int subfieldIdx) {
+ final IndexedEntry entryData, final int subfieldIdx) {
// final Matcher matcher = DE_NOUN.matcher(field);
// while (matcher.find()) {
}
private String parseField_EN(final IndexBuilder indexBuilder, String field,
- final EntryData entryData, final int subfieldIdx) {
+ final IndexedEntry entryData, final int subfieldIdx) {
if (field.startsWith("to ")) {
field = field.substring(3);
}
projects / DictionaryPC.git / blobdiff