Wiktionary upgrade!

[DictionaryPC.git] / src / com / hughes / android / dictionary / parser / DictFileParser.java

diff --git a/src/com/hughes/android/dictionary/parser/DictFileParser.java b/src/com/hughes/android/dictionary/parser/DictFileParser.java
index cea4f90bb4494edf8a81c7302c6e4978ac997fc0..f4f68c8fae466586f1b7f39b87bb77854efe02ff 100644 (file)
--- a/src/com/hughes/android/dictionary/parser/DictFileParser.java
+++ b/src/com/hughes/android/dictionary/parser/DictFileParser.java
@@ -21,6 +21,7 @@ import java.io.IOException;
 import java.io.InputStreamReader;
 import java.nio.charset.Charset;
 import java.util.Arrays;
+import java.util.Collections;
 import java.util.LinkedHashSet;
 import java.util.List;
 import java.util.Set;
@@ -29,6 +30,7 @@ import java.util.regex.Matcher;
 import java.util.regex.Pattern;
 
 import com.hughes.android.dictionary.engine.DictionaryBuilder;
+import com.hughes.android.dictionary.engine.EntrySource;
 import com.hughes.android.dictionary.engine.IndexedEntry;
 import com.hughes.android.dictionary.engine.EntryTypeName;
 import com.hughes.android.dictionary.engine.IndexBuilder;
@@ -70,6 +72,8 @@ public class DictFileParser {
   final IndexBuilder[] langIndexBuilders;
   final IndexBuilder bothIndexBuilder;
   
+  EntrySource entrySource;
+  
   // final Set<String> alreadyDone = new HashSet<String>();
     
   public DictFileParser(final Charset charset, boolean flipCols,
@@ -85,7 +89,8 @@ public class DictFileParser {
     this.bothIndexBuilder = bothIndexBuilder;
   }
 
-  public void parseFile(final File file) throws IOException {
+  public void parseFile(final File file, final EntrySource entrySouce) throws IOException {
+    this.entrySource = entrySouce;
     final BufferedReader reader = new BufferedReader(new InputStreamReader(new FileInputStream(file), charset));
     String line;
     int count = 0;
@@ -130,10 +135,20 @@ public class DictFileParser {
       subfields[1] = new String[] { fields[1] };
     }
     
-    final PairEntry pairEntry = new PairEntry();
+    final PairEntry pairEntry = new PairEntry(entrySource);
     for (int i = 0; i < subfields[0].length; ++i) {
       subfields[0][i] = subfields[0][i].trim();
       subfields[1][i] = subfields[1][i].trim();
+      if (subfields[0][i].length() == 0 && subfields[1][i].length() == 0) {
+        logger.warning("Empty pair: " + line);
+        continue;
+      }
+      if (subfields[0][i].length() == 0) {
+        subfields[0][i] = "__";
+      }
+      if (subfields[1][i].length() == 0) {
+        subfields[1][i] = "__";
+      }
       pairEntry.pairs.add(new Pair(subfields[0][i], subfields[1][i]));
     }
     final IndexedEntry entryData = new IndexedEntry(pairEntry);
@@ -211,8 +226,7 @@ public class DictFileParser {
     for (String token : tokens) {
       token = TRIM_PUNC.matcher(token).replaceAll("");
       if (/*!alreadyDone.contains(token) && */token.length() > 0) {
-        final List<IndexedEntry> entries = indexBuilder.getOrCreateEntries(token, entryTypeName);
-        entries.add(entryData);
+        indexBuilder.addEntryWithTokens(entryData, Collections.singleton(token), entryTypeName);
         // alreadyDone.add(token);
         
         // also split words on dashes, do them, too.
@@ -220,8 +234,7 @@ public class DictFileParser {
           final String[] dashed = token.split("-");
           for (final String dashedToken : dashed) {
             if (/*!alreadyDone.contains(dashedToken) && */dashedToken.length() > 0) {
-              final List<IndexedEntry> dashEntries = indexBuilder.getOrCreateEntries(dashedToken, EntryTypeName.PART_OF_HYPHENATED);
-              dashEntries.add(entryData);
+              indexBuilder.addEntryWithTokens(entryData, Collections.singleton(dashedToken), EntryTypeName.PART_OF_HYPHENATED);
             }
           }
         }
@@ -234,8 +247,7 @@ public class DictFileParser {
     for (final String token : bracketedTokens) {
       assert !token.contains("-");
       if (/*!alreadyDone.contains(token) && */token.length() > 0) {
-        final List<IndexedEntry> entries = indexBuilder.getOrCreateEntries(token, EntryTypeName.BRACKETED);
-        entries.add(entryData);
+        indexBuilder.addEntryWithTokens(entryData, Collections.singleton(token), EntryTypeName.BRACKETED);
       }
     }
     
@@ -244,8 +256,7 @@ public class DictFileParser {
     for (final String token : parenTokens) {
       assert !token.contains("-");
       if (/*!alreadyDone.contains(token) && */token.length() > 0) {
-        final List<IndexedEntry> entries = indexBuilder.getOrCreateEntries(token, EntryTypeName.PARENTHESIZED);
-        entries.add(entryData);
+        indexBuilder.addEntryWithTokens(entryData, Collections.singleton(token), EntryTypeName.PARENTHESIZED);
       }
     }