go

[DictionaryPC.git] / src / com / hughes / android / dictionary / parser / DictFileParser.java

diff --git a/src/com/hughes/android/dictionary/parser/DictFileParser.java b/src/com/hughes/android/dictionary/parser/DictFileParser.java
index 1e01ae2bf9b2e4be43b44c400137a5649af8061c..909a6b77577f573e50f6f3175454023b60c4d363 100644 (file)
--- a/src/com/hughes/android/dictionary/parser/DictFileParser.java
+++ b/src/com/hughes/android/dictionary/parser/DictFileParser.java
@@ -6,13 +6,16 @@ import java.io.FileInputStream;
 import java.io.IOException;
 import java.io.InputStreamReader;
 import java.nio.charset.Charset;
+import java.util.Arrays;
+import java.util.LinkedHashSet;
 import java.util.List;
+import java.util.Set;
 import java.util.logging.Logger;
 import java.util.regex.Matcher;
 import java.util.regex.Pattern;
 
 import com.hughes.android.dictionary.engine.DictionaryBuilder;
-import com.hughes.android.dictionary.engine.EntryData;
+import com.hughes.android.dictionary.engine.IndexedEntry;
 import com.hughes.android.dictionary.engine.EntryTypeName;
 import com.hughes.android.dictionary.engine.IndexBuilder;
 import com.hughes.android.dictionary.engine.Language;
@@ -31,8 +34,8 @@ public class DictFileParser {
   public static final Pattern PIPE = Pattern.compile("\\|");
   
   static final Pattern SPACES = Pattern.compile("\\s+");
-  static final Pattern DE_NOUN = Pattern.compile("([^ ]+) *\\{(m|f|n|pl)\\}");
-  static final Pattern EN_VERB = Pattern.compile("^to ([^ ]+)");
+//  static final Pattern DE_NOUN = Pattern.compile("([^ ]+) *\\{(m|f|n|pl)\\}");
+//  static final Pattern EN_VERB = Pattern.compile("^to ([^ ]+)");
   
   static final Pattern BRACKETED = Pattern.compile("\\[([^]]+)\\]");
   static final Pattern PARENTHESIZED = Pattern.compile("\\(([^)]+)\\)");
@@ -99,7 +102,7 @@ public class DictFileParser {
       fields[0] = fields[1];
       fields[1] = temp;
     }
-    
+
     final String[][] subfields = new String[2][];
       if (subfieldSplit != null) {
       subfields[0] = subfieldSplit.split(fields[0]);
@@ -113,15 +116,13 @@ public class DictFileParser {
       subfields[1] = new String[] { fields[1] };
     }
     
-    final Pair[] pairs = new Pair[subfields[0].length];
-    for (int i = 0; i < pairs.length; ++i) {
+    final PairEntry pairEntry = new PairEntry();
+    for (int i = 0; i < subfields[0].length; ++i) {
       subfields[0][i] = subfields[0][i].trim();
       subfields[1][i] = subfields[1][i].trim();
-      pairs[i] = new Pair(subfields[0][i], subfields[1][i]);
+      pairEntry.pairs.add(new Pair(subfields[0][i], subfields[1][i]));
     }
-    final PairEntry pairEntry = new PairEntry(pairs);
-    final EntryData entryData = new EntryData(dictBuilder.dictionary.pairEntries.size(), pairEntry);
-    dictBuilder.dictionary.pairEntries.add(pairEntry);
+    final IndexedEntry entryData = new IndexedEntry(pairEntry);
     
     for (int l = 0; l < 2; ++l) {
       // alreadyDone.clear();
@@ -140,7 +141,7 @@ public class DictFileParser {
   }
 
   private void parseFieldGeneric(final IndexBuilder indexBuilder, String field,
-      final EntryData entryData, final int subfieldIdx, final int numSubFields) {
+      final IndexedEntry entryData, final int subfieldIdx, final int numSubFields) {
     // remove bracketed and parenthesized stuff.
     final StringBuilder bracketed = new StringBuilder(); 
     final StringBuilder parenthesized = new StringBuilder();
@@ -196,7 +197,7 @@ public class DictFileParser {
     for (String token : tokens) {
       token = TRIM_PUNC.matcher(token).replaceAll("");
       if (/*!alreadyDone.contains(token) && */token.length() > 0) {
-        final List<EntryData> entries = indexBuilder.getOrCreateEntries(token, entryTypeName);
+        final List<IndexedEntry> entries = indexBuilder.getOrCreateEntries(token, entryTypeName);
         entries.add(entryData);
         // alreadyDone.add(token);
         
@@ -205,7 +206,7 @@ public class DictFileParser {
           final String[] dashed = token.split("-");
           for (final String dashedToken : dashed) {
             if (/*!alreadyDone.contains(dashedToken) && */dashedToken.length() > 0) {
-              final List<EntryData> dashEntries = indexBuilder.getOrCreateEntries(dashedToken, EntryTypeName.PART_OF_HYPHENATED);
+              final List<IndexedEntry> dashEntries = indexBuilder.getOrCreateEntries(dashedToken, EntryTypeName.PART_OF_HYPHENATED);
               dashEntries.add(entryData);
             }
           }
@@ -219,7 +220,7 @@ public class DictFileParser {
     for (final String token : bracketedTokens) {
       assert !token.contains("-");
       if (/*!alreadyDone.contains(token) && */token.length() > 0) {
-        final List<EntryData> entries = indexBuilder.getOrCreateEntries(token, EntryTypeName.BRACKETED);
+        final List<IndexedEntry> entries = indexBuilder.getOrCreateEntries(token, EntryTypeName.BRACKETED);
         entries.add(entryData);
       }
     }
@@ -229,7 +230,7 @@ public class DictFileParser {
     for (final String token : parenTokens) {
       assert !token.contains("-");
       if (/*!alreadyDone.contains(token) && */token.length() > 0) {
-        final List<EntryData> entries = indexBuilder.getOrCreateEntries(token, EntryTypeName.PARENTHESIZED);
+        final List<IndexedEntry> entries = indexBuilder.getOrCreateEntries(token, EntryTypeName.PARENTHESIZED);
         entries.add(entryData);
       }
     }
@@ -237,7 +238,7 @@ public class DictFileParser {
   }
 
   private String parseField_DE(final IndexBuilder indexBuilder, String field,
-      final EntryData entryData, final int subfieldIdx) {
+      final IndexedEntry entryData, final int subfieldIdx) {
     
 //    final Matcher matcher = DE_NOUN.matcher(field);
 //    while (matcher.find()) {
@@ -257,12 +258,19 @@ public class DictFileParser {
   }
   
   private String parseField_EN(final IndexBuilder indexBuilder, String field,
-      final EntryData entryData, final int subfieldIdx) {
+      final IndexedEntry entryData, final int subfieldIdx) {
     if (field.startsWith("to ")) {
       field = field.substring(3);
     }
     return field;
   }
+  
+  public static final Set<String> tokenize(final String text, final Pattern pattern) {
+    final String[] split = pattern.split(text);
+    final Set<String> result = new LinkedHashSet<String>(Arrays.asList(split));
+    result.remove("");
+    return result;
+  }
 
 
 }