go

[DictionaryPC.git] / src / com / hughes / android / dictionary / parser / DictFileParser.java

diff --git a/src/com/hughes/android/dictionary/parser/DictFileParser.java b/src/com/hughes/android/dictionary/parser/DictFileParser.java
index 1e01ae2bf9b2e4be43b44c400137a5649af8061c..2f53fdcf36f130605904c54bcb1a5084cd21ff1b 100644 (file)
--- a/src/com/hughes/android/dictionary/parser/DictFileParser.java
+++ b/src/com/hughes/android/dictionary/parser/DictFileParser.java
@@ -6,7 +6,10 @@ import java.io.FileInputStream;
 import java.io.IOException;
 import java.io.InputStreamReader;
 import java.nio.charset.Charset;
+import java.util.Arrays;
+import java.util.LinkedHashSet;
 import java.util.List;
+import java.util.Set;
 import java.util.logging.Logger;
 import java.util.regex.Matcher;
 import java.util.regex.Pattern;
@@ -263,6 +266,13 @@ public class DictFileParser {
     }
     return field;
   }
+  
+  public static final Set<String> tokenize(final String text, final Pattern pattern) {
+    final String[] split = pattern.split(text);
+    final Set<String> result = new LinkedHashSet<String>(Arrays.asList(split));
+    result.remove("");
+    return result;
+  }
 
 
 }