Stoplists, fix location of wikisplits.

[DictionaryPC.git] / src / com / hughes / android / dictionary / parser / DictFileParser.java

diff --git a/src/com/hughes/android/dictionary/parser/DictFileParser.java b/src/com/hughes/android/dictionary/parser/DictFileParser.java
index 909a6b77577f573e50f6f3175454023b60c4d363..67ca43293cbdbf9bde04bd27de5ff03c7f55ce50 100644 (file)
--- a/src/com/hughes/android/dictionary/parser/DictFileParser.java
+++ b/src/com/hughes/android/dictionary/parser/DictFileParser.java
@@ -1,3 +1,17 @@
+// Copyright 2011 Google Inc. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
 package com.hughes.android.dictionary.parser;
 
 import java.io.BufferedReader;
@@ -7,6 +21,7 @@ import java.io.IOException;
 import java.io.InputStreamReader;
 import java.nio.charset.Charset;
 import java.util.Arrays;
+import java.util.Collections;
 import java.util.LinkedHashSet;
 import java.util.List;
 import java.util.Set;
@@ -42,7 +57,7 @@ public class DictFileParser {
   static final Pattern CURLY_BRACED = Pattern.compile("\\{([^}]+)\\}");
   
   static final Pattern NON_CHAR_DASH = Pattern.compile("[^-'\\p{L}0-9]+");
-  static final Pattern NON_CHAR = Pattern.compile("[^\\p{L}0-9]+");
+  public static final Pattern NON_CHAR = Pattern.compile("[^\\p{L}0-9]+");
 
   static final Pattern TRIM_PUNC = Pattern.compile("^[^\\p{L}0-9]+|[^\\p{L}0-9]+$");
 
@@ -197,8 +212,7 @@ public class DictFileParser {
     for (String token : tokens) {
       token = TRIM_PUNC.matcher(token).replaceAll("");
       if (/*!alreadyDone.contains(token) && */token.length() > 0) {
-        final List<IndexedEntry> entries = indexBuilder.getOrCreateEntries(token, entryTypeName);
-        entries.add(entryData);
+        indexBuilder.addEntryWithTokens(entryData, Collections.singleton(token), entryTypeName);
         // alreadyDone.add(token);
         
         // also split words on dashes, do them, too.
@@ -206,8 +220,7 @@ public class DictFileParser {
           final String[] dashed = token.split("-");
           for (final String dashedToken : dashed) {
             if (/*!alreadyDone.contains(dashedToken) && */dashedToken.length() > 0) {
-              final List<IndexedEntry> dashEntries = indexBuilder.getOrCreateEntries(dashedToken, EntryTypeName.PART_OF_HYPHENATED);
-              dashEntries.add(entryData);
+              indexBuilder.addEntryWithTokens(entryData, Collections.singleton(dashedToken), EntryTypeName.PART_OF_HYPHENATED);
             }
           }
         }
@@ -220,8 +233,7 @@ public class DictFileParser {
     for (final String token : bracketedTokens) {
       assert !token.contains("-");
       if (/*!alreadyDone.contains(token) && */token.length() > 0) {
-        final List<IndexedEntry> entries = indexBuilder.getOrCreateEntries(token, EntryTypeName.BRACKETED);
-        entries.add(entryData);
+        indexBuilder.addEntryWithTokens(entryData, Collections.singleton(token), EntryTypeName.BRACKETED);
       }
     }
     
@@ -230,8 +242,7 @@ public class DictFileParser {
     for (final String token : parenTokens) {
       assert !token.contains("-");
       if (/*!alreadyDone.contains(token) && */token.length() > 0) {
-        final List<IndexedEntry> entries = indexBuilder.getOrCreateEntries(token, EntryTypeName.PARENTHESIZED);
-        entries.add(entryData);
+        indexBuilder.addEntryWithTokens(entryData, Collections.singleton(token), EntryTypeName.PARENTHESIZED);
       }
     }