Fixing tests after moving out data.

[DictionaryPC.git] / src / com / hughes / android / dictionary / parser / WikiTokenizer.java

diff --git a/src/com/hughes/android/dictionary/parser/WikiTokenizer.java b/src/com/hughes/android/dictionary/parser/WikiTokenizer.java
index fffe35788ad18d7e9a62427a34bb45ff911c2743..f80605d791b8dd63bd792f6dc9f6f2ac47ac9f15 100644 (file)
--- a/src/com/hughes/android/dictionary/parser/WikiTokenizer.java
+++ b/src/com/hughes/android/dictionary/parser/WikiTokenizer.java
@@ -1,3 +1,17 @@
+// Copyright 2011 Google Inc. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
 package com.hughes.android.dictionary.parser;
 
 import java.util.ArrayList;
@@ -50,10 +64,15 @@ public final class WikiTokenizer {
   
 
   public WikiTokenizer(final String wikiText) {
+    this(wikiText, true);
+  }
+
+  public WikiTokenizer(final String wikiText, final boolean isNewline) {
     this.wikiText = wikiText;
     this.matcher = wikiTokenEvent.matcher(wikiText);
+    justReturnedNewline = isNewline;
   }
-    
+
   private void clear() {
     errors.clear();
     tokenStack.clear();
@@ -197,7 +216,7 @@ public final class WikiTokenizer {
     }
     
     // Eat a newline if we're looking at one:
-    final boolean atNewline = wikiText.charAt(end) == '\n';
+    final boolean atNewline = wikiText.charAt(end) == '\n' || wikiText.charAt(end) == '\u2028';
     if (atNewline) {
       justReturnedNewline = true;
       ++end;