Bug-fixes to WikiTokenizer (handle weird line-feed), update to newest

[DictionaryPC.git] / src / com / hughes / android / dictionary / parser / WikiTokenizer.java

diff --git a/src/com/hughes/android/dictionary/parser/WikiTokenizer.java b/src/com/hughes/android/dictionary/parser/WikiTokenizer.java
index 5ac7d4598b990efcd31cc3d52b63da601675889f..493abf2c52d27e9f67fc8283e49e5884b7f7be83 100644 (file)
--- a/src/com/hughes/android/dictionary/parser/WikiTokenizer.java
+++ b/src/com/hughes/android/dictionary/parser/WikiTokenizer.java
@@ -82,7 +82,7 @@ public final class WikiTokenizer {
   }
 
   public WikiTokenizer(final String wikiText, final boolean isNewline) {
-    this.wikiText = wikiText;
+    this.wikiText = wikiText.replaceAll("\u2028", "\n");
     this.matcher = wikiTokenEvent.matcher(wikiText);
     justReturnedNewline = isNewline;
   }
@@ -150,6 +150,10 @@ public final class WikiTokenizer {
     }
   }
   
+  public List<String> errors() {
+    return errors;
+  }
+  
   public boolean isNewline() {
     return justReturnedNewline;
   }
@@ -419,7 +423,7 @@ public final class WikiTokenizer {
         
         assert matcher.end() > end || matchText.length() == 0: "Group=" + matcher.group();
         if (matchText.length() == 0) {
-          assert matchStart == wikiText.length() || wikiText.charAt(matchStart) == '\n';
+          assert matchStart == wikiText.length() || wikiText.charAt(matchStart) == '\n' : wikiText + ", " + matchStart;
           if (firstNewline == -1) {
             firstNewline = matcher.end();
           }