X-Git-Url: http://gitweb.fperrin.net/?a=blobdiff_plain;f=src%2Fcom%2Fhughes%2Fandroid%2Fdictionary%2Fparser%2FWikiTokenizer.java;h=f8d212f0be6677b4a55fb4bf82e912a0a7d06ee8;hb=b6cdf8f82c2e84dc0f72b0914439ad31d953b314;hp=a7863c7f42ee3dc24882019e4bbdbf88658e8409;hpb=e87d071962ee37719c9bea6740d93913ed4d8c7b;p=DictionaryPC.git

diff --git a/src/com/hughes/android/dictionary/parser/WikiTokenizer.java b/src/com/hughes/android/dictionary/parser/WikiTokenizer.java
index a7863c7..f8d212f 100644
--- a/src/com/hughes/android/dictionary/parser/WikiTokenizer.java
+++ b/src/com/hughes/android/dictionary/parser/WikiTokenizer.java
@@ -338,7 +338,7 @@ public final class WikiTokenizer {
             }
 
             // Eat a newline if we're looking at one:
-            final boolean atNewline = wikiText.charAt(end) == '\n' || wikiText.charAt(end) == '\u2028';
+            final boolean atNewline = wikiText.charAt(end) == '\n' || wikiText.charAt(end) == '\u2028' || wikiText.charAt(end) == '\u2029';
             if (atNewline) {
                 justReturnedNewline = true;
                 ++end;
@@ -448,8 +448,11 @@ public final class WikiTokenizer {
                 end = this.matcher.start(1);
                 isPlainText = true;
                 if (end == start) {
-                    errors.add("Empty group: " + this.matcher.group());
+                    // stumbled over a new type of newline?
+                    // Or matcher is out of sync with checks above
+                    errors.add("Empty group: " + this.matcher.group() + " char: " + (int)wikiText.charAt(end));
                     assert false;
+                    throw new RuntimeException("matcher not in sync with code, or new type of newline, errors :" + errors);
                 }
                 return this;
             }
@@ -579,7 +582,7 @@ public final class WikiTokenizer {
             // We were looking for the end, we got it.
             return end;
         }
-        errors.add("Couldn't find: " + toFind + ", "+ wikiText.substring(start));
+        errors.add("Couldn't find: " + (toFind.equals("\n") ? "newline" : toFind) + ", "+ wikiText.substring(start));
         if (firstNewline != -1) {
             return firstNewline;
         }