X-Git-Url: http://gitweb.fperrin.net/?p=DictionaryPC.git;a=blobdiff_plain;f=src%2Fcom%2Fhughes%2Fandroid%2Fdictionary%2Fparser%2FWikiTokenizer.java;h=886e4f9074a20de21b36e6d85199dfa680aa7ca0;hp=0aaf9f6458da8ac16f4561fc5c87d8694eaa8aa2;hb=2f2eaf2360096aa926fba1b03916a5fe23fbb707;hpb=fbf8c3b696ef166133e2b362f5059ff7db0c8351

diff --git a/src/com/hughes/android/dictionary/parser/WikiTokenizer.java b/src/com/hughes/android/dictionary/parser/WikiTokenizer.java
index 0aaf9f6..886e4f9 100644
--- a/src/com/hughes/android/dictionary/parser/WikiTokenizer.java
+++ b/src/com/hughes/android/dictionary/parser/WikiTokenizer.java
@@ -463,8 +463,15 @@ public final class WikiTokenizer {
             }
 
 
-            if (this.matcher.find(start)) {
-                end = this.matcher.start();
+            while (end < wikiText.length()) {
+                int c = wikiText.charAt(end);
+                if (c == '\n' || c == '\'' || ((c - 0x1b) & 0xff9f) < 3) {
+                    matcher.region(end, wikiText.length());
+                    if (matcher.lookingAt()) break;
+                }
+                end++;
+            }
+            if (end != wikiText.length()) {
                 isPlainText = true;
                 if (end == start) {
                     // stumbled over a new type of newline?
@@ -477,7 +484,6 @@ public final class WikiTokenizer {
                 return this;
             }
 
-            end = wikiText.length();
             isPlainText = true;
             return this;