X-Git-Url: http://gitweb.fperrin.net/?a=blobdiff_plain;f=src%2Fcom%2Fhughes%2Fandroid%2Fdictionary%2Fparser%2FWikiTokenizer.java;h=56bda7fcb62d731e538800204392110d0b6c68de;hb=2fc669d88306d563fc9c899d8d91b25d591692ea;hp=9f3444f8f06d5ee27cbb8fa5d1fd7741add3e653;hpb=054097689b08815b3587cee6955c143b1cc285e4;p=DictionaryPC.git

diff --git a/src/com/hughes/android/dictionary/parser/WikiTokenizer.java b/src/com/hughes/android/dictionary/parser/WikiTokenizer.java
index 9f3444f..56bda7f 100644
--- a/src/com/hughes/android/dictionary/parser/WikiTokenizer.java
+++ b/src/com/hughes/android/dictionary/parser/WikiTokenizer.java
@@ -14,16 +14,13 @@
 
 package com.hughes.android.dictionary.parser;
 
-import java.util.ArrayList;
-import java.util.LinkedHashMap;
-import java.util.List;
-import java.util.Map;
+import java.util.*;
 import java.util.regex.Matcher;
 import java.util.regex.Pattern;
 
 public final class WikiTokenizer {
 
-    public static interface Callback {
+    public interface Callback {
         void onPlainText(final String text);
         void onMarkup(WikiTokenizer wikiTokenizer);
         void onWikiLink(WikiTokenizer wikiTokenizer);
@@ -99,8 +96,8 @@ public final class WikiTokenizer {
     int end = 0;
     int start = -1;
 
-    final List<String> errors = new ArrayList<String>();
-    final List<String> tokenStack = new ArrayList<String>();
+    final List<String> errors = new ArrayList<>();
+    final List<String> tokenStack = new ArrayList<>();
 
 
     private String headingWikiText;
@@ -116,8 +113,8 @@ public final class WikiTokenizer {
 
     private int lastUnescapedPipePos;
     private int lastUnescapedEqualsPos;
-    private final List<String> positionArgs = new ArrayList<String>();
-    private final Map<String,String> namedArgs = new LinkedHashMap<String,String>();
+    private final List<String> positionArgs = new ArrayList<>();
+    private final Map<String,String> namedArgs = new LinkedHashMap<>();
 
 
     public WikiTokenizer(final String wikiText) {
@@ -338,7 +335,7 @@ public final class WikiTokenizer {
             }
 
             // Eat a newline if we're looking at one:
-            final boolean atNewline = wikiText.charAt(end) == '\n' || wikiText.charAt(end) == '\u2028';
+            final boolean atNewline = wikiText.charAt(end) == '\n' || wikiText.charAt(end) == '\u2028' || wikiText.charAt(end) == '\u2029';
             if (atNewline) {
                 justReturnedNewline = true;
                 ++end;
@@ -448,8 +445,11 @@ public final class WikiTokenizer {
                 end = this.matcher.start(1);
                 isPlainText = true;
                 if (end == start) {
-                    errors.add("Empty group: " + this.matcher.group());
+                    // stumbled over a new type of newline?
+                    // Or matcher is out of sync with checks above
+                    errors.add("Empty group: " + this.matcher.group() + " char: " + (int)wikiText.charAt(end));
                     assert false;
+                    throw new RuntimeException("matcher not in sync with code, or new type of newline, errors :" + errors);
                 }
                 return this;
             }
@@ -471,7 +471,7 @@ public final class WikiTokenizer {
         return token;
     }
 
-    final static String[] patterns = { "\n", "{{", "}}", "[[", "]]", "|", "=", "<!--" };
+    static final String[] patterns = { "\n", "{{", "}}", "[[", "]]", "[", "]", "|", "=", "<!--" };
     private int escapedFindEnd(final int start, final String toFind) {
         assert tokenStack.isEmpty();
 
@@ -479,17 +479,19 @@ public final class WikiTokenizer {
 
         int end = start;
         int firstNewline = -1;
-        int[] nextMatch = new int[8];
-        for (int i = 0; i < 8; ++i) {
-            nextMatch[i] = wikiText.indexOf(patterns[i], start);
-            if (nextMatch[i] == -1) nextMatch[i] = i > 0 ? 0x7fffffff : wikiText.length();
-        }
+        int[] nextMatch = new int[patterns.length];
+        Arrays.fill(nextMatch, -2);
+        int singleBrackets = 0;
         while (end < wikiText.length()) {
             // Manual replacement for matcher.find(end),
             // because Java regexp is a ridiculously slow implementation.
             // Initialize to always match the end.
             int matchIdx = 0;
-            for (int i = 1; i < 8; ++i) {
+            for (int i = 0; i < nextMatch.length; ++i) {
+                if (nextMatch[i] <= end) {
+                    nextMatch[i] = wikiText.indexOf(patterns[i], end);
+                    if (nextMatch[i] == -1) nextMatch[i] = i > 0 ? 0x7fffffff : wikiText.length();
+                }
                 if (nextMatch[i] < nextMatch[matchIdx]) {
                     matchIdx = i;
                 }
@@ -498,8 +500,6 @@ public final class WikiTokenizer {
             int matchStart = nextMatch[matchIdx];
             String matchText = patterns[matchIdx];
             int matchEnd = matchStart + matchText.length();
-            nextMatch[matchIdx] = wikiText.indexOf(patterns[matchIdx], matchEnd);
-            if (nextMatch[matchIdx] == -1) nextMatch[matchIdx] = matchIdx > 0 ? 0x7fffffff : wikiText.length();
             if (matchIdx == 0) {
                 matchText = "";
                 matchEnd = matchStart;
@@ -521,14 +521,23 @@ public final class WikiTokenizer {
                     addFunctionArg(insideFunction, matchStart);
                 }
                 return matchEnd;
+            } else if (matchText.equals("[")) {
+                singleBrackets++;
+            } else if (matchText.equals("]")) {
+                if (singleBrackets > 0) singleBrackets--;
             } else if (matchText.equals("[[") || matchText.equals("{{")) {
                 tokenStack.add(matchText);
             } else if (matchText.equals("]]") || matchText.equals("}}")) {
                 if (tokenStack.size() > 0) {
                     final String removed = tokenStack.remove(tokenStack.size() - 1);
                     if (removed.equals("{{") && !matchText.equals("}}")) {
-                        errors.add("Unmatched {{ error: " + wikiText.substring(start, matchEnd));
-                        return safeIndexOf(wikiText, start, "\n", "\n");
+                        if (singleBrackets >= 2) { // assume this is really two closing single ]
+                            singleBrackets -= 2;
+                            tokenStack.add(removed);
+                        } else {
+                            errors.add("Unmatched {{ error: " + wikiText.substring(start, matchEnd));
+                            return safeIndexOf(wikiText, start, "\n", "\n");
+                        }
                     } else if (removed.equals("[[") && !matchText.equals("]]")) {
                         errors.add("Unmatched [[ error: " + wikiText.substring(start, matchEnd));
                         return safeIndexOf(wikiText, start, "\n", "\n");
@@ -568,7 +577,7 @@ public final class WikiTokenizer {
             // We were looking for the end, we got it.
             return end;
         }
-        errors.add("Couldn't find: " + toFind + ", "+ wikiText.substring(start));
+        errors.add("Couldn't find: " + (toFind.equals("\n") ? "newline" : toFind) + ", "+ wikiText.substring(start));
         if (firstNewline != -1) {
             return firstNewline;
         }
@@ -591,7 +600,7 @@ public final class WikiTokenizer {
         lastUnescapedPipePos = matchStart;
     }
 
-    static final String trimNewlines(String s) {
+    static String trimNewlines(String s) {
         while (s.startsWith("\n")) {
             s = s.substring(1);
         }