X-Git-Url: http://gitweb.fperrin.net/?a=blobdiff_plain;f=src%2Fcom%2Fhughes%2Fandroid%2Fdictionary%2Fparser%2FWikiTokenizer.java;h=c8b150e815d6e33e4806bb296a09dd21d7f8dc43;hb=8f3cf0a0d01b1ffeab4fa2c4ef1f69203be9a891;hp=6c81749006ea96d5dfe362459206d9ded25cf7e0;hpb=52123581b0c4aa46298b9d6cbc4697accffc1cc7;p=DictionaryPC.git

diff --git a/src/com/hughes/android/dictionary/parser/WikiTokenizer.java b/src/com/hughes/android/dictionary/parser/WikiTokenizer.java
index 6c81749..c8b150e 100644
--- a/src/com/hughes/android/dictionary/parser/WikiTokenizer.java
+++ b/src/com/hughes/android/dictionary/parser/WikiTokenizer.java
@@ -24,15 +24,56 @@ import java.util.regex.Pattern;
 public final class WikiTokenizer {
   
   public static interface Callback {
-    void onPlainText(WikiTokenizer wikiTokenizer);
+    void onPlainText(final String text);
     void onMarkup(WikiTokenizer wikiTokenizer);
     void onWikiLink(WikiTokenizer wikiTokenizer);
     void onNewline(WikiTokenizer wikiTokenizer);
-    void onFunction(String functionName, List<String> functionPositionArgs,
+    void onFunction(final WikiTokenizer tokenizer, String functionName, List<String> functionPositionArgs,
         Map<String, String> functionNamedArgs);
     void onHeading(WikiTokenizer wikiTokenizer);
     void onListItem(WikiTokenizer wikiTokenizer);
     void onComment(WikiTokenizer wikiTokenizer);
+    void onHtml(WikiTokenizer wikiTokenizer);
+  }
+  
+  public static class DoNothingCallback implements Callback {
+
+    @Override
+    public void onPlainText(String text) {
+    }
+
+    @Override
+    public void onMarkup(WikiTokenizer wikiTokenizer) {
+    }
+
+    @Override
+    public void onWikiLink(WikiTokenizer wikiTokenizer) {
+    }
+
+    @Override
+    public void onNewline(WikiTokenizer wikiTokenizer) {
+    }
+
+    @Override
+    public void onFunction(WikiTokenizer tokenizer, String functionName,
+        List<String> functionPositionArgs, Map<String, String> functionNamedArgs) {
+    }
+
+    @Override
+    public void onHeading(WikiTokenizer wikiTokenizer) {
+    }
+
+    @Override
+    public void onListItem(WikiTokenizer wikiTokenizer) {
+    }
+
+    @Override
+    public void onComment(WikiTokenizer wikiTokenizer) {
+    }
+
+    @Override
+    public void onHtml(WikiTokenizer wikiTokenizer) {
+    }
   }
   
   //private static final Pattern wikiTokenEvent = Pattern.compile("($)", Pattern.MULTILINE);
@@ -43,6 +84,9 @@ public final class WikiTokenizer {
         "=|" +  // Need the = because we might have to find unescaped =
   		"<!--|" +
   		"''|" +
+        "<pre>|" +
+        "<math>|" +
+        "<ref>|" +
   		"$)", Pattern.MULTILINE);
   private static final String listChars = "*#:;";
   
@@ -67,6 +111,7 @@ public final class WikiTokenizer {
   private boolean isComment;
   private boolean isFunction;
   private boolean isWikiLink;
+  private boolean isHtml;
   private int firstUnescapedPipePos;
   
   private int lastUnescapedPipePos;
@@ -79,7 +124,9 @@ public final class WikiTokenizer {
     this(wikiText, true);
   }
 
-  public WikiTokenizer(final String wikiText, final boolean isNewline) {
+  public WikiTokenizer(String wikiText, final boolean isNewline) {
+    wikiText = wikiText.replaceAll("\u2028", "\n");
+    wikiText = wikiText.replaceAll("\u0085", "\n");
     this.wikiText = wikiText;
     this.matcher = wikiTokenEvent.matcher(wikiText);
     justReturnedNewline = isNewline;
@@ -97,6 +144,7 @@ public final class WikiTokenizer {
     isComment = false;
     isFunction = false;
     isWikiLink = false;
+    isHtml = false;
     
     firstUnescapedPipePos = -1;
     lastUnescapedPipePos = -1;
@@ -104,31 +152,56 @@ public final class WikiTokenizer {
     positionArgs.clear();
     namedArgs.clear();
   }
-  
-  public void dispatch(final Callback callback) {
-    while (nextToken() != null) {
-      if (isPlainText()) {
-        callback.onPlainText(this);
-      } else if (isMarkup()) {
-        callback.onMarkup(this);
-      } else if (isWikiLink) {
-        callback.onWikiLink(this);
-      } else if (isNewline()) {
-        callback.onNewline(this);
-      } else if (isFunction()) {
-        callback.onFunction(functionName(), functionPositionArgs(), functionNamedArgs());
-      } else if (isHeading()) {
-        callback.onHeading(this);
-      } else if (isListItem()) {
-        callback.onListItem(this);
-      } else if (isComment()) {
-        callback.onComment(this);
-      } else {
-        throw new IllegalStateException("Unknown wiki state.");
+
+  private static final Pattern POSSIBLE_WIKI_TEXT = Pattern.compile(
+      "\\{\\{|" +
+      "\\[\\[|" +
+      "<!--|" +
+      "''|" +
+      "<pre>|" +
+      "<math>|" +
+      "<ref>|" +
+      "[\n]"
+      );
+
+  public static void dispatch(final String wikiText, final boolean isNewline, final Callback callback) {
+    // Optimization...
+    if (!POSSIBLE_WIKI_TEXT.matcher(wikiText).find()) {
+      callback.onPlainText(wikiText);
+    } else {
+      final WikiTokenizer tokenizer = new WikiTokenizer(wikiText, isNewline);
+      while (tokenizer.nextToken() != null) {
+        if (tokenizer.isPlainText()) {
+          callback.onPlainText(tokenizer.token());
+        } else if (tokenizer.isMarkup()) {
+          callback.onMarkup(tokenizer);
+        } else if (tokenizer.isWikiLink) {
+          callback.onWikiLink(tokenizer);
+        } else if (tokenizer.isNewline()) {
+          callback.onNewline(tokenizer);
+        } else if (tokenizer.isFunction()) {
+          callback.onFunction(tokenizer, tokenizer.functionName(), tokenizer.functionPositionArgs(), tokenizer.functionNamedArgs());
+        } else if (tokenizer.isHeading()) {
+          callback.onHeading(tokenizer);
+        } else if (tokenizer.isListItem()) {
+          callback.onListItem(tokenizer);
+        } else if (tokenizer.isComment()) {
+          callback.onComment(tokenizer);
+        } else if (tokenizer.isHtml()) {
+          callback.onHtml(tokenizer);
+        } else if (!tokenizer.errors.isEmpty()) {
+          // Log was already printed....
+        } else {
+          throw new IllegalStateException("Unknown wiki state: " + tokenizer.token());
+        }
       }
     }
   }
   
+  public List<String> errors() {
+    return errors;
+  }
+  
   public boolean isNewline() {
     return justReturnedNewline;
   }
@@ -168,6 +241,13 @@ public final class WikiTokenizer {
     assert isListItem();
     return wikiText.substring(start, listPrefixEnd);
   }
+  
+  public static String getListTag(char c) {
+    if (c == '#') {
+      return "ol";
+    }
+    return "ul";
+  }
 
   public String listItemWikiText() {
     assert isListItem();
@@ -182,9 +262,9 @@ public final class WikiTokenizer {
     assert isFunction();
     // "{{.."
     if (firstUnescapedPipePos != -1) {
-      return wikiText.substring(start + 2, firstUnescapedPipePos);
+      return trimNewlines(wikiText.substring(start + 2, firstUnescapedPipePos).trim());
     }
-    return wikiText.substring(start + 2, end - 2);
+    return trimNewlines(wikiText.substring(start + 2, end - 2).trim());
   }
   
   public List<String> functionPositionArgs() {
@@ -207,21 +287,25 @@ public final class WikiTokenizer {
     assert isWikiLink();
     // "[[.."
     if (lastUnescapedPipePos != -1) {
-      return wikiText.substring(lastUnescapedPipePos + 1, end - 2);
+      return trimNewlines(wikiText.substring(lastUnescapedPipePos + 1, end - 2));
     }
     assert start + 2 < wikiText.length() && end >= 2: wikiText;
-    return wikiText.substring(start + 2, end - 2);
+    return trimNewlines(wikiText.substring(start + 2, end - 2));
   }
 
   public String wikiLinkDest() {
     assert isWikiLink();
     // "[[.."
     if (firstUnescapedPipePos != -1) {
-      return wikiText.substring(start + 2, firstUnescapedPipePos);
+      return trimNewlines(wikiText.substring(start + 2, firstUnescapedPipePos));
     }
     return null;
   }
   
+  public boolean isHtml() {
+    return isHtml;
+  }
+
   public boolean remainderStartsWith(final String prefix) {
     return wikiText.startsWith(prefix, start);
   }
@@ -260,7 +344,7 @@ public final class WikiTokenizer {
       return this;
     }
     
-    if (justReturnedNewline) {    
+    if (justReturnedNewline) {   
       justReturnedNewline = false;
 
       final char firstChar = wikiText.charAt(end);
@@ -324,11 +408,19 @@ public final class WikiTokenizer {
 
     if (wikiText.startsWith("<pre>", start)) {
       end = safeIndexOf(wikiText, start, "</pre>", "\n");
+      isHtml = true;
       return this;
     }
 
+    if (wikiText.startsWith("<ref>", start)) {
+        end = safeIndexOf(wikiText, start, "</ref>", "\n");
+        isHtml = true;
+        return this;
+      }
+
     if (wikiText.startsWith("<math>", start)) {
       end = safeIndexOf(wikiText, start, "</math>", "\n");
+      isHtml = true;
       return this;
     }
 
@@ -392,7 +484,7 @@ public final class WikiTokenizer {
         
         assert matcher.end() > end || matchText.length() == 0: "Group=" + matcher.group();
         if (matchText.length() == 0) {
-          assert matchStart == wikiText.length() || wikiText.charAt(matchStart) == '\n';
+          assert matchStart == wikiText.length() || wikiText.charAt(matchStart) == '\n' : wikiText + ", " + matchStart;
           if (firstNewline == -1) {
             firstNewline = matcher.end();
           }
@@ -439,7 +531,7 @@ public final class WikiTokenizer {
             errors.add("Unmatched <!-- error: " + wikiText.substring(start));
             return safeIndexOf(wikiText, start, "\n", "\n");
           }
-        } else if (matchText.equals("''")) {
+        } else if (matchText.equals("''") || (matchText.startsWith("<") && matchText.endsWith(">"))) {
           // Don't care.
         } else {
           assert false : "Match text='" + matchText + "'";
@@ -472,14 +564,24 @@ public final class WikiTokenizer {
       if (lastUnescapedEqualsPos > lastUnescapedPipePos) {
         final String key = wikiText.substring(lastUnescapedPipePos + 1, lastUnescapedEqualsPos);
         final String value = wikiText.substring(lastUnescapedEqualsPos + 1, matchStart);
-        namedArgs.put(key, value);
+        namedArgs.put(trimNewlines(key), trimNewlines(value));
       } else {
         final String value = wikiText.substring(lastUnescapedPipePos + 1, matchStart);
-        positionArgs.add(value);
+        positionArgs.add(trimNewlines(value));
       }
     }
     lastUnescapedPipePos = matchStart;
   }
+  
+  static final String trimNewlines(String s) {
+    while (s.startsWith("\n")) {
+      s = s.substring(1);
+    }
+    while (s.endsWith("\n")) {
+      s = s.substring(0, s.length() - 1);
+    }
+    return s.replaceAll("\n", " ");
+  }
 
   static int safeIndexOf(final String s, final int start, final String target, final String backup) {
     int close = s.indexOf(target, start);