Baseline HTML parsing done, goldens updated!

[DictionaryPC.git] / src / com / hughes / android / dictionary / parser / WikiTokenizer.java

diff --git a/src/com/hughes/android/dictionary/parser/WikiTokenizer.java b/src/com/hughes/android/dictionary/parser/WikiTokenizer.java
index 5ac7d4598b990efcd31cc3d52b63da601675889f..cdf2f0428fc9f2048b8d5f3efaf27124d7415e7d 100644 (file)
--- a/src/com/hughes/android/dictionary/parser/WikiTokenizer.java
+++ b/src/com/hughes/android/dictionary/parser/WikiTokenizer.java
@@ -36,6 +36,46 @@ public final class WikiTokenizer {
     void onHtml(WikiTokenizer wikiTokenizer);
   }
   
+  public static class DoNothingCallback implements Callback {
+
+    @Override
+    public void onPlainText(String text) {
+    }
+
+    @Override
+    public void onMarkup(WikiTokenizer wikiTokenizer) {
+    }
+
+    @Override
+    public void onWikiLink(WikiTokenizer wikiTokenizer) {
+    }
+
+    @Override
+    public void onNewline(WikiTokenizer wikiTokenizer) {
+    }
+
+    @Override
+    public void onFunction(WikiTokenizer tokenizer, String functionName,
+        List<String> functionPositionArgs, Map<String, String> functionNamedArgs) {
+    }
+
+    @Override
+    public void onHeading(WikiTokenizer wikiTokenizer) {
+    }
+
+    @Override
+    public void onListItem(WikiTokenizer wikiTokenizer) {
+    }
+
+    @Override
+    public void onComment(WikiTokenizer wikiTokenizer) {
+    }
+
+    @Override
+    public void onHtml(WikiTokenizer wikiTokenizer) {
+    }
+  }
+  
   //private static final Pattern wikiTokenEvent = Pattern.compile("($)", Pattern.MULTILINE);
   private static final Pattern wikiTokenEvent = Pattern.compile("(" +
                "\\{\\{|\\}\\}|" +
@@ -82,7 +122,7 @@ public final class WikiTokenizer {
   }
 
   public WikiTokenizer(final String wikiText, final boolean isNewline) {
-    this.wikiText = wikiText;
+    this.wikiText = wikiText.replaceAll("\u2028", "\n");
     this.matcher = wikiTokenEvent.matcher(wikiText);
     justReturnedNewline = isNewline;
   }
@@ -150,6 +190,10 @@ public final class WikiTokenizer {
     }
   }
   
+  public List<String> errors() {
+    return errors;
+  }
+  
   public boolean isNewline() {
     return justReturnedNewline;
   }
@@ -189,6 +233,13 @@ public final class WikiTokenizer {
     assert isListItem();
     return wikiText.substring(start, listPrefixEnd);
   }
+  
+  public static String getListTag(char c) {
+    if (c == '#') {
+      return "ol";
+    }
+    return "ul";
+  }
 
   public String listItemWikiText() {
     assert isListItem();
@@ -419,7 +470,7 @@ public final class WikiTokenizer {
         
         assert matcher.end() > end || matchText.length() == 0: "Group=" + matcher.group();
         if (matchText.length() == 0) {
-          assert matchStart == wikiText.length() || wikiText.charAt(matchStart) == '\n';
+          assert matchStart == wikiText.length() || wikiText.charAt(matchStart) == '\n' : wikiText + ", " + matchStart;
           if (firstNewline == -1) {
             firstNewline = matcher.end();
           }