From e55f0ff1e195e1238e91909d0366eac391395ab8 Mon Sep 17 00:00:00 2001
From: Thad Hughes <thad.hughes@gmail.com>
Date: Mon, 7 Mar 2011 18:17:54 -0800
Subject: [PATCH] go

---
 .../engine/DictionaryBuilderMain.java         |  4 +-
 .../parser/EnWiktionaryXmlParser.java         | 12 ++---
 .../android/dictionary/parser/WikiParser.java | 54 +++++++++----------
 .../dictionary/parser/WikiParserTest.java     |  8 ++-
 4 files changed, 41 insertions(+), 37 deletions(-)

diff --git a/src/com/hughes/android/dictionary/engine/DictionaryBuilderMain.java b/src/com/hughes/android/dictionary/engine/DictionaryBuilderMain.java
index 7438dc2..c49305e 100644
--- a/src/com/hughes/android/dictionary/engine/DictionaryBuilderMain.java
+++ b/src/com/hughes/android/dictionary/engine/DictionaryBuilderMain.java
@@ -22,7 +22,6 @@ public class DictionaryBuilderMain extends TestCase {
   
   
   public static void main(final String[] args) throws Exception {
-
     DictionaryBuilder.main(new String[] {
         "--dictOut=dictOutputs/DE-EN_chemnitz.quickdic",
         "--lang1=DE",
@@ -35,6 +34,7 @@ public class DictionaryBuilderMain extends TestCase {
         "--input1Format=chemnitz",
     });
 
+
     Lang[] langs1 = new Lang[] { 
         new Lang("^English$", "EN"),
         new Lang("^German$", "DE"),
@@ -108,7 +108,7 @@ public class DictionaryBuilderMain extends TestCase {
             String.format("--lang2=%s", lang2.code),
             String.format("--dictInfo=(EN)Wikitionary-based %s-%s dictionary", lang1.code, lang2.code),
 
-            "--input1=dictInputs/enwiktionary-20101015-pages-articles",
+            "--input1=dictInputs/enwiktionary-20110205-pages-articles.xml",
             "--input1Name=enwiktionary",
             "--input1Format=enwiktionary",
             String.format("--input1TranslationPattern1=%s", lang1.nameRegex),
diff --git a/src/com/hughes/android/dictionary/parser/EnWiktionaryXmlParser.java b/src/com/hughes/android/dictionary/parser/EnWiktionaryXmlParser.java
index 761c5dc..5277ce4 100644
--- a/src/com/hughes/android/dictionary/parser/EnWiktionaryXmlParser.java
+++ b/src/com/hughes/android/dictionary/parser/EnWiktionaryXmlParser.java
@@ -186,9 +186,9 @@ public class EnWiktionaryXmlParser extends org.xml.sax.helpers.DefaultHandler im
   static final Set<String> useRemainingArgTemplates = new LinkedHashSet<String>(Arrays.asList(
       "Arab", "Cyrl", "fa-Arab", "italbrac", "Khmr", "ku-Arab", "IPAchar", "Laoo", 
       "sd-Arab", "Thai", "ttbc", "unicode", "ur-Arab", "yue-yue-j", "zh-ts", 
-      "zh-tsp", "zh-zh-p"));
-  static final Set<String> ignoreTemplates = new LinkedHashSet<String>(Arrays.asList(""));
-  static final Set<String> grammarTemplates = new LinkedHashSet<String>(Arrays.asList("impf", "pf"));
+      "zh-tsp", "zh-zh-p", "ug-Arab", "ko-inline", "Jpan", "Kore", "rfscript", "Latinx"));
+  static final Set<String> ignoreTemplates = new LinkedHashSet<String>(Arrays.asList("audio", "rhymes", "hyphenation", "homophones", "wikipedia", "rel-top", "rel-bottom", "sense", "wikisource1911Enc", "g"));
+  static final Set<String> grammarTemplates = new LinkedHashSet<String>(Arrays.asList("impf", "pf", "pf.", "indeclinable"));
   static final Set<String> passThroughTemplates = new LinkedHashSet<String>(Arrays.asList("zzzzzzzzzzzzzzz"));
 
   @Override
@@ -267,7 +267,7 @@ public class EnWiktionaryXmlParser extends org.xml.sax.helpers.DefaultHandler im
         return;
       }
       
-      if (name.equals("audio") || name.equals("rhymes") || name.equals("hyphenation")) {
+      if (ignoreTemplates.contains(name)) {
         return;
       }
       
@@ -358,7 +358,7 @@ public class EnWiktionaryXmlParser extends org.xml.sax.helpers.DefaultHandler im
       wikiBuilder.append("sg.");
       
     } else  if (grammarTemplates.contains(name)) {
-      assert positionalArgs.size() == 1 && namedArgs.isEmpty();
+      assert positionalArgs.size() == 1 && namedArgs.isEmpty() : positionalArgs.toString() + namedArgs;
       wikiBuilder.append(name).append(".");
 
     } else  if (name.equals("l")) {
@@ -627,7 +627,7 @@ public class EnWiktionaryXmlParser extends org.xml.sax.helpers.DefaultHandler im
 
   @Override
   public void onUnterminated(String start, String rest) {
-    throw new RuntimeException(start + rest);
+    System.err.printf("OnUnterminated: %s %s %s\n", title, start, rest);
   }
   @Override
   public void onInvalidHeaderEnd(String rest) {
diff --git a/src/com/hughes/android/dictionary/parser/WikiParser.java b/src/com/hughes/android/dictionary/parser/WikiParser.java
index 5b73b87..37c7a53 100644
--- a/src/com/hughes/android/dictionary/parser/WikiParser.java
+++ b/src/com/hughes/android/dictionary/parser/WikiParser.java
@@ -7,6 +7,8 @@ import java.util.Map;
 import java.util.regex.Matcher;
 import java.util.regex.Pattern;
 
+import com.hughes.util.StringUtil;
+
 public class WikiParser {
   
   private static final Pattern markup = Pattern.compile("$|''|\\{\\{|\\[\\[|(==+)\\s*$|<!--|<pre>", Pattern.MULTILINE);
@@ -84,13 +86,12 @@ public class WikiParser {
           callback.onFormatItalic(italicOn);
           rest = rest.substring(2);
         } else if (rest.startsWith("{{")) {
-          int end = rest.indexOf("}}");
+          int end = StringUtil.nestedIndexOf(rest, 2, "{{", "}}");
           if (end == -1) {
             callback.onUnterminated("{{", rest);
-            return;
+            end = StringUtil.safeIndexOf(rest, "\n") - 2;
           }
           final String template = rest.substring(2, end).trim();
-          //todo: this doesn't work.  can't split pipes inside [[asdf|asdf]]
           final List<String> templateArray = new ArrayList<String>();
           contextSensitivePipeSplit(template, templateArray);
           positionalArgs.clear();
@@ -114,7 +115,7 @@ public class WikiParser {
           int end = rest.indexOf("]]");
           if (end == -1) {
             callback.onUnterminated("[[", rest);
-            return;
+            end = StringUtil.safeIndexOf(rest, "\n") - 2;
           }
           final String wikiLink = rest.substring(2, end);
           final String[] args = pipeSplit.split(wikiLink);
@@ -136,7 +137,7 @@ public class WikiParser {
           int end = rest.indexOf("-->");
           if (end == -1) {
             callback.onUnterminated("<!--", rest);
-            return;
+            end = StringUtil.safeIndexOf(rest, "\n") - 3;
           }
           callback.onComment(rest.substring(4, end));
           rest = rest.substring(end + 3);
@@ -144,7 +145,7 @@ public class WikiParser {
           int end = rest.indexOf("</pre>");
           if (end == -1) {
             callback.onUnterminated("<pre>", rest);
-            return;
+            end = StringUtil.safeIndexOf(rest, "\n") - 6;
           }
           callback.onText(rest.substring(5, end));
           rest = rest.substring(end + 6);
@@ -155,32 +156,29 @@ public class WikiParser {
     }
   }
   
-  private static final Pattern openBracketOrPipe = Pattern.compile("($)|(\\[\\[)|(\\s*\\|\\s*)");
   private static void contextSensitivePipeSplit(String template, final List<String> result) {
-    StringBuilder builder = new StringBuilder();
-    while (template.length() > 0) {
-      final Matcher matcher = openBracketOrPipe.matcher(template);
-      if (matcher.find()) {
-        // append to the match.
-        builder.append(template.substring(0, matcher.start()));
-        if (matcher.group(2) != null) {  // [[
-          // append to the close ]].
-          final int closeIndex = template.indexOf("]]", matcher.end());
-          builder.append(template.substring(matcher.start(), closeIndex + 2));
-          template = template.substring(closeIndex + 2);
-        } else if (matcher.group(3) != null) { // |
-          result.add(builder.toString());
-          builder = new StringBuilder();
-          template = template.substring(matcher.end());
-        } else {
-          template = template.substring(matcher.start());
-          assert template.length() == 0 : template;
+    int depth = 0;
+    int lastStart = 0;
+    for (int i = 1; i < template.length(); ) {
+      if (template.charAt(i) == '|' && depth == 0) {
+        final String s = template.substring(lastStart, i);
+        result.add(s.trim());
+        ++i;
+        lastStart = i;
+      } else if (template.startsWith("[[", i) || template.startsWith("{{", i)) {
+        ++depth;
+        i += 2;
+      } else if (template.startsWith("]]", i) || template.startsWith("}}", i)) {
+        --depth;
+        if (depth < 0) {
+          throw new RuntimeException("too many closings: " + template);
         }
+        i += 2;
       } else {
-        assert false;
+        ++i;
       }
     }
-    result.add(builder.toString());
+    result.add(template.substring(lastStart).trim());
   }
 
   // ------------------------------------------------------------------------
@@ -249,7 +247,7 @@ public class WikiParser {
 
     @Override
     public void onUnterminated(String start, String rest) {
-      throw new RuntimeException(start + rest);
+      System.err.printf("onUnterminated: %s, %s\n", start, rest);
     }
 
     @Override
diff --git a/src/com/hughes/android/dictionary/parser/WikiParserTest.java b/src/com/hughes/android/dictionary/parser/WikiParserTest.java
index 813b073..a94ad43 100644
--- a/src/com/hughes/android/dictionary/parser/WikiParserTest.java
+++ b/src/com/hughes/android/dictionary/parser/WikiParserTest.java
@@ -26,6 +26,8 @@ public class WikiParserTest extends TestCase {
       "here's a {{template|this has an = sign|blah=2|blah2=3|" + "\n" +
       "blah3=3,[[asdf]|[asdf asdf]|[asdf asdf asdf]],blah4=4}} and some more text." + "\n" +
       "== Header 2 ==" + "\n" +
+      "{{some-func|blah={{nested-func|n2}}|blah2=asdf}}" + "\n" +
+      "{{unterminated}" + "\n" +
 //      "==== Header 4 ====" + "\n" +
 //      "===== Header 5 =====" + "\n" +
       "=== {{header-template}} ===" + "\n";
@@ -50,6 +52,10 @@ public class WikiParserTest extends TestCase {
         "template:[template, this has an = sign]{blah=2, blah2=3, blah3=3,[[asdf]|[asdf asdf]|[asdf asdf asdf]],blah4=4}\n" +
         " and some more text.\n" +
         "HEADER   Header 2 \n" +
+        " \n" +
+        "template:[some-func]{blah={{nested-func|n2}}, blah2=asdf}\n" +
+        " \n" +
+        "template:[unterminate]{}" + "\n" +
         "\n" +
         "HEADER    \n" +
         "template:[header-template]{}\n" +
@@ -130,7 +136,7 @@ public class WikiParserTest extends TestCase {
 
     @Override
     public void onUnterminated(String start, String rest) {
-      throw new RuntimeException("bad");
+      //throw new RuntimeException("bad");
     }
 
     @Override
-- 
2.43.0