src/com/hughes/android/dictionary/parser/WikiTokenizerTest.java

   1 package com.hughes.android.dictionary.parser;
   2
   3 import java.util.ArrayList;
   4 import java.util.Arrays;
   5 import java.util.List;
   6
   7 import junit.framework.TestCase;
   8
   9 public class WikiTokenizerTest extends TestCase {
  10
  11   public void testSimple() {
  12     final String wikiText =
  13       "Hi" + "\n" +
  14       "Hello thad you're <!-- not --> '''pretty''' cool '''''over''''' there." + "\n" +
  15       "hi <!--" + "\n" +
  16       "multi-line" + "\n" +
  17       "# comment -->" + "\n" +
  18       "" + "\n" +
  19       "asdf\n" +
  20       "{{template_not_in_list}}" + "\n" +
  21       "# {{template_in_list}}" + "\n" +
  22       "[[wikitext]]:[[wikitext]]" + "\n" +  // don't want this to trigger a list
  23       ": but this is a list!" + "\n" +
  24       "*:* and so is this :::" + "\n" +
  25       "here's [[some blah|some]] wikitext." + "\n" +
  26       "here's a {{template|this has an = sign|blah=2|blah2=3|" + "\n" +
  27       "blah3=3,[[asdf]|[asdf asdf]|[asdf asdf asdf]],blah4=4}} and some more text." + "\n" +
  28       "== Header 2 ==" + "\n" +
  29       "{{some-func|blah={{nested-func|n2}}|blah2=asdf}}" + "\n" +
  30       "{{mismatched]]" + "\n" +
  31       "[[mismatched}}" + "\n" +
  32       "{extraterminated}}" + "\n" +
  33       "[extraterminated]]" + "\n" +
  34       "=== {{header-template}} ===" + "\n";
  35
  36     final String[] expectedTokens = new String[] {
  37         "Hi",
  38         "\n",
  39         "Hello thad you're ",
  40         "<!-- not -->",
  41         " ",
  42         "'''",
  43         "pretty",
  44         "'''",
  45         " cool ",
  46         "'''",
  47         "''",
  48         "over",
  49         "'''",
  50         "''",
  51         " there.",
  52         "\n",
  53         "hi ",
  54         "<!--\nmulti-line\n# comment -->",
  55         "\n",
  56         "\n",
  57         "asdf",
  58         "\n",
  59         "{{template_not_in_list}}",
  60         "\n",
  61         "# {{template_in_list}}",
  62         "\n",
  63         "[[wikitext]]",
  64         ":",
  65         "[[wikitext]]",
  66         "\n",
  67         ": but this is a list!",
  68         "\n",
  69         "*:* and so is this :::",
  70         "\n",
  71         "here's ",
  72         "[[some blah|some]]",
  73         " wikitext.",
  74         "\n",
  75         "here's a ",
  76         "{{template|this has an = sign|blah=2|blah2=3|\nblah3=3,[[asdf]|[asdf asdf]|[asdf asdf asdf]],blah4=4}}",
  77         " and some more text.",
  78         "\n",
  79         "== Header 2 ==",
  80         "\n",
  81         "{{some-func|blah={{nested-func|n2}}|blah2=asdf}}",
  82         "\n",
  83         "{{mismatched]]\n",
  84         "[[mismatched}}\n",
  85         "{extraterminated",
  86         "}}",
  87         "\n",
  88         "[extraterminated",
  89         "]]",
  90         "\n",
  91         "=== {{header-template}} ===",
  92         "\n",
  93         };
  94
  95     final List<String> actualTokens = new ArrayList<String>();
  96
  97     final WikiTokenizer wikiTokenizer = new WikiTokenizer(wikiText);
  98     WikiTokenizer token;
  99     int i = 0;
 100     while ((token = wikiTokenizer.nextToken()) != null) {
 101       actualTokens.add(token.token());
 102       System.out.println("\"" + token.token().replace("\n", "\\n") + "\",");
 103       assertEquals(expectedTokens[i++], token.token());
 104     }
 105     assertEquals(Arrays.asList(expectedTokens), actualTokens);
 106   }
 107
 108   public void testWikiHeading() {
 109     assertNull(WikiHeading.getHeading(""));
 110     assertNull(WikiHeading.getHeading("="));
 111     assertNull(WikiHeading.getHeading("=="));
 112     assertNull(WikiHeading.getHeading("=a"));
 113     assertNull(WikiHeading.getHeading("=a=="));
 114     assertNull(WikiHeading.getHeading("===a=="));
 115     assertNull(WikiHeading.getHeading("===a===="));
 116     assertNull(WikiHeading.getHeading("a="));
 117     assertEquals("a", WikiHeading.getHeading("=a=").name);
 118     assertEquals(1, WikiHeading.getHeading("=a=").depth);
 119     assertEquals("aa", WikiHeading.getHeading("==aa==").name);
 120     assertEquals(2, WikiHeading.getHeading("==aa==").depth);
 121   }
 122
 123
 124   public void testWikiFunction() {
 125     assertNull(WikiFunction.getFunction(""));
 126     assertNull(WikiFunction.getFunction("[[asdf]]"));
 127     assertNull(WikiFunction.getFunction("asd [[asdf]]asdf "));
 128     assertEquals("a", WikiFunction.getFunction("{{a}}").name);
 129     assertEquals("a", WikiFunction.getFunction("{{a|b}}").name);
 130     assertEquals("a", WikiFunction.getFunction("a{{a|b}}a").name);
 131     assertEquals("a[[a]]", WikiFunction.getFunction("a{{a[[a]]|b}}a").name);
 132     assertEquals("a", WikiFunction.getFunction("a{{a|b[[abc|def]]|[[fgh|jkl]]|qwer}}a").name);
 133     assertEquals(Arrays.asList("b[[abc|d=f]]", "qwer", "[[fgh|jkl]]", "qwer"), WikiFunction.getFunction("a{{a|b[[abc|d=f]]|qwer|[[fgh|jkl]]|qwer}}a").args);
 134     assertEquals("[[abc|def]]", WikiFunction.getFunction("a{{a|b=[[abc|def]]|qwer|[[fgh|jkl]]|qwer={{asdf}}}}a").namedArgs.get("b"));
 135     assertEquals("{{asdf}}", WikiFunction.getFunction("a{{a|b=[[abc|def]]|qwer|[[fgh|jkl]]|qwer={{asdf}}}}a").namedArgs.get("qwer"));
 136   }
 137
 138 }