Bug-fixes to WikiTokenizer (handle weird line-feed), update to newest

[DictionaryPC.git] / src / com / hughes / android / dictionary / parser / WikiTokenizerTest.java

diff --git a/src/com/hughes/android/dictionary/parser/WikiTokenizerTest.java b/src/com/hughes/android/dictionary/parser/WikiTokenizerTest.java
index 4079f1b7bd0133b5f6ea0cc201e0cefbb7665589..8e1d5a87733300b4fd7c72001f63d6cea08d4d6f 100644 (file)
--- a/src/com/hughes/android/dictionary/parser/WikiTokenizerTest.java
+++ b/src/com/hughes/android/dictionary/parser/WikiTokenizerTest.java
@@ -55,6 +55,19 @@ public class WikiTokenizerTest extends TestCase {
 
     wikiText = "* This is ''bold''' asdf.";
     assertEquals(wikiText, new WikiTokenizer(wikiText).nextToken().token());
+
+    wikiText = "* {{a|US}} {{IPA|[ˈfɔɹ.wɝd]]}}\nasdf\n";
+    assertEquals("* {{a|US}} {{IPA|[ˈfɔɹ.wɝd]]}}", new WikiTokenizer(wikiText).nextToken().token());
+    assertTrue(new WikiTokenizer(wikiText).nextToken().isListItem());
+    assertEquals("\n", new WikiTokenizer(wikiText).nextToken().nextToken().token());
+
+    
+    wikiText = "* [[asdf|\u2028" +
+               "asdf]]";
+    assertEquals("* [[asdf|\n" +
+        "asdf]]", new WikiTokenizer(wikiText).nextToken().token());
+    assertTrue(new WikiTokenizer(wikiText).nextToken().isListItem());
+
   }
   
   public void testFunction() {
@@ -91,6 +104,19 @@ public class WikiTokenizerTest extends TestCase {
     assertEquals("202", new WikiTokenizer(wikiText).nextToken().functionNamedArgs().get("arg2"));
     assertEquals("{{n1|n2=7|n3}}", new WikiTokenizer(wikiText).nextToken().functionNamedArgs().get("arg3"));
 
+    wikiText = "{{gloss|asdf}\nAsdf\n\n";
+    assertEquals("{{gloss|asdf}", new WikiTokenizer(wikiText).nextToken().token());
+
+    wikiText = "#*{{quote-book|year=1960|author={{w|P. G. Wodehouse}}\n" +
+    "|title={{w|Jeeves in the Offing}}\n" +
+    "|section=chapter XI\n" +
+    "|passage=“I'm sorely beset, Jeeves. Do you recall telling me once about someone who told somebody he could tell him something which would make him think a bit? Knitted socks and porcu\n" +
+    "pines entered into it, I remember.” “I think you may be referring to the ghost of the father of Hamlet, Prince of Denmark, sir. Addressing his son, he said ‘I could a tale unfold whos\n" +
+    "e lightest word would harrow up thy soul, freeze thy young blood, make thy two eyes, like stars, start from their spheres, thy knotted and combined locks to part and each particular h\n" +
+    "air to stand on end like quills upon the fretful '''porpentine'''.’ ” “That's right. Locks, of course, not socks. Odd that he should have said '''porpentine''' when he meant porc\n" +
+    "upine. Slip of the tongue, no doubt, as so often happens with ghosts.”}}";
+    assertEquals(wikiText, new WikiTokenizer(wikiText).nextToken().token());
+
     
   }
   
@@ -142,7 +168,7 @@ public class WikiTokenizerTest extends TestCase {
     assertTrue(new WikiTokenizer(wikiText).nextToken().isHeading());
     assertEquals(1, new WikiTokenizer(wikiText).nextToken().headingDepth());
     assertEquals("a", new WikiTokenizer(wikiText).nextToken().headingWikiText());
-    assertEquals(1, new WikiTokenizer(wikiText).nextToken().errors.size());
+    assertEquals(2, new WikiTokenizer(wikiText).nextToken().errors.size());
 
     wikiText = "=a==";
     assertEquals("=a==", new WikiTokenizer(wikiText).nextToken().token());
@@ -168,6 +194,7 @@ public class WikiTokenizerTest extends TestCase {
     assertEquals(2, new WikiTokenizer(wikiText).nextToken().headingDepth());
     assertEquals("aa[[|=]] {{|={{=}} }}", new WikiTokenizer(wikiText).nextToken().headingWikiText());
     assertEquals(0, new WikiTokenizer(wikiText).nextToken().errors.size());
+    
   }