X-Git-Url: http://gitweb.fperrin.net/?a=blobdiff_plain;f=src%2Fcom%2Fhughes%2Fandroid%2Fdictionary%2Fparser%2FWikiTokenizerTest.java;h=8398719f18a08a32577c4ee9db7530fb7cfd0625;hb=8f3cf0a0d01b1ffeab4fa2c4ef1f69203be9a891;hp=8e1d5a87733300b4fd7c72001f63d6cea08d4d6f;hpb=481738c6610def8dcd0378f3c419fa8c0a0f6cf3;p=DictionaryPC.git

diff --git a/src/com/hughes/android/dictionary/parser/WikiTokenizerTest.java b/src/com/hughes/android/dictionary/parser/WikiTokenizerTest.java
index 8e1d5a8..8398719 100644
--- a/src/com/hughes/android/dictionary/parser/WikiTokenizerTest.java
+++ b/src/com/hughes/android/dictionary/parser/WikiTokenizerTest.java
@@ -302,4 +302,31 @@ public class WikiTokenizerTest extends TestCase {
     assertEquals(Arrays.asList(expectedTokens), actualTokens);
   }
   
+  public void testHtml() {
+      String wikiText;
+
+      {
+      wikiText = " zz <pre> asdf </pre> ZZ <math> 1234 </math> XX ";
+      final WikiTokenizer tokenizer = new WikiTokenizer(wikiText);
+      assertEquals(" zz ", tokenizer.nextToken().token());
+      assertEquals("<pre> asdf </pre>", tokenizer.nextToken().token());
+      assertEquals(" ZZ ", tokenizer.nextToken().token());
+      assertEquals("<math> 1234 </math>", tokenizer.nextToken().token());
+      assertEquals(" XX ", tokenizer.nextToken().token());
+      }
+      {
+      wikiText = "\n<math> 1234 </math>";
+      final WikiTokenizer tokenizer = new WikiTokenizer(wikiText);
+      assertEquals("<math> 1234 </math>", tokenizer.nextToken().nextToken().token());
+      }
+
+      {
+      wikiText = "# z'' is the '''free''' variable in \"<math>\\forall x\\exists y:xy=z</math>\".''";
+      final WikiTokenizer tokenizer = new WikiTokenizer(wikiText);
+      assertEquals(wikiText, tokenizer.nextToken().token());
+      }
+
+      
+  }
+  
 }