Initialism, changes in regex matching.

diff --git a/bugs b/bugs
index 6e0af80967d670ff7677ab041c4bc3fa37e501f6..42931c4d0abb12ca1d41066ab2b20e354b68f156 100644 (file)
--- a/bugs
+++ b/bugs
@@ -8,6 +8,7 @@ PC:
 sub-levels in translations.
 handle word-info in English.
 italian verbs... (show conjugation, pulled from a linked place....)
+add unit test for: Errors: [Unmatched {{ error: * {{a|US}} {{IPA|[ˈfɔɹ.wɝd]]}}
 
 
 

diff --git a/src/com/hughes/android/dictionary/engine/DictionaryBuilderMain.java b/src/com/hughes/android/dictionary/engine/DictionaryBuilderMain.java
index 20bf4ba4437909286baf5b6593199c19ee8a6b8d..86c91e8922afe1049399a05543242ae85bc81816 100644 (file)
--- a/src/com/hughes/android/dictionary/engine/DictionaryBuilderMain.java
+++ b/src/com/hughes/android/dictionary/engine/DictionaryBuilderMain.java
@@ -18,6 +18,7 @@ import java.io.File;
 import java.io.PrintWriter;
 import java.io.RandomAccessFile;
 import java.util.ArrayList;
+import java.util.Arrays;
 import java.util.Collections;
 import java.util.LinkedHashMap;
 import java.util.List;
@@ -44,7 +45,6 @@ public class DictionaryBuilderMain extends TestCase {
     // German handled in file.
     isoToDedication.put("EL", "Greek dictionary dedicated to Noah Egge.");
     isoToDedication.put("IT", "Italian dictionary dedicated to Carolina Tropini, my favorite stardust in the whole universe!  Ti amo!");
-    isoToDedication.put("JA", "Japanese dictionary dedicated to Akane Watanabe.");
     isoToDedication.put("KO", "Korean dictionary dedicated to Ande Elwood--fall fashion und Fernsehturms!");
     isoToDedication.put("PT", "Portuguese dictionary dedicated to Carlos Melo, one Tough Mudder.");
     isoToDedication.put("RO", "Romanian dictionary dedicated to Radu Teodorescu.");
@@ -61,12 +61,14 @@ public class DictionaryBuilderMain extends TestCase {
     isoToStoplist.put("FR", "fr.txt");
 
     final Map<String,String>  isoToRegex = new LinkedHashMap<String, String>();
-    isoToRegex.put("ZH", ".*Chinese.*|.*Mandarin.*|.*Cantonese.*");
+    isoToRegex.put("ZH", "Chinese|Mandarin|Cantonese");
     
-    boolean go = false; 
-    isoToWikiName.clear();
+    isoToWikiName.keySet().retainAll(Arrays.asList("UK", "HR", "FI"));
+    
+    boolean go = true; 
+//    isoToWikiName.clear();
     for (final String foreignIso : isoToWikiName.keySet()) {
-      if (foreignIso.equals("GA")) {
+      if (foreignIso.equals("SV")) {
         go = true;
       }
       if (!go) {
@@ -83,7 +85,7 @@ public class DictionaryBuilderMain extends TestCase {
           isoToDedication.put(foreignIso, "");
         }
         if (!isoToRegex.containsKey(foreignIso)) {
-          isoToRegex.put(foreignIso, ".*" + isoToWikiName.get(foreignIso) + ".*");
+          isoToRegex.put(foreignIso, isoToWikiName.get(foreignIso));
         }
   
         DictionaryBuilder.main(new String[] {

diff --git a/src/com/hughes/android/dictionary/engine/DictionaryBuilderTest.java b/src/com/hughes/android/dictionary/engine/DictionaryBuilderTest.java
index 99e4e84a9438518ef1142768151e81dd9dfadb0e..8f79c3655c80f6a2f35980c3e97ac233a73ec71e 100644 (file)
--- a/src/com/hughes/android/dictionary/engine/DictionaryBuilderTest.java
+++ b/src/com/hughes/android/dictionary/engine/DictionaryBuilderTest.java
@@ -43,7 +43,6 @@ public class DictionaryBuilderTest extends TestCase {
     wiktionaryTestWithLangToEn("wiktionary.zh_en.quickdic", "ZH", "empty.txt",
         "EN.data", "enwiktionary.english", "Chinese|Mandarin|Cantonese", "zh");
   }
-
   
   // German
   public void testWiktionary_DE_DE() throws Exception {

diff --git a/src/com/hughes/android/dictionary/parser/EnWiktionaryXmlParser.java b/src/com/hughes/android/dictionary/parser/EnWiktionaryXmlParser.java
index a7bf170c09e145b582919d43fb5d5ed0963dc20b..9c867a0a8de132a068d748557cd45d60046acea4 100644 (file)
--- a/src/com/hughes/android/dictionary/parser/EnWiktionaryXmlParser.java
+++ b/src/com/hughes/android/dictionary/parser/EnWiktionaryXmlParser.java
@@ -46,7 +46,7 @@ public class EnWiktionaryXmlParser {
       "Noun|Verb|Adjective|Adverb|Pronoun|Conjunction|Interjection|" +
       "Preposition|Proper noun|Article|Prepositional phrase|Acronym|" +
       "Abbreviation|Initialism|Contraction|Prefix|Suffix|Symbol|Letter|" +
-      "Ligature|Idiom|Phrase|" +
+      "Ligature|Idiom|Phrase|{{initialism}}|" +
       // These are @deprecated:
       "Noun form|Verb form|Adjective form|Nominal phrase|Noun phrase|" +
       "Verb phrase|Transitive verb|Intransitive verb|Reflexive verb|" +
@@ -201,6 +201,7 @@ public class EnWiktionaryXmlParser {
           // TODO: would also be nice...
         } else if (functionName.startsWith("picdic")) {
         } else if (functionName.startsWith("checktrans")) {
+          done = true;
         } else if (functionName.startsWith("ttbc")) {
           wikiTokenizer.nextLine();
           // TODO: would be great to handle ttbc
@@ -213,7 +214,7 @@ public class EnWiktionaryXmlParser {
         // This line could produce an output...
         
         if (line.contains("ich hoan dich gear")) {
-          System.out.println();
+          //System.out.println();
         }
         
         // First strip the language and check whether it matches.
@@ -704,14 +705,16 @@ public class EnWiktionaryXmlParser {
           pairEntry.pairs.add(pair);
         }
       } else if (nextPrefix.equals("#::") || nextPrefix.equals("#**")) {
-        if (lastForeign != null) {
+        if (lastForeign != null && pairEntry.pairs.size() > 0) {
           pairEntry.pairs.remove(pairEntry.pairs.size() - 1);
           final Pair pair = new Pair(formatAndIndexExampleString(nextLine, enIndexBuilder, indexedEntry), formatAndIndexExampleString(lastForeign, otherIndexBuilder, indexedEntry), swap);
           if (pair.lang1 != "--" && pair.lang1 != "--") {
             pairEntry.pairs.add(pair);
           }
+          lastForeign = null;
         } else {
-          LOG.warning("English example with no foreign: " + title + ", " + nextLine);
+          LOG.warning("TODO: English example with no foreign: " + title + ", " + nextLine);
+          // TODO: add something.
         }
       } else if (nextPrefix.equals("#*")) {
         // Can't really index these.

diff --git a/src/com/hughes/android/dictionary/parser/WikiTokenizer.java b/src/com/hughes/android/dictionary/parser/WikiTokenizer.java
index 403b27c14ccb34ee1225b352b05df9a77d2f5ccf..e12185b7f323a3abf92c6d353ad14a3caf59d247 100644 (file)
--- a/src/com/hughes/android/dictionary/parser/WikiTokenizer.java
+++ b/src/com/hughes/android/dictionary/parser/WikiTokenizer.java
@@ -173,6 +173,7 @@ public final class WikiTokenizer {
     if (lastUnescapedPipePos != -1) {
       return wikiText.substring(lastUnescapedPipePos + 1, end - 2);
     }
+    assert start + 2 < wikiText.length() && end >= 2: wikiText;
     return wikiText.substring(start + 2, end - 2);
   }