Fixing examples...

[DictionaryPC.git] / src / com / hughes / android / dictionary / parser / EnWiktionaryXmlParser.java

diff --git a/src/com/hughes/android/dictionary/parser/EnWiktionaryXmlParser.java b/src/com/hughes/android/dictionary/parser/EnWiktionaryXmlParser.java
index 6eac29d5e4d3054e214217febb120d22bb6660c6..c2e6e7c2726e1f7ea5d8b5c0aa3d33fbd295bc2e 100644 (file)
--- a/src/com/hughes/android/dictionary/parser/EnWiktionaryXmlParser.java
+++ b/src/com/hughes/android/dictionary/parser/EnWiktionaryXmlParser.java
@@ -23,7 +23,6 @@ import java.io.IOException;
 import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.Collection;
-import java.util.Collections;
 import java.util.LinkedHashSet;
 import java.util.List;
 import java.util.Map;
@@ -127,10 +126,11 @@ public class EnWiktionaryXmlParser {
   
   // -------------------------------------------------------------------------
   
-  String pos = null;
-  int posDepth = -1;
-
   private void doEnglishWord(String title, String text) {
+    
+    String pos = null;
+    int posDepth = -1;
+
     final WikiTokenizer wikiTokenizer = new WikiTokenizer(text);
     while (wikiTokenizer.nextToken() != null) {
       
@@ -146,7 +146,10 @@ public class EnWiktionaryXmlParser {
           posDepth = wikiTokenizer.headingDepth();
           pos = wikiTokenizer.headingWikiText();
         } else if (headerName.equals("Translations")) {
-          doTranslations(title, wikiTokenizer);
+          if (pos == null) {
+            LOG.warning("Translations without POS: " + title);
+          }
+          doTranslations(title, wikiTokenizer, pos);
         } else if (headerName.equals("Pronunciation")) {
           //doPronunciation(wikiLineReader);
         }
@@ -161,7 +164,7 @@ public class EnWiktionaryXmlParser {
       "Jpan", "Kore", "Hebr", "rfscript", "Beng", "Mong", "Knda", "Cyrs",
       "yue-tsj", "Mlym", "Tfng", "Grek", "yue-yue-j"));
   
-  private void doTranslations(final String title, final WikiTokenizer wikiTokenizer) {
+  private void doTranslations(final String title, final WikiTokenizer wikiTokenizer, final String pos) {
     if (title.equals("absolutely")) {
       System.out.println();
     }
@@ -207,7 +210,7 @@ public class EnWiktionaryXmlParser {
         } else {
           LOG.warning("Unexpected translation wikifunction: " + wikiTokenizer.token() + ", title=" + title);
         }
-      } else if (wikiTokenizer.isListItem() && wikiTokenizer.listItemPrefix().startsWith("*")) {
+      } else if (wikiTokenizer.isListItem()) {
         final String line = wikiTokenizer.listItemWikiText();
         // This line could produce an output...
         
@@ -225,7 +228,7 @@ public class EnWiktionaryXmlParser {
         
         String rest = line.substring(colonIndex + 1).trim();
         if (rest.length() > 0) {
-          doTranslationLine(line, title, sense, rest);
+          doTranslationLine(line, title, pos, sense, rest);
         } else {
           // TODO: do lines that are like Greek:
         }
@@ -256,7 +259,7 @@ public class EnWiktionaryXmlParser {
     return index < list.size() ? list.get(index) : null;
   }
   
-  private void doTranslationLine(final String line, final String title, final String sense, final String rest) {
+  private void doTranslationLine(final String line, final String title, final String pos, final String sense, final String rest) {
     // Good chance we'll actually file this one...
     final PairEntry pairEntry = new PairEntry();
     final IndexedEntry indexedEntry = new IndexedEntry(pairEntry);
@@ -554,11 +557,12 @@ public class EnWiktionaryXmlParser {
     } finally {
       // Here's where we exit.
       // TODO: Should we make an entry even if there are no foreign list items?
-      if (foreignBuilder.indexOf(title) == -1) {
-        foreignBuilder.insert(0, title + " ");
+      String foreign = foreignBuilder.toString().trim();
+      if (!foreign.toLowerCase().startsWith(title.toLowerCase())) {
+        foreign = title + " " + foreign;
       }
       for (final ListSection listSection : listSections) {
-        doForeignListItem(foreignBuilder.toString(), title, wordForms, listSection);
+        doForeignListItem(foreign, title, wordForms, listSection);
       }
     }
   }
@@ -603,10 +607,11 @@ public class EnWiktionaryXmlParser {
             englishBuilder.append(text);
             otherIndexBuilder.addEntryWithString(indexedEntry, text, EntryTypeName.WIKTIONARY_ENGLISH_DEF_OTHER_LANG);
           } else if (link.equals("plural")) {
-            englishBuilder.append(englishTokenizer.wikiLinkText());
+            englishBuilder.append(text);
           } else {
             //LOG.warning("Special link: " + englishTokenizer.token());
-            englishBuilder.append(englishTokenizer.wikiLinkText());
+            enIndexBuilder.addEntryWithString(indexedEntry, text, EntryTypeName.WIKTIONARY_ENGLISH_DEF_WIKI_LINK);
+            englishBuilder.append(text);
           }
         } else {
           // link == null
@@ -634,6 +639,7 @@ public class EnWiktionaryXmlParser {
         }
       }
     }
+        
     final String english = trim(englishBuilder.toString());
     if (english.length() > 0) {
       final Pair pair = new Pair(english, trim(foreignText), this.swap);
@@ -643,6 +649,80 @@ public class EnWiktionaryXmlParser {
         otherIndexBuilder.addEntryWithString(indexedEntry, form, EntryTypeName.WIKTIONARY_FORM_SINGLE, EntryTypeName.WIKTIONARY_FORM_MULTI);
       }
     }
+    
+    // Do examples.
+    String lastForeign = null;
+    for (int i = 0; i < listSection.nextPrefixes.size(); ++i) {
+      final String nextPrefix = listSection.nextPrefixes.get(i);
+      final String nextLine = listSection.nextLines.get(i);
+      int dash = nextLine.indexOf("&mdash;");
+      int mdashLen = 7;
+      if (dash == -1) {
+        dash = nextLine.indexOf("—");
+        mdashLen = 1;
+      }
+      if (dash == -1) {
+        dash = nextLine.indexOf(" - ");
+        mdashLen = 3;
+      }
+      
+      // TODO: index and clean these!!!
+      if (nextPrefix.equals("#:") && dash != -1) {
+        final String foreignEx = nextLine.substring(0, dash);
+        final String englishEx = nextLine.substring(dash + mdashLen);
+        final Pair pair = new Pair(formatAndIndexExampleString(englishEx, enIndexBuilder), formatAndIndexExampleString(foreignEx, otherIndexBuilder), swap);
+        pairEntry.pairs.add(pair);
+        lastForeign = null;
+      } else if (nextPrefix.equals("#:")){
+        final Pair pair = new Pair("--", formatAndIndexExampleString(nextLine, null), swap);
+        lastForeign = nextLine;
+        pairEntry.pairs.add(pair);
+      } else if (nextPrefix.equals("#::") || nextPrefix.equals("#**")) {
+        if (lastForeign != null) {
+          pairEntry.pairs.remove(pairEntry.pairs.size() - 1);
+          final Pair pair = new Pair(formatAndIndexExampleString(nextLine, enIndexBuilder), formatAndIndexExampleString(lastForeign, otherIndexBuilder), swap);
+          pairEntry.pairs.add(pair);
+        } else {
+          LOG.warning("English example with no foreign: " + title + ", " + nextLine);
+        }
+      } else if (nextPrefix.equals("#*")) {
+        // Can't really index these.
+        final Pair pair = new Pair("--", formatAndIndexExampleString(nextLine, null), swap);
+        lastForeign = nextLine;
+        pairEntry.pairs.add(pair);
+      } else if (nextPrefix.equals("#::*")) {
+        final Pair pair = new Pair("--", formatAndIndexExampleString(nextLine, null), swap);
+        pairEntry.pairs.add(pair);
+      } else {
+        assert false;
+      }
+    }
+  }
+  
+  private String formatAndIndexExampleString(final String example, final IndexBuilder indexBuilder) {
+    final WikiTokenizer wikiTokenizer = new WikiTokenizer(example, false);
+    final StringBuilder builder = new StringBuilder();
+    boolean insideTripleQuotes = false;
+    while (wikiTokenizer.nextToken() != null) {
+      if (wikiTokenizer.isPlainText()) {
+        builder.append(wikiTokenizer.token());
+        
+      } else if (wikiTokenizer.isWikiLink()) {
+        builder.append(wikiTokenizer.wikiLinkText());
+        
+      } else if (wikiTokenizer.isFunction()) {
+        builder.append(wikiTokenizer.token());
+      } else if (wikiTokenizer.isMarkup()) {
+        if (wikiTokenizer.token().equals("'''")) {
+          insideTripleQuotes = !insideTripleQuotes;
+        }
+      } else if (wikiTokenizer.isComment() || wikiTokenizer.isNewline()) {
+        // Do nothing.
+      } else {
+        LOG.warning("unexpected token: " + wikiTokenizer.token());
+      }
+    }
+    return trim(builder.toString()); 
   }