First pass at examples.

[DictionaryPC.git] / src / com / hughes / android / dictionary / parser / EnWiktionaryXmlParser.java

diff --git a/src/com/hughes/android/dictionary/parser/EnWiktionaryXmlParser.java b/src/com/hughes/android/dictionary/parser/EnWiktionaryXmlParser.java
index 6eac29d5e4d3054e214217febb120d22bb6660c6..d9a1249afc074bde77cec95db0b18e91f611e41a 100644 (file)
--- a/src/com/hughes/android/dictionary/parser/EnWiktionaryXmlParser.java
+++ b/src/com/hughes/android/dictionary/parser/EnWiktionaryXmlParser.java
@@ -127,10 +127,11 @@ public class EnWiktionaryXmlParser {
   
   // -------------------------------------------------------------------------
   
-  String pos = null;
-  int posDepth = -1;
-
   private void doEnglishWord(String title, String text) {
+    
+    String pos = null;
+    int posDepth = -1;
+
     final WikiTokenizer wikiTokenizer = new WikiTokenizer(text);
     while (wikiTokenizer.nextToken() != null) {
       
@@ -146,7 +147,10 @@ public class EnWiktionaryXmlParser {
           posDepth = wikiTokenizer.headingDepth();
           pos = wikiTokenizer.headingWikiText();
         } else if (headerName.equals("Translations")) {
-          doTranslations(title, wikiTokenizer);
+          if (pos == null) {
+            LOG.warning("Translations without POS: " + title);
+          }
+          doTranslations(title, wikiTokenizer, pos);
         } else if (headerName.equals("Pronunciation")) {
           //doPronunciation(wikiLineReader);
         }
@@ -161,7 +165,7 @@ public class EnWiktionaryXmlParser {
       "Jpan", "Kore", "Hebr", "rfscript", "Beng", "Mong", "Knda", "Cyrs",
       "yue-tsj", "Mlym", "Tfng", "Grek", "yue-yue-j"));
   
-  private void doTranslations(final String title, final WikiTokenizer wikiTokenizer) {
+  private void doTranslations(final String title, final WikiTokenizer wikiTokenizer, final String pos) {
     if (title.equals("absolutely")) {
       System.out.println();
     }
@@ -207,7 +211,7 @@ public class EnWiktionaryXmlParser {
         } else {
           LOG.warning("Unexpected translation wikifunction: " + wikiTokenizer.token() + ", title=" + title);
         }
-      } else if (wikiTokenizer.isListItem() && wikiTokenizer.listItemPrefix().startsWith("*")) {
+      } else if (wikiTokenizer.isListItem()) {
         final String line = wikiTokenizer.listItemWikiText();
         // This line could produce an output...
         
@@ -225,7 +229,7 @@ public class EnWiktionaryXmlParser {
         
         String rest = line.substring(colonIndex + 1).trim();
         if (rest.length() > 0) {
-          doTranslationLine(line, title, sense, rest);
+          doTranslationLine(line, title, pos, sense, rest);
         } else {
           // TODO: do lines that are like Greek:
         }
@@ -256,7 +260,7 @@ public class EnWiktionaryXmlParser {
     return index < list.size() ? list.get(index) : null;
   }
   
-  private void doTranslationLine(final String line, final String title, final String sense, final String rest) {
+  private void doTranslationLine(final String line, final String title, final String pos, final String sense, final String rest) {
     // Good chance we'll actually file this one...
     final PairEntry pairEntry = new PairEntry();
     final IndexedEntry indexedEntry = new IndexedEntry(pairEntry);
@@ -554,11 +558,12 @@ public class EnWiktionaryXmlParser {
     } finally {
       // Here's where we exit.
       // TODO: Should we make an entry even if there are no foreign list items?
-      if (foreignBuilder.indexOf(title) == -1) {
-        foreignBuilder.insert(0, title + " ");
+      String foreign = foreignBuilder.toString().trim();
+      if (!foreign.toLowerCase().startsWith(title.toLowerCase())) {
+        foreign = title + " " + foreign;
       }
       for (final ListSection listSection : listSections) {
-        doForeignListItem(foreignBuilder.toString(), title, wordForms, listSection);
+        doForeignListItem(foreign, title, wordForms, listSection);
       }
     }
   }
@@ -603,10 +608,11 @@ public class EnWiktionaryXmlParser {
             englishBuilder.append(text);
             otherIndexBuilder.addEntryWithString(indexedEntry, text, EntryTypeName.WIKTIONARY_ENGLISH_DEF_OTHER_LANG);
           } else if (link.equals("plural")) {
-            englishBuilder.append(englishTokenizer.wikiLinkText());
+            englishBuilder.append(text);
           } else {
             //LOG.warning("Special link: " + englishTokenizer.token());
-            englishBuilder.append(englishTokenizer.wikiLinkText());
+            enIndexBuilder.addEntryWithString(indexedEntry, text, EntryTypeName.WIKTIONARY_ENGLISH_DEF_WIKI_LINK);
+            englishBuilder.append(text);
           }
         } else {
           // link == null
@@ -634,6 +640,7 @@ public class EnWiktionaryXmlParser {
         }
       }
     }
+        
     final String english = trim(englishBuilder.toString());
     if (english.length() > 0) {
       final Pair pair = new Pair(english, trim(foreignText), this.swap);
@@ -643,6 +650,60 @@ public class EnWiktionaryXmlParser {
         otherIndexBuilder.addEntryWithString(indexedEntry, form, EntryTypeName.WIKTIONARY_FORM_SINGLE, EntryTypeName.WIKTIONARY_FORM_MULTI);
       }
     }
+    
+    // Do examples.
+    String lastForeign = null;
+    for (int i = 0; i < listSection.nextPrefixes.size(); ++i) {
+      final String nextPrefix = listSection.nextPrefixes.get(i);
+      final String nextLine = listSection.nextLines.get(i);
+      int mdash = nextLine.indexOf("&mdash;");
+      int mdashLen = 7;
+      if (mdash == -1) {
+        mdash = nextLine.indexOf("—");
+        mdashLen = 1;
+      }
+      if (mdash == -1) {
+        mdash = nextLine.indexOf("'',");
+        mdashLen = 3;
+      }
+      if (mdash == -1) {
+        mdash = nextLine.indexOf(" - ");
+        mdashLen = 3;
+      }
+      
+      // TODO: index and clean these!!!
+      if (nextPrefix.equals("#:") && mdash != -1) {
+        final String foreignEx = nextLine.substring(0, mdash);
+        final String englishEx = nextLine.substring(mdash + mdashLen);
+        final Pair pair = new Pair(trim(englishEx), trim(foreignEx), swap);
+        pairEntry.pairs.add(pair);
+        lastForeign = null;
+      } else if (nextPrefix.equals("#:")){
+        final Pair pair = new Pair("--", trim(nextLine), swap);
+        lastForeign = nextLine;
+        pairEntry.pairs.add(pair);
+      } else if (nextPrefix.equals("#::") || nextPrefix.equals("#**")) {
+        if (lastForeign != null) {
+          pairEntry.pairs.remove(pairEntry.pairs.size() - 1);
+          final Pair pair = new Pair(nextLine, lastForeign, swap);
+          pairEntry.pairs.add(pair);
+        } else {
+          LOG.warning("English example with no foreign: " + title + ", " + nextLine);
+        }
+      } else if (nextPrefix.equals("#*")) {
+        // Can't really index these.
+        final Pair pair = new Pair("--", trim(nextLine), swap);
+        lastForeign = nextLine;
+        pairEntry.pairs.add(pair);
+      } else if (nextPrefix.equals("#::*")) {
+        final Pair pair = new Pair("--", trim(nextLine), swap);
+        pairEntry.pairs.add(pair);
+      } else {
+        assert false;
+      }
+    }
+    
+
   }