First pass at examples.

[DictionaryPC.git] / src / com / hughes / android / dictionary / parser / EnWiktionaryXmlParser.java

diff --git a/src/com/hughes/android/dictionary/parser/EnWiktionaryXmlParser.java b/src/com/hughes/android/dictionary/parser/EnWiktionaryXmlParser.java
index 54a11e33dcfdfc0dc7b9c2da0b6b882ea67083d1..d9a1249afc074bde77cec95db0b18e91f611e41a 100644 (file)
--- a/src/com/hughes/android/dictionary/parser/EnWiktionaryXmlParser.java
+++ b/src/com/hughes/android/dictionary/parser/EnWiktionaryXmlParser.java
@@ -127,10 +127,11 @@ public class EnWiktionaryXmlParser {
   
   // -------------------------------------------------------------------------
   
-  String pos = null;
-  int posDepth = -1;
-
   private void doEnglishWord(String title, String text) {
+    
+    String pos = null;
+    int posDepth = -1;
+
     final WikiTokenizer wikiTokenizer = new WikiTokenizer(text);
     while (wikiTokenizer.nextToken() != null) {
       
@@ -146,7 +147,10 @@ public class EnWiktionaryXmlParser {
           posDepth = wikiTokenizer.headingDepth();
           pos = wikiTokenizer.headingWikiText();
         } else if (headerName.equals("Translations")) {
-          doTranslations(title, wikiTokenizer);
+          if (pos == null) {
+            LOG.warning("Translations without POS: " + title);
+          }
+          doTranslations(title, wikiTokenizer, pos);
         } else if (headerName.equals("Pronunciation")) {
           //doPronunciation(wikiLineReader);
         }
@@ -161,7 +165,7 @@ public class EnWiktionaryXmlParser {
       "Jpan", "Kore", "Hebr", "rfscript", "Beng", "Mong", "Knda", "Cyrs",
       "yue-tsj", "Mlym", "Tfng", "Grek", "yue-yue-j"));
   
-  private void doTranslations(final String title, final WikiTokenizer wikiTokenizer) {
+  private void doTranslations(final String title, final WikiTokenizer wikiTokenizer, final String pos) {
     if (title.equals("absolutely")) {
       System.out.println();
     }
@@ -207,7 +211,7 @@ public class EnWiktionaryXmlParser {
         } else {
           LOG.warning("Unexpected translation wikifunction: " + wikiTokenizer.token() + ", title=" + title);
         }
-      } else if (wikiTokenizer.isListItem() && wikiTokenizer.listItemPrefix().startsWith("*")) {
+      } else if (wikiTokenizer.isListItem()) {
         final String line = wikiTokenizer.listItemWikiText();
         // This line could produce an output...
         
@@ -225,7 +229,7 @@ public class EnWiktionaryXmlParser {
         
         String rest = line.substring(colonIndex + 1).trim();
         if (rest.length() > 0) {
-          doTranslationLine(line, title, sense, rest);
+          doTranslationLine(line, title, pos, sense, rest);
         } else {
           // TODO: do lines that are like Greek:
         }
@@ -256,7 +260,7 @@ public class EnWiktionaryXmlParser {
     return index < list.size() ? list.get(index) : null;
   }
   
-  private void doTranslationLine(final String line, final String title, final String sense, final String rest) {
+  private void doTranslationLine(final String line, final String title, final String pos, final String sense, final String rest) {
     // Good chance we'll actually file this one...
     final PairEntry pairEntry = new PairEntry();
     final IndexedEntry indexedEntry = new IndexedEntry(pairEntry);
@@ -435,14 +439,17 @@ public class EnWiktionaryXmlParser {
   }
 
 
+  int foreignCount = 0;
   private void doForeignPartOfSpeech(String title, final String posHeading, final int posDepth, WikiTokenizer wikiTokenizer) {
-    LOG.info("***" + title + ", pos=" + posHeading);
+    if (++foreignCount % 1000 == 0) {
+      LOG.info("***" + title + ", pos=" + posHeading + ", foreignCount=" + foreignCount);
+    }
     if (title.equals("moro")) {
       System.out.println();
     }
     
     final StringBuilder foreignBuilder = new StringBuilder();
-    Collection<String> wordForms = Collections.emptyList();
+    final Collection<String> wordForms = new ArrayList<String>();
     final List<ListSection> listSections = new ArrayList<ListSection>();
     
     try {
@@ -495,13 +502,13 @@ public class EnWiktionaryXmlParser {
           //foreignBuilder.append(title);
         }
       } else if (name.equals("it-noun")) {
-          assert wordForms.isEmpty();
           final String base = get(args, 0);
           final String gender = get(args, 1);
           final String singular = base + get(args, 2);
           final String plural = base + get(args, 3);
-          foreignBuilder.append(String.format("%s {%s}, %s {pl}", singular, gender, plural, plural));
-          wordForms = Arrays.asList(singular, plural);
+          foreignBuilder.append(String.format(" %s {%s}, %s {pl}", singular, gender, plural, plural));
+          wordForms.add(singular);
+          wordForms.add(plural);
         } else if (name.equals("it-proper noun")) {
           foreignBuilder.append(wikiTokenizer.token());
         } else if (name.equals("it-adj")) {
@@ -551,11 +558,12 @@ public class EnWiktionaryXmlParser {
     } finally {
       // Here's where we exit.
       // TODO: Should we make an entry even if there are no foreign list items?
-      if (foreignBuilder.indexOf(title) == -1) {
-        foreignBuilder.insert(0, title + " ");
+      String foreign = foreignBuilder.toString().trim();
+      if (!foreign.toLowerCase().startsWith(title.toLowerCase())) {
+        foreign = title + " " + foreign;
       }
       for (final ListSection listSection : listSections) {
-        doForeignListItem(foreignBuilder.toString(), title, wordForms, listSection);
+        doForeignListItem(foreign, title, wordForms, listSection);
       }
     }
   }
@@ -599,9 +607,12 @@ public class EnWiktionaryXmlParser {
           } else if (link.contains("#") && this.langPattern.matcher(link).find()) {
             englishBuilder.append(text);
             otherIndexBuilder.addEntryWithString(indexedEntry, text, EntryTypeName.WIKTIONARY_ENGLISH_DEF_OTHER_LANG);
+          } else if (link.equals("plural")) {
+            englishBuilder.append(text);
           } else {
-            LOG.warning("Special link: " + englishTokenizer.token());
-            // TODO: something here...
+            //LOG.warning("Special link: " + englishTokenizer.token());
+            enIndexBuilder.addEntryWithString(indexedEntry, text, EntryTypeName.WIKTIONARY_ENGLISH_DEF_WIKI_LINK);
+            englishBuilder.append(text);
           }
         } else {
           // link == null
@@ -612,14 +623,15 @@ public class EnWiktionaryXmlParser {
         }
       } else if (englishTokenizer.isFunction()) {
         final String name = englishTokenizer.functionName();
-        if (name.contains(" conjugation of ") || 
-            name.contains(" form of ") || 
-            name.contains(" feminine of ") || 
-            name.contains(" plural of ")) {
+        if (name.contains("conjugation of ") || 
+            name.contains("form of ") || 
+            name.contains("feminine of ") || 
+            name.contains("plural of ")) {
           // Ignore these in the index, they're really annoying....
           englishBuilder.append(englishTokenizer.token());
         } else {
-          LOG.warning("Unexpected function: " + englishTokenizer.token());
+          englishBuilder.append(englishTokenizer.token());
+//          LOG.warning("Unexpected function: " + englishTokenizer.token());
         }
       } else {
         if (englishTokenizer.isComment() || englishTokenizer.isMarkup()) {
@@ -628,6 +640,7 @@ public class EnWiktionaryXmlParser {
         }
       }
     }
+        
     final String english = trim(englishBuilder.toString());
     if (english.length() > 0) {
       final Pair pair = new Pair(english, trim(foreignText), this.swap);
@@ -637,6 +650,60 @@ public class EnWiktionaryXmlParser {
         otherIndexBuilder.addEntryWithString(indexedEntry, form, EntryTypeName.WIKTIONARY_FORM_SINGLE, EntryTypeName.WIKTIONARY_FORM_MULTI);
       }
     }
+    
+    // Do examples.
+    String lastForeign = null;
+    for (int i = 0; i < listSection.nextPrefixes.size(); ++i) {
+      final String nextPrefix = listSection.nextPrefixes.get(i);
+      final String nextLine = listSection.nextLines.get(i);
+      int mdash = nextLine.indexOf("&mdash;");
+      int mdashLen = 7;
+      if (mdash == -1) {
+        mdash = nextLine.indexOf("—");
+        mdashLen = 1;
+      }
+      if (mdash == -1) {
+        mdash = nextLine.indexOf("'',");
+        mdashLen = 3;
+      }
+      if (mdash == -1) {
+        mdash = nextLine.indexOf(" - ");
+        mdashLen = 3;
+      }
+      
+      // TODO: index and clean these!!!
+      if (nextPrefix.equals("#:") && mdash != -1) {
+        final String foreignEx = nextLine.substring(0, mdash);
+        final String englishEx = nextLine.substring(mdash + mdashLen);
+        final Pair pair = new Pair(trim(englishEx), trim(foreignEx), swap);
+        pairEntry.pairs.add(pair);
+        lastForeign = null;
+      } else if (nextPrefix.equals("#:")){
+        final Pair pair = new Pair("--", trim(nextLine), swap);
+        lastForeign = nextLine;
+        pairEntry.pairs.add(pair);
+      } else if (nextPrefix.equals("#::") || nextPrefix.equals("#**")) {
+        if (lastForeign != null) {
+          pairEntry.pairs.remove(pairEntry.pairs.size() - 1);
+          final Pair pair = new Pair(nextLine, lastForeign, swap);
+          pairEntry.pairs.add(pair);
+        } else {
+          LOG.warning("English example with no foreign: " + title + ", " + nextLine);
+        }
+      } else if (nextPrefix.equals("#*")) {
+        // Can't really index these.
+        final Pair pair = new Pair("--", trim(nextLine), swap);
+        lastForeign = nextLine;
+        pairEntry.pairs.add(pair);
+      } else if (nextPrefix.equals("#::*")) {
+        final Pair pair = new Pair("--", trim(nextLine), swap);
+        pairEntry.pairs.add(pair);
+      } else {
+        assert false;
+      }
+    }
+    
+
   }