X-Git-Url: http://gitweb.fperrin.net/?a=blobdiff_plain;f=src%2Fcom%2Fhughes%2Fandroid%2Fdictionary%2Fparser%2FEnWiktionaryXmlParser.java;h=428e11d41489c16ba9804a46546f0f674175033a;hb=2ef2645d18d49dafc11e4b1c02e548849ac82ed2;hp=54a11e33dcfdfc0dc7b9c2da0b6b882ea67083d1;hpb=d49dab6bd67bd257fc05dba9f288c5ada3bd2071;p=DictionaryPC.git

diff --git a/src/com/hughes/android/dictionary/parser/EnWiktionaryXmlParser.java b/src/com/hughes/android/dictionary/parser/EnWiktionaryXmlParser.java
index 54a11e3..428e11d 100644
--- a/src/com/hughes/android/dictionary/parser/EnWiktionaryXmlParser.java
+++ b/src/com/hughes/android/dictionary/parser/EnWiktionaryXmlParser.java
@@ -23,7 +23,6 @@ import java.io.IOException;
 import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.Collection;
-import java.util.Collections;
 import java.util.LinkedHashSet;
 import java.util.List;
 import java.util.Map;
@@ -41,14 +40,13 @@ public class EnWiktionaryXmlParser {
   
   static final Logger LOG = Logger.getLogger(EnWiktionaryXmlParser.class.getName());
   
-  // TODO: look for {{ and [[ and <adf> <!-- in output.
   // TODO: process {{ttbc}} lines
   
   static final Pattern partOfSpeechHeader = Pattern.compile(
       "Noun|Verb|Adjective|Adverb|Pronoun|Conjunction|Interjection|" +
       "Preposition|Proper noun|Article|Prepositional phrase|Acronym|" +
       "Abbreviation|Initialism|Contraction|Prefix|Suffix|Symbol|Letter|" +
-      "Ligature|Idiom|Phrase|" +
+      "Ligature|Idiom|Phrase|" + //|\\{\\{acronym\\}\\}|\\{\\{initialism\\}\\}
       // These are @deprecated:
       "Noun form|Verb form|Adjective form|Nominal phrase|Noun phrase|" +
       "Verb phrase|Transitive verb|Intransitive verb|Reflexive verb|" +
@@ -127,10 +125,11 @@ public class EnWiktionaryXmlParser {
   
   // -------------------------------------------------------------------------
   
-  String pos = null;
-  int posDepth = -1;
-
   private void doEnglishWord(String title, String text) {
+    
+    String pos = null;
+    int posDepth = -1;
+
     final WikiTokenizer wikiTokenizer = new WikiTokenizer(text);
     while (wikiTokenizer.nextToken() != null) {
       
@@ -146,7 +145,10 @@ public class EnWiktionaryXmlParser {
           posDepth = wikiTokenizer.headingDepth();
           pos = wikiTokenizer.headingWikiText();
         } else if (headerName.equals("Translations")) {
-          doTranslations(title, wikiTokenizer);
+          if (pos == null) {
+            LOG.warning("Translations without POS: " + title);
+          }
+          doTranslations(title, wikiTokenizer, pos);
         } else if (headerName.equals("Pronunciation")) {
           //doPronunciation(wikiLineReader);
         }
@@ -161,11 +163,12 @@ public class EnWiktionaryXmlParser {
       "Jpan", "Kore", "Hebr", "rfscript", "Beng", "Mong", "Knda", "Cyrs",
       "yue-tsj", "Mlym", "Tfng", "Grek", "yue-yue-j"));
   
-  private void doTranslations(final String title, final WikiTokenizer wikiTokenizer) {
+  private void doTranslations(final String title, final WikiTokenizer wikiTokenizer, final String pos) {
     if (title.equals("absolutely")) {
-      System.out.println();
+      //System.out.println();
     }
     
+    String topLevelLang = null;
     String sense = null;
     boolean done = false;
     while (wikiTokenizer.nextToken() != null) {
@@ -179,8 +182,6 @@ public class EnWiktionaryXmlParser {
       
       // Check whether we care about this line:
       
-      //line = WikiLineReader.removeSquareBrackets(line);
-      
       if (wikiTokenizer.isFunction()) {
         final String functionName = wikiTokenizer.functionName();
         final List<String> positionArgs = wikiTokenizer.functionPositionArgs();
@@ -197,20 +198,25 @@ public class EnWiktionaryXmlParser {
           sense = null;
         } else if (functionName.equals("trans-mid")) {
         } else if (functionName.equals("trans-see")) {
-          // TODO
+          // TODO: would also be nice...
         } else if (functionName.startsWith("picdic")) {
         } else if (functionName.startsWith("checktrans")) {
+          done = true;
         } else if (functionName.startsWith("ttbc")) {
           wikiTokenizer.nextLine();
-          // TODO: would be great to handle
-          //TODO: Check this: done = true;
+          // TODO: would be great to handle ttbc
+          // TODO: Check this: done = true;
         } else {
           LOG.warning("Unexpected translation wikifunction: " + wikiTokenizer.token() + ", title=" + title);
         }
-      } else if (wikiTokenizer.isListItem() && wikiTokenizer.listItemPrefix().startsWith("*")) {
+      } else if (wikiTokenizer.isListItem()) {
         final String line = wikiTokenizer.listItemWikiText();
         // This line could produce an output...
         
+        if (line.contains("ich hoan dich gear")) {
+          //System.out.println();
+        }
+        
         // First strip the language and check whether it matches.
         // And hold onto it for sub-lines.
         final int colonIndex = line.indexOf(":");
@@ -218,16 +224,28 @@ public class EnWiktionaryXmlParser {
           continue;
         }
         
-        final String lang = line.substring(0, colonIndex);
-        if (!this.langPattern.matcher(lang).find()) {
+        final String lang = trim(WikiTokenizer.toPlainText(line.substring(0, colonIndex)));
+        final boolean appendLang;
+        if (wikiTokenizer.listItemPrefix().length() == 1) {
+          topLevelLang = lang;
+          final boolean thisFind = langPattern.matcher(lang).find();
+          if (!thisFind) {
+            continue;
+          }
+          appendLang = !langPattern.matcher(lang).matches();
+        } else if (topLevelLang == null) {
           continue;
+        } else {
+          // Two-level -- the only way we won't append is if this second level matches exactly.
+          if (!langPattern.matcher(lang).matches() && !langPattern.matcher(topLevelLang).find()) {
+            continue;
+          }
+          appendLang = !langPattern.matcher(lang).matches();
         }
         
         String rest = line.substring(colonIndex + 1).trim();
         if (rest.length() > 0) {
-          doTranslationLine(line, title, sense, rest);
-        } else {
-          // TODO: do lines that are like Greek:
+          doTranslationLine(line, appendLang ? lang : null, title, pos, sense, rest);
         }
         
       } else if (wikiTokenizer.remainderStartsWith("''See''")) {
@@ -256,7 +274,7 @@ public class EnWiktionaryXmlParser {
     return index < list.size() ? list.get(index) : null;
   }
   
-  private void doTranslationLine(final String line, final String title, final String sense, final String rest) {
+  private void doTranslationLine(final String line, final String lang, final String title, final String pos, final String sense, final String rest) {
     // Good chance we'll actually file this one...
     final PairEntry pairEntry = new PairEntry();
     final IndexedEntry indexedEntry = new IndexedEntry(pairEntry);
@@ -304,12 +322,16 @@ public class EnWiktionaryXmlParser {
             }
           //}
         } else if (functionName.equals("qualifier")) {
-          String qualifier = args.get(0);
-          if (!namedArgs.isEmpty() || args.size() > 1) {
-            LOG.warning("weird qualifier: " + line);
+          if (args.size() == 0) {
+           otherText.append(wikiTokenizer.token()); 
+          } else { 
+            String qualifier = args.get(0);
+            if (!namedArgs.isEmpty() || args.size() > 1) {
+              LOG.warning("weird qualifier: " + line);
+            }
+            // Unindexed!
+            otherText.append("(").append(qualifier).append(")");
           }
-          // Unindexed!
-          otherText.append("(").append(qualifier).append(")");
         } else if (encodings.contains(functionName)) {
           otherText.append("").append(args.get(0));
           otherIndexBuilder.addEntryWithString(indexedEntry, args.get(0), EntryTypeName.WIKTIONARY_TRANSLATION_OTHER_TEXT);
@@ -357,6 +379,10 @@ public class EnWiktionaryXmlParser {
       return;
     }
     
+    if (lang != null) {
+      otherText.insert(0, "(" + lang + ") ");
+    }
+    
     StringBuilder englishText = new StringBuilder();
     
     englishText.append(title);
@@ -435,14 +461,17 @@ public class EnWiktionaryXmlParser {
   }
 
 
+  int foreignCount = 0;
   private void doForeignPartOfSpeech(String title, final String posHeading, final int posDepth, WikiTokenizer wikiTokenizer) {
-    LOG.info("***" + title + ", pos=" + posHeading);
+    if (++foreignCount % 1000 == 0) {
+      LOG.info("***" + title + ", pos=" + posHeading + ", foreignCount=" + foreignCount);
+    }
     if (title.equals("moro")) {
       System.out.println();
     }
     
     final StringBuilder foreignBuilder = new StringBuilder();
-    Collection<String> wordForms = Collections.emptyList();
+    final Collection<String> wordForms = new ArrayList<String>();
     final List<ListSection> listSections = new ArrayList<ListSection>();
     
     try {
@@ -461,7 +490,7 @@ public class EnWiktionaryXmlParser {
       }
       
       if (currentHeadingDepth > posDepth) {
-        // TODO
+        // TODO: deal with other neat info sections
         continue;
       }
       
@@ -495,13 +524,13 @@ public class EnWiktionaryXmlParser {
           //foreignBuilder.append(title);
         }
       } else if (name.equals("it-noun")) {
-          assert wordForms.isEmpty();
           final String base = get(args, 0);
           final String gender = get(args, 1);
           final String singular = base + get(args, 2);
           final String plural = base + get(args, 3);
-          foreignBuilder.append(String.format("%s {%s}, %s {pl}", singular, gender, plural, plural));
-          wordForms = Arrays.asList(singular, plural);
+          foreignBuilder.append(String.format(" %s {%s}, %s {pl}", singular, gender, plural, plural));
+          wordForms.add(singular);
+          wordForms.add(plural);
         } else if (name.equals("it-proper noun")) {
           foreignBuilder.append(wikiTokenizer.token());
         } else if (name.equals("it-adj")) {
@@ -550,12 +579,13 @@ public class EnWiktionaryXmlParser {
     
     } finally {
       // Here's where we exit.
-      // TODO: Should we make an entry even if there are no foreign list items?
-      if (foreignBuilder.indexOf(title) == -1) {
-        foreignBuilder.insert(0, title + " ");
+      // Should we make an entry even if there are no foreign list items?
+      String foreign = foreignBuilder.toString().trim();
+      if (!foreign.toLowerCase().startsWith(title.toLowerCase())) {
+        foreign = title + " " + foreign;
       }
       for (final ListSection listSection : listSections) {
-        doForeignListItem(foreignBuilder.toString(), title, wordForms, listSection);
+        doForeignListItem(foreign, title, wordForms, listSection);
       }
     }
   }
@@ -572,6 +602,7 @@ public class EnWiktionaryXmlParser {
     
     final String prefix = listSection.firstPrefix;
     if (prefix.length() > 1) {
+      // Could just get looser and say that any prefix longer than first is a sublist.
       LOG.warning("Prefix too long: " + listSection);
       return;
     }
@@ -599,9 +630,12 @@ public class EnWiktionaryXmlParser {
           } else if (link.contains("#") && this.langPattern.matcher(link).find()) {
             englishBuilder.append(text);
             otherIndexBuilder.addEntryWithString(indexedEntry, text, EntryTypeName.WIKTIONARY_ENGLISH_DEF_OTHER_LANG);
+          } else if (link.equals("plural")) {
+            englishBuilder.append(text);
           } else {
-            LOG.warning("Special link: " + englishTokenizer.token());
-            // TODO: something here...
+            //LOG.warning("Special link: " + englishTokenizer.token());
+            enIndexBuilder.addEntryWithString(indexedEntry, text, EntryTypeName.WIKTIONARY_ENGLISH_DEF_WIKI_LINK);
+            englishBuilder.append(text);
           }
         } else {
           // link == null
@@ -612,14 +646,15 @@ public class EnWiktionaryXmlParser {
         }
       } else if (englishTokenizer.isFunction()) {
         final String name = englishTokenizer.functionName();
-        if (name.contains(" conjugation of ") || 
-            name.contains(" form of ") || 
-            name.contains(" feminine of ") || 
-            name.contains(" plural of ")) {
+        if (name.contains("conjugation of ") || 
+            name.contains("form of ") || 
+            name.contains("feminine of ") || 
+            name.contains("plural of ")) {
           // Ignore these in the index, they're really annoying....
           englishBuilder.append(englishTokenizer.token());
         } else {
-          LOG.warning("Unexpected function: " + englishTokenizer.token());
+          englishBuilder.append(englishTokenizer.token());
+//          LOG.warning("Unexpected function: " + englishTokenizer.token());
         }
       } else {
         if (englishTokenizer.isComment() || englishTokenizer.isMarkup()) {
@@ -628,6 +663,7 @@ public class EnWiktionaryXmlParser {
         }
       }
     }
+        
     final String english = trim(englishBuilder.toString());
     if (english.length() > 0) {
       final Pair pair = new Pair(english, trim(foreignText), this.swap);
@@ -637,6 +673,97 @@ public class EnWiktionaryXmlParser {
         otherIndexBuilder.addEntryWithString(indexedEntry, form, EntryTypeName.WIKTIONARY_FORM_SINGLE, EntryTypeName.WIKTIONARY_FORM_MULTI);
       }
     }
+    
+    // Do examples.
+    String lastForeign = null;
+    for (int i = 0; i < listSection.nextPrefixes.size(); ++i) {
+      final String nextPrefix = listSection.nextPrefixes.get(i);
+      final String nextLine = listSection.nextLines.get(i);
+      int dash = nextLine.indexOf("&mdash;");
+      int mdashLen = 7;
+      if (dash == -1) {
+        dash = nextLine.indexOf("—");
+        mdashLen = 1;
+      }
+      if (dash == -1) {
+        dash = nextLine.indexOf(" - ");
+        mdashLen = 3;
+      }
+      
+      if ((nextPrefix.equals("#:") || nextPrefix.equals("##:")) && dash != -1) {
+        final String foreignEx = nextLine.substring(0, dash);
+        final String englishEx = nextLine.substring(dash + mdashLen);
+        final Pair pair = new Pair(formatAndIndexExampleString(englishEx, enIndexBuilder, indexedEntry), formatAndIndexExampleString(foreignEx, otherIndexBuilder, indexedEntry), swap);
+        if (pair.lang1 != "--" && pair.lang1 != "--") {
+          pairEntry.pairs.add(pair);
+        }
+        lastForeign = null;
+      } else if (nextPrefix.equals("#:") || nextPrefix.equals("##:")){
+        final Pair pair = new Pair("--", formatAndIndexExampleString(nextLine, null, indexedEntry), swap);
+        lastForeign = nextLine;
+        if (pair.lang1 != "--" && pair.lang1 != "--") {
+          pairEntry.pairs.add(pair);
+        }
+      } else if (nextPrefix.equals("#::") || nextPrefix.equals("#**")) {
+        if (lastForeign != null && pairEntry.pairs.size() > 0) {
+          pairEntry.pairs.remove(pairEntry.pairs.size() - 1);
+          final Pair pair = new Pair(formatAndIndexExampleString(nextLine, enIndexBuilder, indexedEntry), formatAndIndexExampleString(lastForeign, otherIndexBuilder, indexedEntry), swap);
+          if (pair.lang1 != "--" && pair.lang1 != "--") {
+            pairEntry.pairs.add(pair);
+          }
+          lastForeign = null;
+        } else {
+          LOG.warning("TODO: English example with no foreign: " + title + ", " + nextLine);
+          // TODO: add something.
+        }
+      } else if (nextPrefix.equals("#*")) {
+        // Can't really index these.
+        final Pair pair = new Pair("--", formatAndIndexExampleString(nextLine, null, indexedEntry), swap);
+        lastForeign = nextLine;
+        if (pair.lang1 != "--" && pair.lang1 != "--") {
+          pairEntry.pairs.add(pair);
+        }
+      } else if (nextPrefix.equals("#::*") || nextPrefix.equals("##") || nextPrefix.equals("#*:") || nextPrefix.equals("#:*") || true) {
+        final Pair pair = new Pair("--", formatAndIndexExampleString(nextLine, null, indexedEntry), swap);
+        if (pair.lang1 != "--" && pair.lang1 != "--") {
+          pairEntry.pairs.add(pair);
+        }
+//      } else {
+//        assert false;
+      }
+    }
+  }
+  
+  private String formatAndIndexExampleString(final String example, final IndexBuilder indexBuilder, final IndexedEntry indexedEntry) {
+    final WikiTokenizer wikiTokenizer = new WikiTokenizer(example, false);
+    final StringBuilder builder = new StringBuilder();
+    boolean insideTripleQuotes = false;
+    while (wikiTokenizer.nextToken() != null) {
+      if (wikiTokenizer.isPlainText()) {
+        builder.append(wikiTokenizer.token());
+        if (indexBuilder != null) {
+          indexBuilder.addEntryWithString(indexedEntry, wikiTokenizer.token(), EntryTypeName.WIKTIONARY_EXAMPLE);
+        }
+      } else if (wikiTokenizer.isWikiLink()) {
+        final String text = wikiTokenizer.wikiLinkText().replaceAll("'", ""); 
+        builder.append(text);
+        if (indexBuilder != null) {
+          indexBuilder.addEntryWithString(indexedEntry, text, EntryTypeName.WIKTIONARY_EXAMPLE);
+        }
+      } else if (wikiTokenizer.isFunction()) {
+        builder.append(wikiTokenizer.token());
+      } else if (wikiTokenizer.isMarkup()) {
+        if (wikiTokenizer.token().equals("'''")) {
+          insideTripleQuotes = !insideTripleQuotes;
+        }
+      } else if (wikiTokenizer.isComment() || wikiTokenizer.isNewline()) {
+        // Do nothing.
+      } else {
+        LOG.warning("unexpected token: " + wikiTokenizer.token());
+      }
+    }
+    final String result = trim(builder.toString());
+    return result.length() > 0 ? result : "--";
   }