zipSize, overrideStoplist-> special isMainEntry, tagalog, trying to

[DictionaryPC.git] / src / com / hughes / android / dictionary / parser / enwiktionary / EnWiktionaryXmlParser.java

diff --git a/src/com/hughes/android/dictionary/parser/enwiktionary/EnWiktionaryXmlParser.java b/src/com/hughes/android/dictionary/parser/enwiktionary/EnWiktionaryXmlParser.java
index 1b23270d1f3ac11c5fecd0502f547a4c8df23e30..62aa430a4f09840172f4c14d4279015e8d3f3617 100644 (file)
--- a/src/com/hughes/android/dictionary/parser/enwiktionary/EnWiktionaryXmlParser.java
+++ b/src/com/hughes/android/dictionary/parser/enwiktionary/EnWiktionaryXmlParser.java
@@ -27,21 +27,20 @@ import java.util.LinkedHashSet;
 import java.util.List;
 import java.util.Map;
 import java.util.Set;
+import java.util.logging.Level;
 import java.util.logging.Logger;
 import java.util.regex.Pattern;
 
+import com.hughes.android.dictionary.engine.EntrySource;
 import com.hughes.android.dictionary.engine.EntryTypeName;
 import com.hughes.android.dictionary.engine.IndexBuilder;
 import com.hughes.android.dictionary.engine.IndexedEntry;
 import com.hughes.android.dictionary.engine.PairEntry;
 import com.hughes.android.dictionary.engine.PairEntry.Pair;
 import com.hughes.android.dictionary.parser.WikiTokenizer;
-import com.hughes.util.ListUtil;
 
 public class EnWiktionaryXmlParser {
   
-  private static final String TRANSLITERATION_FORMAT = " (tr. %s)";
-
   static final Logger LOG = Logger.getLogger(EnWiktionaryXmlParser.class.getName());
   
   // TODO: process {{ttbc}} lines
@@ -51,6 +50,7 @@ public class EnWiktionaryXmlParser {
       "Preposition|Proper noun|Article|Prepositional phrase|Acronym|" +
       "Abbreviation|Initialism|Contraction|Prefix|Suffix|Symbol|Letter|" +
       "Ligature|Idiom|Phrase|\\{\\{acronym\\}\\}|\\{\\{initialism\\}\\}|" +
+      "\\{\\{abbreviation\\}\\}|" +
       // These are @deprecated:
       "Noun form|Verb form|Adjective form|Nominal phrase|Noun phrase|" +
       "Verb phrase|Transitive verb|Intransitive verb|Reflexive verb|" +
@@ -59,6 +59,7 @@ public class EnWiktionaryXmlParser {
       "Particle|Interjection|Pronominal adverb" +
       "Han character|Hanzi|Hanja|Kanji|Katakana character|Syllable");
   
+  EntrySource entrySource;
   final IndexBuilder enIndexBuilder;
   final IndexBuilder foreignIndexBuilder;
   final Pattern langPattern;
@@ -84,9 +85,11 @@ public class EnWiktionaryXmlParser {
   }
 
   
-  public void parse(final File file, final int pageLimit) throws IOException {
+  public void parse(final File file, final EntrySource entrySource, final int pageLimit) throws IOException {
+    this.entrySource = entrySource;
     int pageCount = 0;
     final DataInputStream dis = new DataInputStream(new BufferedInputStream(new FileInputStream(file)));
+    try {
     while (true) {
       if (pageLimit >= 0 && pageCount >= pageLimit) {
         return;
@@ -95,7 +98,7 @@ public class EnWiktionaryXmlParser {
       try {
         title = dis.readUTF();
       } catch (EOFException e) {
-        LOG.warning("Error reading split!");
+        LOG.log(Level.INFO, "EOF reading split.");
         dis.close();
         return;
       }
@@ -112,6 +115,11 @@ public class EnWiktionaryXmlParser {
         LOG.info("pageCount=" + pageCount);
       }
     }
+    } finally {
+      System.out.println("lang Counts: " + appendAndIndexWikiCallback.langCodeToTCount);
+      appendAndIndexWikiCallback.langCodeToTCount.keySet().removeAll(EnWiktionaryLangs.isoCodeToWikiName.keySet());
+      System.out.println("unused Counts: " + appendAndIndexWikiCallback.langCodeToTCount);
+    }
   }
   
   private void parseSection(String heading, final String text) {
@@ -162,7 +170,7 @@ public class EnWiktionaryXmlParser {
           
         } else if (headerName.equals("Translations")) {
           if (pos == null) {
-            LOG.warning("Translations without POS: " + title);
+            LOG.info("Translations without POS (but using anyway): " + title);
           }
           doTranslations(wikiTokenizer, pos);
         } else if (headerName.equals("Pronunciation")) {
@@ -170,7 +178,7 @@ public class EnWiktionaryXmlParser {
         }
       } else if (wikiTokenizer.isFunction()) {
         final String name = wikiTokenizer.functionName();
-        if (name.equals("head")) {
+        if (name.equals("head") && pos == null) {
           LOG.warning("{{head}} without POS: " + title);
         }
       }
@@ -292,7 +300,7 @@ public class EnWiktionaryXmlParser {
   private void doTranslationLine(final String line, final String lang, final String pos, final String sense, final String rest) {
     state = State.TRANSLATION_LINE;
     // Good chance we'll actually file this one...
-    final PairEntry pairEntry = new PairEntry();
+    final PairEntry pairEntry = new PairEntry(entrySource);
     final IndexedEntry indexedEntry = new IndexedEntry(pairEntry);
     
     final StringBuilder foreignText = new StringBuilder();
@@ -325,10 +333,6 @@ public class EnWiktionaryXmlParser {
     if (!pairsAdded.add(pair.toString())) {
       LOG.warning("Duplicate pair: " + pair.toString());
     }
-    if (pair.toString().equals("libero {m} :: free (adjective)")) {
-      System.out.println();
-    }
-
   }
 
 
@@ -375,7 +379,6 @@ public class EnWiktionaryXmlParser {
   final Collection<String> wordForms = new ArrayList<String>();
   boolean titleAppended = false;
 
-
   private void doForeignPartOfSpeech(final String lang, String posHeading, final int posDepth, WikiTokenizer wikiTokenizer) {
     if (++foreignCount % 1000 == 0) {
       LOG.info("***" + lang + ", " + title + ", pos=" + posHeading + ", foreignCount=" + foreignCount);
@@ -471,7 +474,7 @@ public class EnWiktionaryXmlParser {
         foreign = String.format("(%s) %s", lang, foreign);
       }
       for (final ListSection listSection : listSections) {
-        doForeignListItem(foreign, title, wordForms, listSection);
+        doForeignListSection(foreign, title, wordForms, listSection);
       }
     }
   }
@@ -484,9 +487,12 @@ public class EnWiktionaryXmlParser {
           "sc",
           "sort",
           "cat",
-          "xs"));
+          "xs",
+          "nodot"));
+
+  public boolean entryIsFormOfSomething = false;
 
-  private void doForeignListItem(final String foreignText, String title, final Collection<String> forms, final ListSection listSection) {
+  private void doForeignListSection(final String foreignText, String title, final Collection<String> forms, final ListSection listSection) {
     state = State.ENGLISH_DEF_OF_FOREIGN;
     final String prefix = listSection.firstPrefix;
     if (prefix.length() > 1) {
@@ -495,12 +501,12 @@ public class EnWiktionaryXmlParser {
       return;
     }
     
-    final PairEntry pairEntry = new PairEntry();
+    final PairEntry pairEntry = new PairEntry(entrySource);
     final IndexedEntry indexedEntry = new IndexedEntry(pairEntry);
-    
+
+    entryIsFormOfSomething = false;
     final StringBuilder englishBuilder = new StringBuilder();
     final String mainLine = listSection.firstLine;
-
     appendAndIndexWikiCallback.reset(englishBuilder, indexedEntry);
     appendAndIndexWikiCallback.dispatch(mainLine, enIndexBuilder, EntryTypeName.WIKTIONARY_ENGLISH_DEF);
 
@@ -508,7 +514,7 @@ public class EnWiktionaryXmlParser {
     if (english.length() > 0) {
       final Pair pair = new Pair(english, trim(foreignText), this.swap);
       pairEntry.pairs.add(pair);
-      foreignIndexBuilder.addEntryWithString(indexedEntry, title, EntryTypeName.WIKTIONARY_TITLE_MULTI);
+      foreignIndexBuilder.addEntryWithString(indexedEntry, title, entryIsFormOfSomething ? EntryTypeName.WIKTIONARY_IS_FORM_OF_SOMETHING_ELSE : EntryTypeName.WIKTIONARY_TITLE_MULTI);
       for (final String form : forms) {
         foreignIndexBuilder.addEntryWithString(indexedEntry, form, EntryTypeName.WIKTIONARY_INFLECTED_FORM_MULTI);
       }
@@ -519,6 +525,8 @@ public class EnWiktionaryXmlParser {
     for (int i = 0; i < listSection.nextPrefixes.size(); ++i) {
       final String nextPrefix = listSection.nextPrefixes.get(i);
       final String nextLine = listSection.nextLines.get(i);
+
+      // TODO: This splitting is not sensitive to wiki code.
       int dash = nextLine.indexOf("&mdash;");
       int mdashLen = 7;
       if (dash == -1) {
@@ -578,32 +586,19 @@ public class EnWiktionaryXmlParser {
   }
   
   private String formatAndIndexExampleString(final String example, final IndexBuilder indexBuilder, final IndexedEntry indexedEntry) {
-    final WikiTokenizer wikiTokenizer = new WikiTokenizer(example, false);
+    // TODO:
+//    if (wikiTokenizer.token().equals("'''")) {
+//      insideTripleQuotes = !insideTripleQuotes;
+//    }
     final StringBuilder builder = new StringBuilder();
-    boolean insideTripleQuotes = false;
-    while (wikiTokenizer.nextToken() != null) {
-      if (wikiTokenizer.isPlainText()) {
-        builder.append(wikiTokenizer.token());
-        if (indexBuilder != null) {
-          indexBuilder.addEntryWithStringNoSingle(indexedEntry, wikiTokenizer.token(), EntryTypeName.WIKTIONARY_EXAMPLE);
-        }
-      } else if (wikiTokenizer.isWikiLink()) {
-        final String text = wikiTokenizer.wikiLinkText().replaceAll("'", ""); 
-        builder.append(text);
-        if (indexBuilder != null) {
-          indexBuilder.addEntryWithStringNoSingle(indexedEntry, text, EntryTypeName.WIKTIONARY_EXAMPLE);
-        }
-      } else if (wikiTokenizer.isFunction()) {
-        builder.append(wikiTokenizer.token());
-      } else if (wikiTokenizer.isMarkup()) {
-        if (wikiTokenizer.token().equals("'''")) {
-          insideTripleQuotes = !insideTripleQuotes;
-        }
-      } else if (wikiTokenizer.isComment() || wikiTokenizer.isNewline()) {
-        // Do nothing.
-      } else {
-        LOG.warning("unexpected token: " + wikiTokenizer.token());
-      }
+    appendAndIndexWikiCallback.reset(builder, indexedEntry);
+    appendAndIndexWikiCallback.entryTypeName = EntryTypeName.WIKTIONARY_EXAMPLE;
+    appendAndIndexWikiCallback.entryTypeNameSticks = true;
+    try {
+      // TODO: this is a hack needed because we don't safely split on the dash.
+      appendAndIndexWikiCallback.dispatch(example, indexBuilder, EntryTypeName.WIKTIONARY_EXAMPLE);
+    } catch (AssertionError e) {
+      return "--";
     }
     final String result = trim(builder.toString());
     return result.length() > 0 ? result : "--";