Update to latest wiktionaries, update unit tests, der-top/mid/bottom.

[DictionaryPC.git] / src / com / hughes / android / dictionary / engine / WiktionarySplitter.java

diff --git a/src/com/hughes/android/dictionary/engine/WiktionarySplitter.java b/src/com/hughes/android/dictionary/engine/WiktionarySplitter.java
index d0423b3bd95b4880117429c06ec9938be588de13..361473e80ac3e0043b9191fb1098ab3a1298c42f 100644 (file)
--- a/src/com/hughes/android/dictionary/engine/WiktionarySplitter.java
+++ b/src/com/hughes/android/dictionary/engine/WiktionarySplitter.java
@@ -28,12 +28,12 @@ import java.util.regex.Pattern;
 
 import javax.xml.parsers.ParserConfigurationException;
 import javax.xml.parsers.SAXParser;
-import javax.xml.parsers.SAXParserFactory;
 
+import org.apache.xerces.jaxp.SAXParserFactoryImpl;
 import org.xml.sax.Attributes;
 import org.xml.sax.SAXException;
 
-import com.hughes.android.dictionary.parser.enwiktionary.WiktionaryLangs;
+import com.hughes.android.dictionary.parser.wiktionary.WiktionaryLangs;
 
 public class WiktionarySplitter extends org.xml.sax.helpers.DefaultHandler {
 
@@ -48,7 +48,7 @@ public class WiktionarySplitter extends org.xml.sax.helpers.DefaultHandler {
   StringBuilder textBuilder;
   StringBuilder currentBuilder = null;
 
-  public static void main(final String[] args) throws SAXException, IOException, ParserConfigurationException {
+  public static void main(final String[] args) throws Exception {
     final WiktionarySplitter wiktionarySplitter = new WiktionarySplitter();
     wiktionarySplitter.go();
   }
@@ -56,7 +56,7 @@ public class WiktionarySplitter extends org.xml.sax.helpers.DefaultHandler {
   private WiktionarySplitter() {
     List<Selector> selectors;
     for (final String code : WiktionaryLangs.wikiCodeToIsoCodeToWikiName.keySet()) {
-      //if (!code.equals("fr")) {continue;}
+      //if (code.equals("en") || code.equals("de") || code.equals("fr")) {continue;}
       selectors = new ArrayList<WiktionarySplitter.Selector>();
       pathToSelectors.put(String.format("data/inputs/%swiktionary-pages-articles.xml", code), selectors);
       for (final Map.Entry<String, String> entry : WiktionaryLangs.wikiCodeToIsoCodeToWikiName.get(code).entrySet()) {
@@ -67,8 +67,8 @@ public class WiktionarySplitter extends org.xml.sax.helpers.DefaultHandler {
     }
   }
 
-  private void go() throws ParserConfigurationException, SAXException, IOException {
-    final SAXParser parser = SAXParserFactory.newInstance().newSAXParser();
+  private void go() throws Exception {
+    final SAXParser parser = SAXParserFactoryImpl.newInstance().newSAXParser();
 
     // Configure things.
     for (final Map.Entry<String, List<Selector>> pathToSelectorsEntry : pathToSelectors.entrySet()) {
@@ -80,7 +80,12 @@ public class WiktionarySplitter extends org.xml.sax.helpers.DefaultHandler {
       }
   
       // Do it.
-      parser.parse(new File(pathToSelectorsEntry.getKey()), this);
+      try {
+        parser.parse(new File(pathToSelectorsEntry.getKey()), this);
+      } catch (Exception e) {
+        System.err.println("Exception during parse, lastPageTitle=" + lastPageTitle + ", titleBuilder=" + titleBuilder.toString());
+        throw e;
+      }
       
       // Shutdown.
       for (final Selector selector : currentSelectors) {
@@ -90,12 +95,68 @@ public class WiktionarySplitter extends org.xml.sax.helpers.DefaultHandler {
     }
   }
 
+  String lastPageTitle = null;
   int pageCount = 0;
   private void endPage() {
     final String title = titleBuilder.toString();
+    lastPageTitle = title;
     if (++pageCount % 1000 == 0) {
       System.out.println("endPage: " + title + ", count=" + pageCount);
     }
+    if (title.startsWith("Wiktionary:") || 
+            title.startsWith("Appendix:") || 
+            title.startsWith("Help:") ||
+            title.startsWith("Index:") ||
+            title.startsWith("MediaWiki:") || 
+            title.startsWith("Citations:") || 
+            title.startsWith("Concordance:") || 
+            title.startsWith("Glossary:") || 
+            title.startsWith("Rhymes:") || 
+            title.startsWith("Category:") || 
+            title.startsWith("Wikisaurus:") || 
+            title.startsWith("Unsupported titles/") || 
+            title.startsWith("Transwiki:") || 
+            title.startsWith("File:") || 
+            title.startsWith("Thread:") || 
+            title.startsWith("Template:") ||
+            title.startsWith("Summary:") ||
+            // DE
+            title.startsWith("Datei:") ||
+            title.startsWith("Verzeichnis:") ||
+            title.startsWith("Vorlage:") ||
+            title.startsWith("Thesaurus:") ||
+            title.startsWith("Kategorie:") ||
+            title.startsWith("Hilfe:") ||
+            // FR:
+            title.startsWith("Annexe:") ||
+            title.startsWith("Catégori:") ||
+            title.startsWith("Modèle:") ||
+            title.startsWith("Thésaurus:") ||
+            title.startsWith("Projet:") ||
+            title.startsWith("Aide:") ||
+            title.startsWith("Fichier:") ||
+            title.startsWith("Wiktionnaire:") ||
+            title.startsWith("Catégorie:") ||
+            title.startsWith("Portail:") ||
+            title.startsWith("utiliusateur:") ||
+            title.startsWith("Kategorio:") ||
+            // IT
+            title.startsWith("Wikizionario:") ||
+            title.startsWith("Appendice:") ||
+            title.startsWith("Categoria:") ||
+            title.startsWith("Aiuto:") ||
+            title.startsWith("Portail:") ||
+
+            // sentinel
+            false
+            ) {
+        return;
+    }
+    if (title.contains(":")) {
+        if (!title.startsWith("Sign gloss:")) {
+            System.err.println("title with colon: " + title);
+        }
+    }
     
     String text = textBuilder.toString();
     
@@ -207,13 +268,10 @@ public class WiktionarySplitter extends org.xml.sax.helpers.DefaultHandler {
       }
     }
     
-
     public void parse(final File file) throws ParserConfigurationException,
         SAXException, IOException {
-      final SAXParser parser = SAXParserFactory.newInstance().newSAXParser();
+      final SAXParser parser = SAXParserFactoryImpl.newInstance().newSAXParser();
       parser.parse(file, this);
     }
-
-    
     
 }