X-Git-Url: http://gitweb.fperrin.net/?a=blobdiff_plain;f=src%2Fcom%2Fhughes%2Fandroid%2Fdictionary%2Fengine%2FWiktionarySplitter.java;h=cc9a3b1a0bc6ee56110dfded92cec93a50b83301;hb=b7b04d01f8d0ed763f0817d0531ecebf9ff50260;hp=056fabe44b0f9e5a050cfdd7a9e70c3f32ca98e6;hpb=349d140dce670c45be4032d7c7286956a389a2b5;p=DictionaryPC.git

diff --git a/src/com/hughes/android/dictionary/engine/WiktionarySplitter.java b/src/com/hughes/android/dictionary/engine/WiktionarySplitter.java
index 056fabe..cc9a3b1 100644
--- a/src/com/hughes/android/dictionary/engine/WiktionarySplitter.java
+++ b/src/com/hughes/android/dictionary/engine/WiktionarySplitter.java
@@ -20,8 +20,8 @@ import java.io.DataOutputStream;
 import java.io.File;
 import java.io.FileInputStream;
 import java.io.FileOutputStream;
-import java.io.InputStream;
 import java.io.IOException;
+import java.io.InputStream;
 import java.io.OutputStream;
 import java.util.ArrayList;
 import java.util.LinkedHashMap;
@@ -32,8 +32,8 @@ import java.util.regex.Pattern;
 
 import javax.xml.parsers.ParserConfigurationException;
 import javax.xml.parsers.SAXParser;
+import javax.xml.parsers.SAXParserFactory;
 
-import org.apache.xerces.jaxp.SAXParserFactoryImpl;
 import org.apache.commons.compress.compressors.CompressorStreamFactory;
 import org.xml.sax.Attributes;
 import org.xml.sax.SAXException;
@@ -75,7 +75,7 @@ public class WiktionarySplitter extends org.xml.sax.helpers.DefaultHandler {
     }
 
     private void go() throws Exception {
-        final SAXParser parser = SAXParserFactoryImpl.newInstance().newSAXParser();
+        final SAXParser parser = SAXParserFactory.newInstance().newSAXParser();
 
         // Configure things.
         for (final Map.Entry<String, List<Selector>> pathToSelectorsEntry : pathToSelectors.entrySet()) {
@@ -152,6 +152,7 @@ public class WiktionarySplitter extends org.xml.sax.helpers.DefaultHandler {
                 title.startsWith("Template:") ||
                 title.startsWith("Summary:") ||
                 title.startsWith("Module:") ||
+                title.startsWith("Reconstruction:") ||
                 // DE
                 title.startsWith("Datei:") ||
                 title.startsWith("Verzeichnis:") ||
@@ -160,6 +161,7 @@ public class WiktionarySplitter extends org.xml.sax.helpers.DefaultHandler {
                 title.startsWith("Kategorie:") ||
                 title.startsWith("Hilfe:") ||
                 title.startsWith("Reim:") ||
+                title.startsWith("Modul:") ||
                 // FR:
                 title.startsWith("Annexe:") ||
                 title.startsWith("Catégori:") ||
@@ -169,16 +171,20 @@ public class WiktionarySplitter extends org.xml.sax.helpers.DefaultHandler {
                 title.startsWith("Aide:") ||
                 title.startsWith("Fichier:") ||
                 title.startsWith("Wiktionnaire:") ||
+                title.startsWith("Translations:Wiktionnaire:") ||
+                title.startsWith("Translations:Projet:") ||
                 title.startsWith("Catégorie:") ||
                 title.startsWith("Portail:") ||
                 title.startsWith("utiliusateur:") ||
                 title.startsWith("Kategorio:") ||
+                title.startsWith("Tutoriel:") ||
                 // IT
                 title.startsWith("Wikizionario:") ||
                 title.startsWith("Appendice:") ||
                 title.startsWith("Categoria:") ||
                 title.startsWith("Aiuto:") ||
                 title.startsWith("Portail:") ||
+                title.startsWith("Modulo:") ||
                 // ES
                 title.startsWith("Apéndice:") ||
                 title.startsWith("Archivo:") ||
@@ -187,15 +193,28 @@ public class WiktionarySplitter extends org.xml.sax.helpers.DefaultHandler {
                 title.startsWith("Plantilla:") ||
                 title.startsWith("Wikcionario:") ||
 
+                // PT
+                title.startsWith("Ajuda:") ||
+                title.startsWith("Apêndice:") ||
+                title.startsWith("Citações:") ||
+                title.startsWith("Portal:") ||
+                title.startsWith("Predefinição:") ||
+                title.startsWith("Vocabulário:") ||
+                title.startsWith("Wikcionário:") ||
+                title.startsWith("Módulo:") ||
+
                 // sentinel
                 false
                ) return;
-            if (!title.startsWith("Sign gloss:")) {
+            // leave the Flexion: pages in for now and do not warn about them
+            if (!title.startsWith("Sign gloss:") && !title.startsWith("Flexion:")) {
                 System.err.println("title with colon: " + title);
             }
         }
 
         String text = textBuilder.toString();
+        // Workaround for Spanish wiktionary {{ES}} and {{ES|word}} patterns
+        text = text.replaceAll("\\{\\{ES(\\|[^{}=]*)?}}", "== {{lengua|es}} ==");
         String translingual = "";
         int start = 0;
         final Matcher startMatcher = headingStart.matcher(text);
@@ -335,7 +354,7 @@ public class WiktionarySplitter extends org.xml.sax.helpers.DefaultHandler {
 
     public void parse(final File file) throws ParserConfigurationException,
         SAXException, IOException {
-        final SAXParser parser = SAXParserFactoryImpl.newInstance().newSAXParser();
+        final SAXParser parser = SAXParserFactory.newInstance().newSAXParser();
         parser.parse(file, this);
     }