X-Git-Url: http://gitweb.fperrin.net/?a=blobdiff_plain;f=src%2Fcom%2Fhughes%2Fandroid%2Fdictionary%2Fengine%2FWiktionarySplitter.java;h=408ecd9de3c539f36cedeb366cbb1ab2c21f6c38;hb=511f5633b8f21a7929b0658e06245bb5092a313d;hp=2e732f03e79628944c17d88867bf25c2429a49db;hpb=52123581b0c4aa46298b9d6cbc4697accffc1cc7;p=DictionaryPC.git

diff --git a/src/com/hughes/android/dictionary/engine/WiktionarySplitter.java b/src/com/hughes/android/dictionary/engine/WiktionarySplitter.java
index 2e732f0..408ecd9 100644
--- a/src/com/hughes/android/dictionary/engine/WiktionarySplitter.java
+++ b/src/com/hughes/android/dictionary/engine/WiktionarySplitter.java
@@ -20,6 +20,7 @@ import java.io.File;
 import java.io.FileOutputStream;
 import java.io.IOException;
 import java.util.ArrayList;
+import java.util.LinkedHashMap;
 import java.util.List;
 import java.util.Map;
 import java.util.regex.Matcher;
@@ -27,81 +28,135 @@ import java.util.regex.Pattern;
 
 import javax.xml.parsers.ParserConfigurationException;
 import javax.xml.parsers.SAXParser;
-import javax.xml.parsers.SAXParserFactory;
 
+import org.apache.xerces.jaxp.SAXParserFactoryImpl;
 import org.xml.sax.Attributes;
 import org.xml.sax.SAXException;
 
+import com.hughes.android.dictionary.parser.wiktionary.WiktionaryLangs;
+
 public class WiktionarySplitter extends org.xml.sax.helpers.DefaultHandler {
+
+  // The matches the whole line, otherwise regexes don't work well on French:
+  // {{=uk=}}
+  static final Pattern headingStart = Pattern.compile("^(=+)[^=].*$", Pattern.MULTILINE);
   
-  private static final String FILE_TO_SPLIT = "data/inputs/enwiktionary-20111224-pages-articles.xml";
+  final Map<String,List<Selector>> pathToSelectors = new LinkedHashMap<String, List<Selector>>();
+  List<Selector> currentSelectors = null;
   
-  static class Section implements java.io.Serializable {
-    private static final long serialVersionUID = -7676549898325856822L;
+  StringBuilder titleBuilder;
+  StringBuilder textBuilder;
+  StringBuilder currentBuilder = null;
 
-    final String title;
-    final String heading;
-    final String text;
-    
-    public Section(final String title, final String heading, final String text) {
-      this.title = title;
-      this.heading = heading;
-      this.text = text;
-      
-      //System.out.printf("TITLE:%s\nHEADING:%s\nTEXT:%s\n\n\n\n\n\n", title, heading, text);
-    }
+  public static void main(final String[] args) throws Exception {
+    final WiktionarySplitter wiktionarySplitter = new WiktionarySplitter();
+    wiktionarySplitter.go();
   }
   
-  static class Selector {
-    DataOutputStream out;
-    Pattern pattern;
-    
-    public Selector(final String filename, final String pattern) throws IOException {
-      this.out = new DataOutputStream(new BufferedOutputStream(new FileOutputStream(filename)));
-      this.pattern = Pattern.compile(pattern, Pattern.CASE_INSENSITIVE);
+  private WiktionarySplitter() {
+    List<Selector> selectors;
+    for (final String code : WiktionaryLangs.wikiCodeToIsoCodeToWikiName.keySet()) {
+      //if (!code.equals("fr")) {continue;}
+      selectors = new ArrayList<WiktionarySplitter.Selector>();
+      pathToSelectors.put(String.format("data/inputs/%swiktionary-pages-articles.xml", code), selectors);
+      for (final Map.Entry<String, String> entry : WiktionaryLangs.wikiCodeToIsoCodeToWikiName.get(code).entrySet()) {
+        final String dir = String.format("data/inputs/wikiSplit/%s", code);
+        new File(dir).mkdirs();
+        selectors.add(new Selector(String.format("%s/%s.data", dir, entry.getKey()), entry.getValue()));
+      }
     }
   }
 
-  final List<Selector> selectors = new ArrayList<Selector>();
-  StringBuilder titleBuilder;
-  StringBuilder textBuilder;
-  StringBuilder currentBuilder = null;
+  private void go() throws Exception {
+    final SAXParser parser = SAXParserFactoryImpl.newInstance().newSAXParser();
 
-  public static void main(final String[] args) throws SAXException, IOException, ParserConfigurationException {
-    final SAXParser parser = SAXParserFactory.newInstance().newSAXParser();
-    final WiktionarySplitter wiktionarySplitter = new WiktionarySplitter();
-    
     // Configure things.
-    
-    final List<Selector> selectors = wiktionarySplitter.selectors;
-    for (int i = 1; i < args.length; i += 2) {
-      final Selector selector = new Selector(args[i], args[i+1]);
-      selectors.add(selector);
-    }
-
-    if (selectors.isEmpty()) {
-      for (final Map.Entry<String, String> entry : Language.isoCodeToWikiName.entrySet()) {
-        selectors.add(new Selector(String.format("data/inputs/enWikiSplit/%s.data", entry.getKey()), entry.getValue()));
+    for (final Map.Entry<String, List<Selector>> pathToSelectorsEntry : pathToSelectors.entrySet()) {
+      
+      currentSelectors = pathToSelectorsEntry.getValue();
+      
+      for (final Selector selector : currentSelectors) {
+        selector.out = new DataOutputStream(new BufferedOutputStream(new FileOutputStream(selector.outFilename)));
       }
-    }
-    
-    // Do it.
-    parser.parse(new File(FILE_TO_SPLIT), wiktionarySplitter);
-    
-    // Shutdown.
-    for (final Selector selector : selectors) {
-      selector.out.close();
+  
+      // Do it.
+      try {
+        parser.parse(new File(pathToSelectorsEntry.getKey()), this);
+      } catch (Exception e) {
+        System.err.println("Exception during parse, lastPageTitle=" + lastPageTitle + ", titleBuilder=" + titleBuilder.toString());
+        throw e;
+      }
+      
+      // Shutdown.
+      for (final Selector selector : currentSelectors) {
+        selector.out.close();
+      }
+      
     }
   }
 
-  static final Pattern headingStart = Pattern.compile("^(=+)[^=]+=+", Pattern.MULTILINE);
-  
+  String lastPageTitle = null;
   int pageCount = 0;
   private void endPage() {
     final String title = titleBuilder.toString();
+    lastPageTitle = title;
     if (++pageCount % 1000 == 0) {
       System.out.println("endPage: " + title + ", count=" + pageCount);
     }
+    if (title.startsWith("Wiktionary:") || 
+            title.startsWith("Appendix:") || 
+            title.startsWith("Help:") ||
+            title.startsWith("Index:") ||
+            title.startsWith("MediaWiki:") || 
+            title.startsWith("Citations:") || 
+            title.startsWith("Concordance:") || 
+            title.startsWith("Glossary:") || 
+            title.startsWith("Rhymes:") || 
+            title.startsWith("Category:") || 
+            title.startsWith("Wikisaurus:") || 
+            title.startsWith("Unsupported titles/") || 
+            title.startsWith("Transwiki:") || 
+            title.startsWith("File:") || 
+            title.startsWith("Thread:") || 
+            title.startsWith("Template:") ||
+            title.startsWith("Summary:") ||
+            // DE
+            title.startsWith("Datei:") ||
+            title.startsWith("Verzeichnis:") ||
+            title.startsWith("Vorlage:") ||
+            title.startsWith("Thesaurus:") ||
+            title.startsWith("Kategorie:") ||
+            title.startsWith("Hilfe:") ||
+            // FR:
+            title.startsWith("Annexe:") ||
+            title.startsWith("Catégori:") ||
+            title.startsWith("Modèle:") ||
+            title.startsWith("Thésaurus:") ||
+            title.startsWith("Projet:") ||
+            title.startsWith("Aide:") ||
+            title.startsWith("Fichier:") ||
+            title.startsWith("Wiktionnaire:") ||
+            title.startsWith("Catégorie:") ||
+            title.startsWith("Portail:") ||
+            title.startsWith("utiliusateur:") ||
+            title.startsWith("Kategorio:") ||
+            // IT
+            title.startsWith("Wikizionario:") ||
+            title.startsWith("Appendice:") ||
+            title.startsWith("Categoria:") ||
+            title.startsWith("Aiuto:") ||
+            title.startsWith("Portail:") ||
+
+            // sentinel
+            false
+            ) {
+        return;
+    }
+    if (title.contains(":")) {
+        if (!title.startsWith("Sign gloss:")) {
+            System.err.println("title with colon: " + title);
+        }
+    }
     
     String text = textBuilder.toString();
     
@@ -114,12 +169,12 @@ public class WiktionarySplitter extends org.xml.sax.helpers.DefaultHandler {
       text = text.substring(startMatcher.end());
       
       final String heading = startMatcher.group();
-      for (final Selector selector : selectors) {
+      for (final Selector selector : currentSelectors) {
         if (selector.pattern.matcher(heading).find()) {
           
           // Find end.
           final int depth = startMatcher.group(1).length();
-          final Pattern endPattern = Pattern.compile(String.format("^={1,%d}[^=]+=+", depth), Pattern.MULTILINE);
+          final Pattern endPattern = Pattern.compile(String.format("^={1,%d}[^=].*$", depth), Pattern.MULTILINE);
           
           final Matcher endMatcher = endPattern.matcher(text);
           final int end;
@@ -149,6 +204,36 @@ public class WiktionarySplitter extends org.xml.sax.helpers.DefaultHandler {
     
   }
 
+  // -----------------------------------------------------------------------
+
+  static class Section implements java.io.Serializable {
+    private static final long serialVersionUID = -7676549898325856822L;
+
+    final String title;
+    final String heading;
+    final String text;
+    
+    public Section(final String title, final String heading, final String text) {
+      this.title = title;
+      this.heading = heading;
+      this.text = text;
+      
+      //System.out.printf("TITLE:%s\nHEADING:%s\nTEXT:%s\n\n\n\n\n\n", title, heading, text);
+    }
+  }
+  
+  static class Selector {
+    final String outFilename;
+    final Pattern pattern;
+
+    DataOutputStream out;
+
+    public Selector(final String filename, final String pattern) {
+      this.outFilename = filename;
+      this.pattern = Pattern.compile(pattern, Pattern.CASE_INSENSITIVE);
+    }
+  }
+
   // -----------------------------------------------------------------------
   
     @Override
@@ -183,10 +268,9 @@ public class WiktionarySplitter extends org.xml.sax.helpers.DefaultHandler {
       }
     }
     
-
     public void parse(final File file) throws ParserConfigurationException,
         SAXException, IOException {
-      final SAXParser parser = SAXParserFactory.newInstance().newSAXParser();
+      final SAXParser parser = SAXParserFactoryImpl.newInstance().newSAXParser();
       parser.parse(file, this);
     }