X-Git-Url: http://gitweb.fperrin.net/?a=blobdiff_plain;f=src%2Fcom%2Fhughes%2Fandroid%2Fdictionary%2Fengine%2FWiktionarySplitter.java;h=5935df8f51646f28d6462f9d59bf679ec65cabc5;hb=7254095347927bc766f1dffc2ba4b27816180d6b;hp=6839904516abd6293c9cd6f6dcedc546ed39ecc7;hpb=e479ba38bbcb261951399326623c20ffacc147d4;p=DictionaryPC.git diff --git a/src/com/hughes/android/dictionary/engine/WiktionarySplitter.java b/src/com/hughes/android/dictionary/engine/WiktionarySplitter.java index 6839904..5935df8 100644 --- a/src/com/hughes/android/dictionary/engine/WiktionarySplitter.java +++ b/src/com/hughes/android/dictionary/engine/WiktionarySplitter.java @@ -14,11 +14,15 @@ package com.hughes.android.dictionary.engine; +import java.io.BufferedInputStream; import java.io.BufferedOutputStream; import java.io.DataOutputStream; import java.io.File; +import java.io.FileInputStream; import java.io.FileOutputStream; +import java.io.InputStream; import java.io.IOException; +import java.io.OutputStream; import java.util.ArrayList; import java.util.LinkedHashMap; import java.util.List; @@ -30,6 +34,7 @@ import javax.xml.parsers.ParserConfigurationException; import javax.xml.parsers.SAXParser; import org.apache.xerces.jaxp.SAXParserFactoryImpl; +import org.apache.commons.compress.compressors.CompressorStreamFactory; import org.xml.sax.Attributes; import org.xml.sax.SAXException; @@ -78,14 +83,29 @@ public class WiktionarySplitter extends org.xml.sax.helpers.DefaultHandler { currentSelectors = pathToSelectorsEntry.getValue(); for (final Selector selector : currentSelectors) { - selector.out = new DataOutputStream(new BufferedOutputStream(new FileOutputStream(selector.outFilename))); + OutputStream tmp = new FileOutputStream(selector.outFilename + ".gz"); + tmp = new BufferedOutputStream(tmp); + tmp = new CompressorStreamFactory().createCompressorOutputStream(CompressorStreamFactory.GZIP, tmp); + tmp = new WriteBuffer(tmp, 20 * 1024 * 1024); + selector.out = new DataOutputStream(tmp); } // Do it. try { - parser.parse(new File(pathToSelectorsEntry.getKey()), this); + File input = new File(pathToSelectorsEntry.getKey() + ".bz2"); + if (!input.exists()) input = new File(pathToSelectorsEntry.getKey() + ".gz"); + if (!input.exists()) input = new File(pathToSelectorsEntry.getKey() + ".xz"); + if (!input.exists()) { + // Fallback to uncompressed file + parser.parse(new File(pathToSelectorsEntry.getKey()), this); + } else { + InputStream compressedIn = new BufferedInputStream(new FileInputStream(input)); + InputStream in = new CompressorStreamFactory().createCompressorInputStream(compressedIn); + in = new ReadAheadBuffer(in, 20 * 1024 * 1024); + parser.parse(new BufferedInputStream(in), this); + } } catch (Exception e) { - System.err.println("Exception during parse, lastPageTitle=" + lastPageTitle + ", titleBuilder=" + titleBuilder.toString()); + System.err.println("Exception during parse, lastPageTitle=" + lastPageTitle + ", titleBuilder=" + titleBuilder.toString() + " of file " + pathToSelectorsEntry.getKey()); throw e; } @@ -99,13 +119,23 @@ public class WiktionarySplitter extends org.xml.sax.helpers.DefaultHandler { String lastPageTitle = null; int pageCount = 0; + Pattern endPatterns[] = new Pattern[100]; + + private Pattern getEndPattern(int depth) { + if (endPatterns[depth] == null) + endPatterns[depth] = Pattern.compile(String.format("^={1,%d}[^=].*$", depth), Pattern.MULTILINE); + return endPatterns[depth]; + } + private void endPage() { final String title = titleBuilder.toString(); lastPageTitle = title; - if (++pageCount % 1000 == 0) { + if (++pageCount % 100000 == 0) { System.out.println("endPage: " + title + ", count=" + pageCount); } - if (title.startsWith("Wiktionary:") || + if (title.startsWith("Unsupported titles/")) return; + if (title.contains(":")) { + if (title.startsWith("Wiktionary:") || title.startsWith("Appendix:") || title.startsWith("Help:") || title.startsWith("Index:") || @@ -116,7 +146,6 @@ public class WiktionarySplitter extends org.xml.sax.helpers.DefaultHandler { title.startsWith("Rhymes:") || title.startsWith("Category:") || title.startsWith("Wikisaurus:") || - title.startsWith("Unsupported titles/") || title.startsWith("Transwiki:") || title.startsWith("File:") || title.startsWith("Thread:") || @@ -158,58 +187,70 @@ public class WiktionarySplitter extends org.xml.sax.helpers.DefaultHandler { title.startsWith("Plantilla:") || title.startsWith("Wikcionario:") || + // PT + title.startsWith("Ajuda:") || + title.startsWith("Apêndice:") || + title.startsWith("Citações:") || + title.startsWith("Portal:") || + title.startsWith("Predefinição:") || + title.startsWith("Vocabulário:") || + title.startsWith("Wikcionário:") || + // sentinel false - ) { - return; - } - if (title.contains(":")) { + ) return; if (!title.startsWith("Sign gloss:")) { System.err.println("title with colon: " + title); } } String text = textBuilder.toString(); + // Workaround for Spanish wiktionary {{ES}} and {{ES|word}} patterns + text = text.replaceAll("\\{\\{ES(\\|[^{}=]*)?}}", "== {{lengua|es}} =="); String translingual = ""; + int start = 0; + final Matcher startMatcher = headingStart.matcher(text); - while (text.length() > 0) { + while (start < text.length()) { // Find start. - final Matcher startMatcher = headingStart.matcher(text); - if (!startMatcher.find()) { + if (!startMatcher.find(start)) { return; } - text = text.substring(startMatcher.end()); + start = startMatcher.end(); final String heading = startMatcher.group(); - for (final Selector selector : currentSelectors) { - if (heading.indexOf("Translingual") != -1) { - // Find end. - final int depth = startMatcher.group(1).length(); - final Pattern endPattern = Pattern.compile(String.format("^={1,%d}[^=].*$", depth), Pattern.MULTILINE); - final Matcher endMatcher = endPattern.matcher(text); - if (endMatcher.find()) { - int end = endMatcher.start(); - translingual = text.substring(0, endMatcher.start()); - text = text.substring(end); - break; - } + // For Translingual entries just store the text for later + // use in the per-language sections + if (heading.indexOf("Translingual") != -1) { + // Find end. + final int depth = startMatcher.group(1).length(); + final Pattern endPattern = getEndPattern(depth); + + final Matcher endMatcher = endPattern.matcher(text); + if (endMatcher.find(start)) { + int end = endMatcher.start(); + translingual = text.substring(start, end); + start = end; + continue; } - if (selector.pattern.matcher(heading).find()) { + } + for (final Selector selector : currentSelectors) { + if (selector.pattern.matcher(heading).find()) { // Find end. final int depth = startMatcher.group(1).length(); - final Pattern endPattern = Pattern.compile(String.format("^={1,%d}[^=].*$", depth), Pattern.MULTILINE); + final Pattern endPattern = getEndPattern(depth); final Matcher endMatcher = endPattern.matcher(text); final int end; - if (endMatcher.find()) { + if (endMatcher.find(start)) { end = endMatcher.start(); } else { end = text.length(); } - String sectionText = text.substring(0, end); + String sectionText = text.substring(start, end); // Hack to remove empty dummy section from French if (sectionText.startsWith("\n=== {{S|étymologie}} ===\n: {{ébauche-étym")) { int dummy_end = sectionText.indexOf("}}", 41) + 2; @@ -231,7 +272,7 @@ public class WiktionarySplitter extends org.xml.sax.helpers.DefaultHandler { throw new RuntimeException(e); } - text = text.substring(end); + start = end; break; } }