X-Git-Url: http://gitweb.fperrin.net/?a=blobdiff_plain;f=src%2Fcom%2Fhughes%2Fandroid%2Fdictionary%2Fengine%2FWiktionarySplitter.java;h=3cee85da6a2a89743f53be952239e4aea8f87d0f;hb=4c7701228aff69ceb4dffd23ed319382b5f0a55c;hp=c1cb09feee66380514a1de7a4aca2a933855e2a6;hpb=5ce5941fa2f306dab71d74db377235c685b70a74;p=DictionaryPC.git diff --git a/src/com/hughes/android/dictionary/engine/WiktionarySplitter.java b/src/com/hughes/android/dictionary/engine/WiktionarySplitter.java index c1cb09f..3cee85d 100644 --- a/src/com/hughes/android/dictionary/engine/WiktionarySplitter.java +++ b/src/com/hughes/android/dictionary/engine/WiktionarySplitter.java @@ -22,6 +22,7 @@ import java.io.FileInputStream; import java.io.FileOutputStream; import java.io.InputStream; import java.io.IOException; +import java.io.OutputStream; import java.util.ArrayList; import java.util.LinkedHashMap; import java.util.List; @@ -82,7 +83,11 @@ public class WiktionarySplitter extends org.xml.sax.helpers.DefaultHandler { currentSelectors = pathToSelectorsEntry.getValue(); for (final Selector selector : currentSelectors) { - selector.out = new DataOutputStream(new BufferedOutputStream(new FileOutputStream(selector.outFilename))); + OutputStream tmp = new FileOutputStream(selector.outFilename + ".gz"); + tmp = new BufferedOutputStream(tmp); + tmp = new CompressorStreamFactory().createCompressorOutputStream(CompressorStreamFactory.GZIP, tmp); + tmp = new WriteBuffer(tmp, 20 * 1024 * 1024); + selector.out = new DataOutputStream(tmp); } // Do it. @@ -194,45 +199,49 @@ public class WiktionarySplitter extends org.xml.sax.helpers.DefaultHandler { String text = textBuilder.toString(); String translingual = ""; + int start = 0; + final Matcher startMatcher = headingStart.matcher(text); - while (text.length() > 0) { + while (start < text.length()) { // Find start. - final Matcher startMatcher = headingStart.matcher(text); - if (!startMatcher.find()) { + if (!startMatcher.find(start)) { return; } - text = text.substring(startMatcher.end()); + start = startMatcher.end(); final String heading = startMatcher.group(); - for (final Selector selector : currentSelectors) { - if (heading.indexOf("Translingual") != -1) { - // Find end. - final int depth = startMatcher.group(1).length(); - final Pattern endPattern = getEndPattern(depth); - final Matcher endMatcher = endPattern.matcher(text); - if (endMatcher.find()) { - int end = endMatcher.start(); - translingual = text.substring(0, endMatcher.start()); - text = text.substring(end); - break; - } + // For Translingual entries just store the text for later + // use in the per-language sections + if (heading.indexOf("Translingual") != -1) { + // Find end. + final int depth = startMatcher.group(1).length(); + final Pattern endPattern = getEndPattern(depth); + + final Matcher endMatcher = endPattern.matcher(text); + if (endMatcher.find(start)) { + int end = endMatcher.start(); + translingual = text.substring(start, end); + start = end; + continue; } - if (selector.pattern.matcher(heading).find()) { + } + for (final Selector selector : currentSelectors) { + if (selector.pattern.matcher(heading).find()) { // Find end. final int depth = startMatcher.group(1).length(); final Pattern endPattern = getEndPattern(depth); final Matcher endMatcher = endPattern.matcher(text); final int end; - if (endMatcher.find()) { + if (endMatcher.find(start)) { end = endMatcher.start(); } else { end = text.length(); } - String sectionText = text.substring(0, end); + String sectionText = text.substring(start, end); // Hack to remove empty dummy section from French if (sectionText.startsWith("\n=== {{S|étymologie}} ===\n: {{ébauche-étym")) { int dummy_end = sectionText.indexOf("}}", 41) + 2; @@ -254,7 +263,7 @@ public class WiktionarySplitter extends org.xml.sax.helpers.DefaultHandler { throw new RuntimeException(e); } - text = text.substring(end); + start = end; break; } }