X-Git-Url: http://gitweb.fperrin.net/?a=blobdiff_plain;f=src%2Fcom%2Fhughes%2Fandroid%2Fdictionary%2Fengine%2FWiktionarySplitter.java;h=0624025387b7f134242de98736ec4082e4915029;hb=b3b942928b2efb012c2b666a3638c546feecee5c;hp=290a58fccc1e38a6c36acdd44f2cb08cf42abb40;hpb=ee1dbfb669462305a1c07e4d804a90af79f5d39f;p=DictionaryPC.git

diff --git a/src/com/hughes/android/dictionary/engine/WiktionarySplitter.java b/src/com/hughes/android/dictionary/engine/WiktionarySplitter.java
index 290a58f..0624025 100644
--- a/src/com/hughes/android/dictionary/engine/WiktionarySplitter.java
+++ b/src/com/hughes/android/dictionary/engine/WiktionarySplitter.java
@@ -22,6 +22,7 @@ import java.io.FileInputStream;
 import java.io.FileOutputStream;
 import java.io.InputStream;
 import java.io.IOException;
+import java.io.OutputStream;
 import java.util.ArrayList;
 import java.util.LinkedHashMap;
 import java.util.List;
@@ -82,7 +83,11 @@ public class WiktionarySplitter extends org.xml.sax.helpers.DefaultHandler {
             currentSelectors = pathToSelectorsEntry.getValue();
 
             for (final Selector selector : currentSelectors) {
-                selector.out = new DataOutputStream(new BufferedOutputStream(new FileOutputStream(selector.outFilename)));
+                OutputStream tmp = new FileOutputStream(selector.outFilename + ".gz");
+                tmp = new BufferedOutputStream(tmp);
+                tmp = new CompressorStreamFactory().createCompressorOutputStream(CompressorStreamFactory.GZIP, tmp);
+                tmp = new WriteBuffer(tmp, 20 * 1024 * 1024);
+                selector.out = new DataOutputStream(tmp);
             }
 
             // Do it.
@@ -96,6 +101,7 @@ public class WiktionarySplitter extends org.xml.sax.helpers.DefaultHandler {
                 } else {
                     InputStream compressedIn = new BufferedInputStream(new FileInputStream(input));
                     InputStream in = new CompressorStreamFactory().createCompressorInputStream(compressedIn);
+                    in = new ReadAheadBuffer(in, 20 * 1024 * 1024);
                     parser.parse(new BufferedInputStream(in), this);
                 }
             } catch (Exception e) {
@@ -113,6 +119,14 @@ public class WiktionarySplitter extends org.xml.sax.helpers.DefaultHandler {
 
     String lastPageTitle = null;
     int pageCount = 0;
+    Pattern endPatterns[] = new Pattern[100];
+
+    private Pattern getEndPattern(int depth) {
+        if (endPatterns[depth] == null)
+            endPatterns[depth] = Pattern.compile(String.format("^={1,%d}[^=].*$", depth), Pattern.MULTILINE);
+        return endPatterns[depth];
+    }
+
     private void endPage() {
         final String title = titleBuilder.toString();
         lastPageTitle = title;
@@ -199,7 +213,7 @@ public class WiktionarySplitter extends org.xml.sax.helpers.DefaultHandler {
                 if (heading.indexOf("Translingual") != -1) {
                     // Find end.
                     final int depth = startMatcher.group(1).length();
-                    final Pattern endPattern = Pattern.compile(String.format("^={1,%d}[^=].*$", depth), Pattern.MULTILINE);
+                    final Pattern endPattern = getEndPattern(depth);
 
                     final Matcher endMatcher = endPattern.matcher(text);
                     if (endMatcher.find()) {
@@ -213,7 +227,7 @@ public class WiktionarySplitter extends org.xml.sax.helpers.DefaultHandler {
 
                     // Find end.
                     final int depth = startMatcher.group(1).length();
-                    final Pattern endPattern = Pattern.compile(String.format("^={1,%d}[^=].*$", depth), Pattern.MULTILINE);
+                    final Pattern endPattern = getEndPattern(depth);
 
                     final Matcher endMatcher = endPattern.matcher(text);
                     final int end;