Add a write buffer to wiktionary splitter outputs.

Around 20% faster processing, and will be useful when
adding compression support as well.

diff --git a/src/com/hughes/android/dictionary/engine/WiktionarySplitter.java b/src/com/hughes/android/dictionary/engine/WiktionarySplitter.java
index c1cb09feee66380514a1de7a4aca2a933855e2a6..097643f1af7e3762cb1d76b16e0a44c4e0f3e240 100644 (file)
--- a/src/com/hughes/android/dictionary/engine/WiktionarySplitter.java
+++ b/src/com/hughes/android/dictionary/engine/WiktionarySplitter.java
@@ -22,6 +22,7 @@ import java.io.FileInputStream;
 import java.io.FileOutputStream;
 import java.io.InputStream;
 import java.io.IOException;
+import java.io.OutputStream;
 import java.util.ArrayList;
 import java.util.LinkedHashMap;
 import java.util.List;
@@ -82,7 +83,9 @@ public class WiktionarySplitter extends org.xml.sax.helpers.DefaultHandler {
             currentSelectors = pathToSelectorsEntry.getValue();
 
             for (final Selector selector : currentSelectors) {
-                selector.out = new DataOutputStream(new BufferedOutputStream(new FileOutputStream(selector.outFilename)));
+                OutputStream tmp = new FileOutputStream(selector.outFilename);
+                tmp = new WriteBuffer(tmp, 20 * 1024 * 1024);
+                selector.out = new DataOutputStream(tmp);
             }
 
             // Do it.

diff --git a/src/com/hughes/android/dictionary/engine/WriteBuffer.java b/src/com/hughes/android/dictionary/engine/WriteBuffer.java
new file mode 100644 (file)
index 0000000..c68264e
--- /dev/null
+++ b/src/com/hughes/android/dictionary/engine/WriteBuffer.java
@@ -0,0 +1,65 @@
+// Copyright 2017 Reimar Döffinger
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.hughes.android.dictionary.engine;
+
+import java.io.OutputStream;
+import java.io.IOException;
+import java.io.PipedInputStream;
+import java.io.PipedOutputStream;
+
+public class WriteBuffer extends PipedOutputStream {
+    static int BLOCK_SIZE = 1024 * 1024;
+    public WriteBuffer(OutputStream out, int size) {
+        assert size >= 2 * BLOCK_SIZE;
+        this.out = out;
+        try {
+            pipe = new PipedInputStream(this, size);
+            buffer = new byte[BLOCK_SIZE];
+            writeThread = new Thread(new Runnable() {
+                public void run() {
+                    int read;
+                    try {
+                        while ((read = pipe.read(buffer)) > 0)
+                        {
+                            out.write(buffer, 0, read);
+                            out.flush();
+                        }
+                    } catch (IOException e) {
+                        System.out.println("Error writing to file " + e);
+                    }
+                    try {
+                        out.close();
+                    } catch (IOException e) {}
+                }
+            });
+            writeThread.start();
+        } catch (IOException e) {}
+    }
+
+    public void close() throws IOException
+    {
+        super.close();
+        try {
+            writeThread.join();
+        } catch (InterruptedException e) {
+            System.out.println("Failed waiting for WriteBuffer thread " + e);
+        }
+    }
+
+    Thread writeThread;
+    OutputStream out;
+    PipedInputStream pipe;
+    byte buffer[];
+}