X-Git-Url: http://gitweb.fperrin.net/?a=blobdiff_plain;f=src%2Fcom%2Fhughes%2Fandroid%2Fdictionary%2Fengine%2FWiktionarySplitter.java;h=37344cae59362fa1e5da2342bb36e2e98f708724;hb=eec1a89b6cdffe7048aefa3cb2b3497b1744be99;hp=28b11bc669f3cbdb146a4a41c4dc2afc802ae2a9;hpb=cde823acd318d713335c4ec5ea35e708063abea5;p=DictionaryPC.git

diff --git a/src/com/hughes/android/dictionary/engine/WiktionarySplitter.java b/src/com/hughes/android/dictionary/engine/WiktionarySplitter.java
index 28b11bc..37344ca 100644
--- a/src/com/hughes/android/dictionary/engine/WiktionarySplitter.java
+++ b/src/com/hughes/android/dictionary/engine/WiktionarySplitter.java
@@ -130,10 +130,12 @@ public class WiktionarySplitter extends org.xml.sax.helpers.DefaultHandler {
     private void endPage() {
         final String title = titleBuilder.toString();
         lastPageTitle = title;
-        if (++pageCount % 1000 == 0) {
+        if (++pageCount % 100000 == 0) {
             System.out.println("endPage: " + title + ", count=" + pageCount);
         }
-        if (title.startsWith("Wiktionary:") ||
+        if (title.startsWith("Unsupported titles/")) return;
+        if (title.contains(":")) {
+            if (title.startsWith("Wiktionary:") ||
                 title.startsWith("Appendix:") ||
                 title.startsWith("Help:") ||
                 title.startsWith("Index:") ||
@@ -144,7 +146,6 @@ public class WiktionarySplitter extends org.xml.sax.helpers.DefaultHandler {
                 title.startsWith("Rhymes:") ||
                 title.startsWith("Category:") ||
                 title.startsWith("Wikisaurus:") ||
-                title.startsWith("Unsupported titles/") ||
                 title.startsWith("Transwiki:") ||
                 title.startsWith("File:") ||
                 title.startsWith("Thread:") ||
@@ -188,10 +189,7 @@ public class WiktionarySplitter extends org.xml.sax.helpers.DefaultHandler {
 
                 // sentinel
                 false
-           ) {
-            return;
-        }
-        if (title.contains(":")) {
+               ) return;
             if (!title.startsWith("Sign gloss:")) {
                 System.err.println("title with colon: " + title);
             }
@@ -199,14 +197,15 @@ public class WiktionarySplitter extends org.xml.sax.helpers.DefaultHandler {
 
         String text = textBuilder.toString();
         String translingual = "";
+        int start = 0;
+        final Matcher startMatcher = headingStart.matcher(text);
 
-        while (text.length() > 0) {
+        while (start < text.length()) {
             // Find start.
-            final Matcher startMatcher = headingStart.matcher(text);
-            if (!startMatcher.find()) {
+            if (!startMatcher.find(start)) {
                 return;
             }
-            text = text.substring(startMatcher.end());
+            start = startMatcher.end();
 
             final String heading = startMatcher.group();
 
@@ -218,10 +217,10 @@ public class WiktionarySplitter extends org.xml.sax.helpers.DefaultHandler {
                 final Pattern endPattern = getEndPattern(depth);
 
                 final Matcher endMatcher = endPattern.matcher(text);
-                if (endMatcher.find()) {
+                if (endMatcher.find(start)) {
                     int end = endMatcher.start();
-                    translingual = text.substring(0, endMatcher.start());
-                    text = text.substring(end);
+                    translingual = text.substring(start, end);
+                    start = end;
                     continue;
                 }
             }
@@ -234,13 +233,13 @@ public class WiktionarySplitter extends org.xml.sax.helpers.DefaultHandler {
 
                     final Matcher endMatcher = endPattern.matcher(text);
                     final int end;
-                    if (endMatcher.find()) {
+                    if (endMatcher.find(start)) {
                         end = endMatcher.start();
                     } else {
                         end = text.length();
                     }
 
-                    String sectionText = text.substring(0, end);
+                    String sectionText = text.substring(start, end);
                     // Hack to remove empty dummy section from French
                     if (sectionText.startsWith("\n=== {{S|étymologie}} ===\n: {{ébauche-étym")) {
                         int dummy_end = sectionText.indexOf("}}", 41) + 2;
@@ -262,7 +261,7 @@ public class WiktionarySplitter extends org.xml.sax.helpers.DefaultHandler {
                         throw new RuntimeException(e);
                     }
 
-                    text = text.substring(end);
+                    start = end;
                     break;
                 }
             }