X-Git-Url: http://gitweb.fperrin.net/?a=blobdiff_plain;f=src%2Fcom%2Fhughes%2Fandroid%2Fdictionary%2Fengine%2FWiktionarySplitter.java;h=34cf2d7436d36ffc99b315f232d81cb283ac1ff5;hb=d46f529d02bf4306a922c521d032f7620020b1e8;hp=628d3567e7a1ad030a5e828599e8458088501db0;hpb=1aa4de25c859304d21acfadd18cb546d1c21415b;p=DictionaryPC.git diff --git a/src/com/hughes/android/dictionary/engine/WiktionarySplitter.java b/src/com/hughes/android/dictionary/engine/WiktionarySplitter.java index 628d356..34cf2d7 100644 --- a/src/com/hughes/android/dictionary/engine/WiktionarySplitter.java +++ b/src/com/hughes/android/dictionary/engine/WiktionarySplitter.java @@ -20,6 +20,7 @@ import java.io.File; import java.io.FileOutputStream; import java.io.IOException; import java.util.ArrayList; +import java.util.LinkedHashMap; import java.util.List; import java.util.Map; import java.util.regex.Matcher; @@ -27,80 +28,78 @@ import java.util.regex.Pattern; import javax.xml.parsers.ParserConfigurationException; import javax.xml.parsers.SAXParser; -import javax.xml.parsers.SAXParserFactory; +import org.apache.xerces.jaxp.SAXParserFactoryImpl; import org.xml.sax.Attributes; import org.xml.sax.SAXException; -import com.hughes.android.dictionary.parser.enwiktionary.EnWiktionaryLangs; +import com.hughes.android.dictionary.parser.wiktionary.WiktionaryLangs; public class WiktionarySplitter extends org.xml.sax.helpers.DefaultHandler { + + // The matches the whole line, otherwise regexes don't work well on French: + // {{=uk=}} + static final Pattern headingStart = Pattern.compile("^(=+)[^=].*$", Pattern.MULTILINE); - private static final String FILE_TO_SPLIT = "data/inputs/enwiktionary-pages-articles.xml"; + final Map> pathToSelectors = new LinkedHashMap>(); + List currentSelectors = null; - static class Section implements java.io.Serializable { - private static final long serialVersionUID = -7676549898325856822L; + StringBuilder titleBuilder; + StringBuilder textBuilder; + StringBuilder currentBuilder = null; - final String title; - final String heading; - final String text; - - public Section(final String title, final String heading, final String text) { - this.title = title; - this.heading = heading; - this.text = text; - - //System.out.printf("TITLE:%s\nHEADING:%s\nTEXT:%s\n\n\n\n\n\n", title, heading, text); - } + public static void main(final String[] args) throws Exception { + final WiktionarySplitter wiktionarySplitter = new WiktionarySplitter(); + wiktionarySplitter.go(); } - static class Selector { - DataOutputStream out; - Pattern pattern; - - public Selector(final String filename, final String pattern) throws IOException { - this.out = new DataOutputStream(new BufferedOutputStream(new FileOutputStream(filename))); - this.pattern = Pattern.compile(pattern, Pattern.CASE_INSENSITIVE); + private WiktionarySplitter() { + List selectors; + for (final String code : WiktionaryLangs.wikiCodeToIsoCodeToWikiName.keySet()) { + //if (!code.equals("fr")) {continue;} + selectors = new ArrayList(); + pathToSelectors.put(String.format("data/inputs/%swiktionary-pages-articles.xml", code), selectors); + for (final Map.Entry entry : WiktionaryLangs.wikiCodeToIsoCodeToWikiName.get(code).entrySet()) { + final String dir = String.format("data/inputs/wikiSplit/%s", code); + new File(dir).mkdirs(); + selectors.add(new Selector(String.format("%s/%s.data", dir, entry.getKey()), entry.getValue())); + } } } - final List selectors = new ArrayList(); - StringBuilder titleBuilder; - StringBuilder textBuilder; - StringBuilder currentBuilder = null; + private void go() throws Exception { + final SAXParser parser = SAXParserFactoryImpl.newInstance().newSAXParser(); - public static void main(final String[] args) throws SAXException, IOException, ParserConfigurationException { - final SAXParser parser = SAXParserFactory.newInstance().newSAXParser(); - final WiktionarySplitter wiktionarySplitter = new WiktionarySplitter(); - // Configure things. - - final List selectors = wiktionarySplitter.selectors; - for (int i = 1; i < args.length; i += 2) { - final Selector selector = new Selector(args[i], args[i+1]); - selectors.add(selector); - } - - if (selectors.isEmpty()) { - for (final Map.Entry entry : EnWiktionaryLangs.isoCodeToWikiName.entrySet()) { - selectors.add(new Selector(String.format("data/inputs/enWikiSplit/%s.data", entry.getKey()), entry.getValue())); + for (final Map.Entry> pathToSelectorsEntry : pathToSelectors.entrySet()) { + + currentSelectors = pathToSelectorsEntry.getValue(); + + for (final Selector selector : currentSelectors) { + selector.out = new DataOutputStream(new BufferedOutputStream(new FileOutputStream(selector.outFilename))); } - } - - // Do it. - parser.parse(new File(FILE_TO_SPLIT), wiktionarySplitter); - - // Shutdown. - for (final Selector selector : selectors) { - selector.out.close(); + + // Do it. + try { + parser.parse(new File(pathToSelectorsEntry.getKey()), this); + } catch (Exception e) { + System.err.println("Exception during parse, lastPageTitle=" + lastPageTitle + ", titleBuilder=" + titleBuilder.toString()); + throw e; + } + + // Shutdown. + for (final Selector selector : currentSelectors) { + selector.out.close(); + } + } } - static final Pattern headingStart = Pattern.compile("^(=+)[^=]+=+", Pattern.MULTILINE); - + String lastPageTitle = null; int pageCount = 0; private void endPage() { final String title = titleBuilder.toString(); + lastPageTitle = title; if (++pageCount % 1000 == 0) { System.out.println("endPage: " + title + ", count=" + pageCount); } @@ -116,12 +115,12 @@ public class WiktionarySplitter extends org.xml.sax.helpers.DefaultHandler { text = text.substring(startMatcher.end()); final String heading = startMatcher.group(); - for (final Selector selector : selectors) { + for (final Selector selector : currentSelectors) { if (selector.pattern.matcher(heading).find()) { // Find end. final int depth = startMatcher.group(1).length(); - final Pattern endPattern = Pattern.compile(String.format("^={1,%d}[^=]+=+", depth), Pattern.MULTILINE); + final Pattern endPattern = Pattern.compile(String.format("^={1,%d}[^=].*$", depth), Pattern.MULTILINE); final Matcher endMatcher = endPattern.matcher(text); final int end; @@ -151,6 +150,36 @@ public class WiktionarySplitter extends org.xml.sax.helpers.DefaultHandler { } + // ----------------------------------------------------------------------- + + static class Section implements java.io.Serializable { + private static final long serialVersionUID = -7676549898325856822L; + + final String title; + final String heading; + final String text; + + public Section(final String title, final String heading, final String text) { + this.title = title; + this.heading = heading; + this.text = text; + + //System.out.printf("TITLE:%s\nHEADING:%s\nTEXT:%s\n\n\n\n\n\n", title, heading, text); + } + } + + static class Selector { + final String outFilename; + final Pattern pattern; + + DataOutputStream out; + + public Selector(final String filename, final String pattern) { + this.outFilename = filename; + this.pattern = Pattern.compile(pattern, Pattern.CASE_INSENSITIVE); + } + } + // ----------------------------------------------------------------------- @Override @@ -188,8 +217,10 @@ public class WiktionarySplitter extends org.xml.sax.helpers.DefaultHandler { public void parse(final File file) throws ParserConfigurationException, SAXException, IOException { - final SAXParser parser = SAXParserFactory.newInstance().newSAXParser(); + final SAXParser parser = SAXParserFactoryImpl.newInstance().newSAXParser(); parser.parse(file, this); } + + }