1 // Copyright 2011 Google Inc. All Rights Reserved.
3 // Licensed under the Apache License, Version 2.0 (the "License");
4 // you may not use this file except in compliance with the License.
5 // You may obtain a copy of the License at
7 // http://www.apache.org/licenses/LICENSE-2.0
9 // Unless required by applicable law or agreed to in writing, software
10 // distributed under the License is distributed on an "AS IS" BASIS,
11 // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 // See the License for the specific language governing permissions and
13 // limitations under the License.
15 package com.hughes.android.dictionary.engine;
17 import java.io.BufferedOutputStream;
18 import java.io.DataOutputStream;
20 import java.io.FileOutputStream;
21 import java.io.IOException;
22 import java.util.ArrayList;
23 import java.util.LinkedHashMap;
24 import java.util.List;
26 import java.util.regex.Matcher;
27 import java.util.regex.Pattern;
29 import javax.xml.parsers.ParserConfigurationException;
30 import javax.xml.parsers.SAXParser;
32 import org.apache.xerces.jaxp.SAXParserFactoryImpl;
33 import org.xml.sax.Attributes;
34 import org.xml.sax.SAXException;
36 import com.hughes.android.dictionary.parser.wiktionary.WiktionaryLangs;
38 public class WiktionarySplitter extends org.xml.sax.helpers.DefaultHandler {
40 // The matches the whole line, otherwise regexes don't work well on French:
42 // Spanish has no initial headings, so also detect {{ES as such...
43 static final Pattern headingStart = Pattern.compile("^(\\{\\{ES|(=+)[^=]).*$", Pattern.MULTILINE);
45 final Map<String,List<Selector>> pathToSelectors = new LinkedHashMap<String, List<Selector>>();
46 List<Selector> currentSelectors = null;
48 StringBuilder titleBuilder;
49 StringBuilder textBuilder;
50 StringBuilder currentBuilder = null;
52 public static void main(final String[] args) throws Exception {
53 final WiktionarySplitter wiktionarySplitter = new WiktionarySplitter();
54 wiktionarySplitter.go();
57 private WiktionarySplitter() {
58 List<Selector> selectors;
59 for (final String code : WiktionaryLangs.wikiCodeToIsoCodeToWikiName.keySet()) {
60 //if (!code.equals("fr")) {continue;}
61 selectors = new ArrayList<WiktionarySplitter.Selector>();
62 pathToSelectors.put(String.format("data/inputs/%swiktionary-pages-articles.xml", code), selectors);
63 for (final Map.Entry<String, String> entry : WiktionaryLangs.wikiCodeToIsoCodeToWikiName.get(code).entrySet()) {
64 final String dir = String.format("data/inputs/wikiSplit/%s", code);
65 new File(dir).mkdirs();
66 selectors.add(new Selector(String.format("%s/%s.data", dir, entry.getKey()), entry.getValue()));
71 private void go() throws Exception {
72 final SAXParser parser = SAXParserFactoryImpl.newInstance().newSAXParser();
75 for (final Map.Entry<String, List<Selector>> pathToSelectorsEntry : pathToSelectors.entrySet()) {
77 currentSelectors = pathToSelectorsEntry.getValue();
79 for (final Selector selector : currentSelectors) {
80 selector.out = new DataOutputStream(new BufferedOutputStream(new FileOutputStream(selector.outFilename)));
85 parser.parse(new File(pathToSelectorsEntry.getKey()), this);
86 } catch (Exception e) {
87 System.err.println("Exception during parse, lastPageTitle=" + lastPageTitle + ", titleBuilder=" + titleBuilder.toString());
92 for (final Selector selector : currentSelectors) {
99 String lastPageTitle = null;
101 private void endPage() {
102 final String title = titleBuilder.toString();
103 lastPageTitle = title;
104 if (++pageCount % 1000 == 0) {
105 System.out.println("endPage: " + title + ", count=" + pageCount);
107 if (title.startsWith("Wiktionary:") ||
108 title.startsWith("Appendix:") ||
109 title.startsWith("Help:") ||
110 title.startsWith("Index:") ||
111 title.startsWith("MediaWiki:") ||
112 title.startsWith("Citations:") ||
113 title.startsWith("Concordance:") ||
114 title.startsWith("Glossary:") ||
115 title.startsWith("Rhymes:") ||
116 title.startsWith("Category:") ||
117 title.startsWith("Wikisaurus:") ||
118 title.startsWith("Unsupported titles/") ||
119 title.startsWith("Transwiki:") ||
120 title.startsWith("File:") ||
121 title.startsWith("Thread:") ||
122 title.startsWith("Template:") ||
123 title.startsWith("Summary:") ||
125 title.startsWith("Datei:") ||
126 title.startsWith("Verzeichnis:") ||
127 title.startsWith("Vorlage:") ||
128 title.startsWith("Thesaurus:") ||
129 title.startsWith("Kategorie:") ||
130 title.startsWith("Hilfe:") ||
132 title.startsWith("Annexe:") ||
133 title.startsWith("Catégori:") ||
134 title.startsWith("Modèle:") ||
135 title.startsWith("Thésaurus:") ||
136 title.startsWith("Projet:") ||
137 title.startsWith("Aide:") ||
138 title.startsWith("Fichier:") ||
139 title.startsWith("Wiktionnaire:") ||
140 title.startsWith("Catégorie:") ||
141 title.startsWith("Portail:") ||
142 title.startsWith("utiliusateur:") ||
143 title.startsWith("Kategorio:") ||
145 title.startsWith("Wikizionario:") ||
146 title.startsWith("Appendice:") ||
147 title.startsWith("Categoria:") ||
148 title.startsWith("Aiuto:") ||
149 title.startsWith("Portail:") ||
151 title.startsWith("Apéndice:") ||
152 title.startsWith("Archivo:") ||
153 title.startsWith("Ayuda:") ||
154 title.startsWith("Categoría:") ||
155 title.startsWith("Plantilla:") ||
156 title.startsWith("Wikcionario:") ||
163 if (title.contains(":")) {
164 if (!title.startsWith("Sign gloss:")) {
165 System.err.println("title with colon: " + title);
169 String text = textBuilder.toString();
171 while (text.length() > 0) {
173 final Matcher startMatcher = headingStart.matcher(text);
174 if (!startMatcher.find()) {
177 text = text.substring(startMatcher.end());
179 final String heading = startMatcher.group();
180 for (final Selector selector : currentSelectors) {
181 if (selector.pattern.matcher(heading).find()) {
184 final int depth = startMatcher.group(1).length();
185 final Pattern endPattern = Pattern.compile(String.format("^={1,%d}[^=].*$", depth), Pattern.MULTILINE);
187 final Matcher endMatcher = endPattern.matcher(text);
189 if (endMatcher.find()) {
190 end = endMatcher.start();
195 final String sectionText = text.substring(0, end);
196 final Section section = new Section(title, heading, sectionText);
199 selector.out.writeUTF(section.title);
200 selector.out.writeUTF(section.heading);
201 final byte[] bytes = section.text.getBytes("UTF8");
202 selector.out.writeInt(bytes.length);
203 selector.out.write(bytes);
204 } catch (IOException e) {
205 throw new RuntimeException(e);
208 text = text.substring(end);
215 // -----------------------------------------------------------------------
217 static class Section implements java.io.Serializable {
218 private static final long serialVersionUID = -7676549898325856822L;
221 final String heading;
224 public Section(final String title, final String heading, final String text) {
226 this.heading = heading;
229 //System.out.printf("TITLE:%s\nHEADING:%s\nTEXT:%s\n\n\n\n\n\n", title, heading, text);
233 static class Selector {
234 final String outFilename;
235 final Pattern pattern;
237 DataOutputStream out;
239 public Selector(final String filename, final String pattern) {
240 this.outFilename = filename;
241 this.pattern = Pattern.compile(pattern, Pattern.CASE_INSENSITIVE);
245 // -----------------------------------------------------------------------
248 public void startElement(String uri, String localName, String qName,
249 Attributes attributes) {
250 currentBuilder = null;
251 if ("page".equals(qName)) {
252 titleBuilder = new StringBuilder();
254 // Start with "\n" to better match certain strings.
255 textBuilder = new StringBuilder("\n");
256 } else if ("title".equals(qName)) {
257 currentBuilder = titleBuilder;
258 } else if ("text".equals(qName)) {
259 currentBuilder = textBuilder;
264 public void characters(char[] ch, int start, int length) throws SAXException {
265 if (currentBuilder != null) {
266 currentBuilder.append(ch, start, length);
271 public void endElement(String uri, String localName, String qName)
272 throws SAXException {
273 currentBuilder = null;
274 if ("page".equals(qName)) {
279 public void parse(final File file) throws ParserConfigurationException,
280 SAXException, IOException {
281 final SAXParser parser = SAXParserFactoryImpl.newInstance().newSAXParser();
282 parser.parse(file, this);