1 // Copyright 2011 Google Inc. All Rights Reserved.
3 // Licensed under the Apache License, Version 2.0 (the "License");
4 // you may not use this file except in compliance with the License.
5 // You may obtain a copy of the License at
7 // http://www.apache.org/licenses/LICENSE-2.0
9 // Unless required by applicable law or agreed to in writing, software
10 // distributed under the License is distributed on an "AS IS" BASIS,
11 // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 // See the License for the specific language governing permissions and
13 // limitations under the License.
15 package com.hughes.android.dictionary.engine;
17 import java.io.BufferedInputStream;
18 import java.io.BufferedOutputStream;
19 import java.io.DataOutputStream;
21 import java.io.FileInputStream;
22 import java.io.FileOutputStream;
23 import java.io.InputStream;
24 import java.io.IOException;
25 import java.io.OutputStream;
26 import java.util.ArrayList;
27 import java.util.LinkedHashMap;
28 import java.util.List;
30 import java.util.regex.Matcher;
31 import java.util.regex.Pattern;
33 import javax.xml.parsers.ParserConfigurationException;
34 import javax.xml.parsers.SAXParser;
36 import org.apache.xerces.jaxp.SAXParserFactoryImpl;
37 import org.apache.commons.compress.compressors.CompressorStreamFactory;
38 import org.xml.sax.Attributes;
39 import org.xml.sax.SAXException;
41 import com.hughes.android.dictionary.parser.wiktionary.WiktionaryLangs;
43 public class WiktionarySplitter extends org.xml.sax.helpers.DefaultHandler {
45 // The matches the whole line, otherwise regexes don't work well on French:
47 // Spanish has no initial headings, tried to also detect {{ES as such
48 // with "^(\\{\\{ES|(=+)[^=]).*$" but that broke English.
49 static final Pattern headingStart = Pattern.compile("^(=+)[^=].*$", Pattern.MULTILINE);
51 final Map<String,List<Selector>> pathToSelectors = new LinkedHashMap<String, List<Selector>>();
52 List<Selector> currentSelectors = null;
54 StringBuilder titleBuilder;
55 StringBuilder textBuilder;
56 StringBuilder currentBuilder = null;
58 public static void main(final String[] args) throws Exception {
59 final WiktionarySplitter wiktionarySplitter = new WiktionarySplitter();
60 wiktionarySplitter.go();
63 private WiktionarySplitter() {
64 List<Selector> selectors;
65 for (final String code : WiktionaryLangs.wikiCodeToIsoCodeToWikiName.keySet()) {
66 //if (!code.equals("fr")) {continue;}
67 selectors = new ArrayList<WiktionarySplitter.Selector>();
68 pathToSelectors.put(String.format("data/inputs/%swiktionary-pages-articles.xml", code), selectors);
69 for (final Map.Entry<String, String> entry : WiktionaryLangs.wikiCodeToIsoCodeToWikiName.get(code).entrySet()) {
70 final String dir = String.format("data/inputs/wikiSplit/%s", code);
71 new File(dir).mkdirs();
72 selectors.add(new Selector(String.format("%s/%s.data", dir, entry.getKey()), entry.getValue()));
77 private void go() throws Exception {
78 final SAXParser parser = SAXParserFactoryImpl.newInstance().newSAXParser();
81 for (final Map.Entry<String, List<Selector>> pathToSelectorsEntry : pathToSelectors.entrySet()) {
83 currentSelectors = pathToSelectorsEntry.getValue();
85 for (final Selector selector : currentSelectors) {
86 OutputStream tmp = new FileOutputStream(selector.outFilename);
87 tmp = new WriteBuffer(tmp, 20 * 1024 * 1024);
88 selector.out = new DataOutputStream(tmp);
93 File input = new File(pathToSelectorsEntry.getKey() + ".bz2");
94 if (!input.exists()) input = new File(pathToSelectorsEntry.getKey() + ".gz");
95 if (!input.exists()) input = new File(pathToSelectorsEntry.getKey() + ".xz");
96 if (!input.exists()) {
97 // Fallback to uncompressed file
98 parser.parse(new File(pathToSelectorsEntry.getKey()), this);
100 InputStream compressedIn = new BufferedInputStream(new FileInputStream(input));
101 InputStream in = new CompressorStreamFactory().createCompressorInputStream(compressedIn);
102 in = new ReadAheadBuffer(in, 20 * 1024 * 1024);
103 parser.parse(new BufferedInputStream(in), this);
105 } catch (Exception e) {
106 System.err.println("Exception during parse, lastPageTitle=" + lastPageTitle + ", titleBuilder=" + titleBuilder.toString());
111 for (final Selector selector : currentSelectors) {
112 selector.out.close();
118 String lastPageTitle = null;
120 Pattern endPatterns[] = new Pattern[100];
122 private Pattern getEndPattern(int depth) {
123 if (endPatterns[depth] == null)
124 endPatterns[depth] = Pattern.compile(String.format("^={1,%d}[^=].*$", depth), Pattern.MULTILINE);
125 return endPatterns[depth];
128 private void endPage() {
129 final String title = titleBuilder.toString();
130 lastPageTitle = title;
131 if (++pageCount % 1000 == 0) {
132 System.out.println("endPage: " + title + ", count=" + pageCount);
134 if (title.startsWith("Wiktionary:") ||
135 title.startsWith("Appendix:") ||
136 title.startsWith("Help:") ||
137 title.startsWith("Index:") ||
138 title.startsWith("MediaWiki:") ||
139 title.startsWith("Citations:") ||
140 title.startsWith("Concordance:") ||
141 title.startsWith("Glossary:") ||
142 title.startsWith("Rhymes:") ||
143 title.startsWith("Category:") ||
144 title.startsWith("Wikisaurus:") ||
145 title.startsWith("Unsupported titles/") ||
146 title.startsWith("Transwiki:") ||
147 title.startsWith("File:") ||
148 title.startsWith("Thread:") ||
149 title.startsWith("Template:") ||
150 title.startsWith("Summary:") ||
151 title.startsWith("Module:") ||
153 title.startsWith("Datei:") ||
154 title.startsWith("Verzeichnis:") ||
155 title.startsWith("Vorlage:") ||
156 title.startsWith("Thesaurus:") ||
157 title.startsWith("Kategorie:") ||
158 title.startsWith("Hilfe:") ||
159 title.startsWith("Reim:") ||
161 title.startsWith("Annexe:") ||
162 title.startsWith("Catégori:") ||
163 title.startsWith("Modèle:") ||
164 title.startsWith("Thésaurus:") ||
165 title.startsWith("Projet:") ||
166 title.startsWith("Aide:") ||
167 title.startsWith("Fichier:") ||
168 title.startsWith("Wiktionnaire:") ||
169 title.startsWith("Catégorie:") ||
170 title.startsWith("Portail:") ||
171 title.startsWith("utiliusateur:") ||
172 title.startsWith("Kategorio:") ||
174 title.startsWith("Wikizionario:") ||
175 title.startsWith("Appendice:") ||
176 title.startsWith("Categoria:") ||
177 title.startsWith("Aiuto:") ||
178 title.startsWith("Portail:") ||
180 title.startsWith("Apéndice:") ||
181 title.startsWith("Archivo:") ||
182 title.startsWith("Ayuda:") ||
183 title.startsWith("Categoría:") ||
184 title.startsWith("Plantilla:") ||
185 title.startsWith("Wikcionario:") ||
192 if (title.contains(":")) {
193 if (!title.startsWith("Sign gloss:")) {
194 System.err.println("title with colon: " + title);
198 String text = textBuilder.toString();
199 String translingual = "";
201 while (text.length() > 0) {
203 final Matcher startMatcher = headingStart.matcher(text);
204 if (!startMatcher.find()) {
207 text = text.substring(startMatcher.end());
209 final String heading = startMatcher.group();
210 for (final Selector selector : currentSelectors) {
211 if (heading.indexOf("Translingual") != -1) {
213 final int depth = startMatcher.group(1).length();
214 final Pattern endPattern = getEndPattern(depth);
216 final Matcher endMatcher = endPattern.matcher(text);
217 if (endMatcher.find()) {
218 int end = endMatcher.start();
219 translingual = text.substring(0, endMatcher.start());
220 text = text.substring(end);
224 if (selector.pattern.matcher(heading).find()) {
227 final int depth = startMatcher.group(1).length();
228 final Pattern endPattern = getEndPattern(depth);
230 final Matcher endMatcher = endPattern.matcher(text);
232 if (endMatcher.find()) {
233 end = endMatcher.start();
238 String sectionText = text.substring(0, end);
239 // Hack to remove empty dummy section from French
240 if (sectionText.startsWith("\n=== {{S|étymologie}} ===\n: {{ébauche-étym")) {
241 int dummy_end = sectionText.indexOf("}}", 41) + 2;
242 while (dummy_end + 1 < sectionText.length() &&
243 sectionText.charAt(dummy_end) == '\n' &&
244 sectionText.charAt(dummy_end + 1) == '\n') ++dummy_end;
245 sectionText = sectionText.substring(dummy_end);
247 if (heading.indexOf("Japanese") == -1) sectionText += translingual;
248 final Section section = new Section(title, heading, sectionText);
251 selector.out.writeUTF(section.title);
252 selector.out.writeUTF(section.heading);
253 final byte[] bytes = section.text.getBytes("UTF8");
254 selector.out.writeInt(bytes.length);
255 selector.out.write(bytes);
256 } catch (IOException e) {
257 throw new RuntimeException(e);
260 text = text.substring(end);
268 // -----------------------------------------------------------------------
270 static class Section implements java.io.Serializable {
271 private static final long serialVersionUID = -7676549898325856822L;
274 final String heading;
277 public Section(final String title, final String heading, final String text) {
279 this.heading = heading;
282 //System.out.printf("TITLE:%s\nHEADING:%s\nTEXT:%s\n\n\n\n\n\n", title, heading, text);
286 static class Selector {
287 final String outFilename;
288 final Pattern pattern;
290 DataOutputStream out;
292 public Selector(final String filename, final String pattern) {
293 this.outFilename = filename;
294 this.pattern = Pattern.compile(pattern, Pattern.CASE_INSENSITIVE);
298 // -----------------------------------------------------------------------
301 public void startElement(String uri, String localName, String qName,
302 Attributes attributes) {
303 currentBuilder = null;
304 if ("page".equals(qName)) {
305 titleBuilder = new StringBuilder();
307 // Start with "\n" to better match certain strings.
308 textBuilder = new StringBuilder("\n");
309 } else if ("title".equals(qName)) {
310 currentBuilder = titleBuilder;
311 } else if ("text".equals(qName)) {
312 currentBuilder = textBuilder;
317 public void characters(char[] ch, int start, int length) throws SAXException {
318 if (currentBuilder != null) {
319 currentBuilder.append(ch, start, length);
324 public void endElement(String uri, String localName, String qName)
325 throws SAXException {
326 currentBuilder = null;
327 if ("page".equals(qName)) {
332 public void parse(final File file) throws ParserConfigurationException,
333 SAXException, IOException {
334 final SAXParser parser = SAXParserFactoryImpl.newInstance().newSAXParser();
335 parser.parse(file, this);