1 package com.hughes.android.dictionary.parser;
4 import java.io.IOException;
5 import java.util.ArrayList;
6 import java.util.Arrays;
7 import java.util.LinkedHashSet;
11 import java.util.regex.Pattern;
13 import javax.xml.parsers.ParserConfigurationException;
14 import javax.xml.parsers.SAXParser;
15 import javax.xml.parsers.SAXParserFactory;
17 import org.xml.sax.Attributes;
18 import org.xml.sax.SAXException;
20 import com.hughes.android.dictionary.engine.DictionaryBuilder;
21 import com.hughes.android.dictionary.engine.IndexBuilder;
22 import com.hughes.android.dictionary.parser.WikiWord.FormOf;
23 import com.hughes.android.dictionary.parser.WikiWord.Translation;
24 import com.hughes.util.ListUtil;
25 import com.hughes.util.StringUtil;
27 public class EnWiktionaryXmlParserOld extends org.xml.sax.helpers.DefaultHandler implements WikiCallback {
29 static final Pattern partOfSpeechHeader = Pattern.compile(
30 "Noun|Verb|Adjective|Adverb|Pronoun|Conjunction|Interjection|" +
31 "Preposition|Proper noun|Article|Prepositional phrase|Acronym|" +
32 "Abbreviation|Initialism|Contraction|Prefix|Suffix|Symbol|Letter|" +
33 "Ligature|Idiom|Phrase|" +
34 // These are @deprecated:
35 "Noun form|Verb form|Adjective form|Nominal phrase|Noun phrase|" +
36 "Verb phrase|Transitive verb|Intransitive verb|Reflexive verb|" +
37 // These are extras I found:
38 "Determiner|Numeral|Number|Cardinal number|Ordinal number|Proverb|" +
39 "Particle|Interjection|Pronominal adverb" +
40 "Han character|Hanzi|Hanja|Kanji|Katakana character|Syllable");
42 static final Pattern wikiMarkup = Pattern.compile("\\[\\[|\\]\\]|''+");
44 final DictionaryBuilder dictBuilder;
46 final IndexBuilder[] indexBuilders;
47 final Pattern[] langPatterns;
48 final int enIndexBuilder;
50 StringBuilder titleBuilder;
51 StringBuilder textBuilder;
52 StringBuilder currentBuilder = null;
54 static void assertTrue(final boolean condition) {
55 assertTrue(condition, "");
58 static void assertTrue(final boolean condition, final String message) {
60 System.err.println("Assertion failed, message: " + message);
61 new RuntimeException().printStackTrace(System.err);
65 public EnWiktionaryXmlParserOld(final DictionaryBuilder dictBuilder, final Pattern[] langPatterns, final int enIndexBuilder) {
66 assertTrue(langPatterns.length == 2);
67 this.dictBuilder = dictBuilder;
68 this.indexBuilders = dictBuilder.indexBuilders.toArray(new IndexBuilder[0]);
69 this.langPatterns = langPatterns;
70 this.enIndexBuilder = enIndexBuilder;
74 public void startElement(String uri, String localName, String qName,
75 Attributes attributes) {
76 currentBuilder = null;
77 if ("page".equals(qName)) {
78 titleBuilder = new StringBuilder();
80 // Start with "\n" to better match certain strings.
81 textBuilder = new StringBuilder("\n");
82 } else if ("title".equals(qName)) {
83 currentBuilder = titleBuilder;
84 } else if ("text".equals(qName)) {
85 currentBuilder = textBuilder;
90 public void characters(char[] ch, int start, int length) throws SAXException {
91 if (currentBuilder != null) {
92 currentBuilder.append(ch, start, length);
97 public void endElement(String uri, String localName, String qName)
99 currentBuilder = null;
100 if ("page".equals(qName)) {
106 public void parse(final File file) throws ParserConfigurationException,
107 SAXException, IOException {
108 final SAXParser parser = SAXParserFactory.newInstance().newSAXParser();
109 parser.parse(file, this);
113 private void endPage() {
114 title = titleBuilder.toString();
116 if (pageCount % 1000 == 0) {
117 System.out.println("pageCount=" + pageCount);
119 if (title.startsWith("Wiktionary:") ||
120 title.startsWith("Template:") ||
121 title.startsWith("Appendix:") ||
122 title.startsWith("Category:") ||
123 title.startsWith("Index:") ||
124 title.startsWith("MediaWiki:") ||
125 title.startsWith("TransWiki:") ||
126 title.startsWith("Citations:") ||
127 title.startsWith("Concordance:") ||
128 title.startsWith("Help:")) {
133 currentHeading = null;
134 insidePartOfSpeech = false;
135 // System.err.println("Working on page: " + title);
137 WikiParser.parse(textBuilder.toString(), this);
138 } catch (Throwable e) {
139 System.err.println("Failure on page: " + title);
140 e.printStackTrace(System.err);
143 for (final WikiWord word : words) {
144 word.wikiWordToQuickDic(dictBuilder, enIndexBuilder);
150 // ------------------------------------------------------------------------
151 // ------------------------------------------------------------------------
152 // ------------------------------------------------------------------------
153 // ------------------------------------------------------------------------
156 * Two things can happen:
158 * We can be in a ==German== section. There we will see English definitions.
159 * Each POS should get its own QuickDic entry. Pretty much everything goes
162 * Or we can be in an ==English== section with English definitions
163 * and maybe see translations for languages we care about.
165 * In either case, we need to differentiate the subsections (Noun, Verb, etc.)
166 * into separate QuickDic entries, but that's tricky--how do we know when we
167 * found a subsection? Just ignore anything containing pronunciation and
170 * How do we decide when to seal the deal on an entry?
172 * Would be nice if the parser told us about leaving sections....
178 String currentHeading;
180 final List<WikiWord> words = new ArrayList<WikiWord>();
181 WikiWord currentWord;
182 WikiWord.PartOfSpeech currentPartOfSpeech;
183 WikiWord.TranslationSense currentTranslationSense;
184 boolean insidePartOfSpeech;
186 StringBuilder wikiBuilder = null;
189 public void onWikiLink(String[] args) {
190 if (wikiBuilder == null) {
193 wikiBuilder.append(args[args.length - 1]);
196 // ttbc: translations to be checked.
197 static final Set<String> useRemainingArgTemplates = new LinkedHashSet<String>(Arrays.asList(
198 "Arab", "Cyrl", "fa-Arab", "italbrac", "Khmr", "ku-Arab", "IPAchar", "Laoo",
199 "sd-Arab", "Thai", "ttbc", "unicode", "ur-Arab", "yue-yue-j", "zh-ts",
200 "zh-tsp", "zh-zh-p", "ug-Arab", "ko-inline", "Jpan", "Kore", "rfscript", "Latinx"));
201 static final Set<String> ignoreTemplates = new LinkedHashSet<String>(Arrays.asList("audio", "rhymes", "hyphenation", "homophones", "wikipedia", "rel-top", "rel-bottom", "sense", "wikisource1911Enc", "g"));
202 static final Set<String> grammarTemplates = new LinkedHashSet<String>(Arrays.asList("impf", "pf", "pf.", "indeclinable"));
203 static final Set<String> passThroughTemplates = new LinkedHashSet<String>(Arrays.asList("zzzzzzzzzzzzzzz"));
206 public void onTemplate(final List<String> positionalArgs, final Map<String,String> namedArgs) {
207 if (positionalArgs.isEmpty()) {
208 // This happens very rarely with special templates.
211 final String name = positionalArgs.get(0);
213 namedArgs.remove("lang");
214 namedArgs.remove("nocat");
215 namedArgs.remove("nocap");
216 namedArgs.remove("sc");
219 if (currentWord != null) {
220 if (name.equals("a")) {
222 currentWord.currentPronunciation = new StringBuilder();
223 currentWord.accentToPronunciation.put(positionalArgs.get(1), currentWord.currentPronunciation);
227 if (name.equals("IPA") || name.equals("SAMPA") || name.equals("X-SAMPA") || name.equals("enPR")) {
228 namedArgs.remove("lang");
229 for (int i = 0; i < 100 && !namedArgs.isEmpty(); ++i) {
230 final String pron = namedArgs.remove("" + i);
232 positionalArgs.add(pron);
239 if (!(positionalArgs.size() >= 2 && namedArgs.isEmpty())) {
240 System.err.println("Invalid pronunciation: " + positionalArgs.toString() + namedArgs.toString());
242 if (currentWord.currentPronunciation == null) {
243 currentWord.currentPronunciation = new StringBuilder();
244 currentWord.accentToPronunciation.put("", currentWord.currentPronunciation);
246 if (currentWord.currentPronunciation.length() > 0) {
247 currentWord.currentPronunciation.append("; ");
249 for (int i = 1; i < positionalArgs.size(); ++i) {
251 currentWord.currentPronunciation.append(",");
253 final String pron = wikiMarkup.matcher(positionalArgs.get(1)).replaceAll("");
254 currentWord.currentPronunciation.append(pron).append("");
256 currentWord.currentPronunciation.append(" (").append(name).append(")");
260 if (name.equals("qualifier")) {
261 //assertTrue(positionalArgs.size() == 2 && namedArgs.isEmpty() : positionalArgs.toString() + namedArgs.toString());
262 if (wikiBuilder == null) {
265 wikiBuilder.append(" (").append(positionalArgs.get(1)).append(")");
269 if (name.equals("...")) {
270 // Skipping any elided text for brevity.
271 wikiBuilder.append("...");
275 if (passThroughTemplates.contains(name)) {
276 assertTrue(positionalArgs.size() == 1 && namedArgs.isEmpty(), positionalArgs.toString() + namedArgs);
277 wikiBuilder.append(name);
281 if (ignoreTemplates.contains(name)) {
285 if ("Pronunciation".equals(currentHeading)) {
286 System.err.println("Unhandled pronunciation template: " + positionalArgs + namedArgs);
292 if (insidePartOfSpeech) {
295 if (name.equals("form of")) {
296 namedArgs.remove("sc");
297 if (positionalArgs.size() < 3 || positionalArgs.size() > 4) {
298 System.err.println("Invalid form of.");
300 final String token = positionalArgs.get(positionalArgs.size() == 3 ? 2 : 3);
301 final String grammarForm = WikiParser.simpleParse(positionalArgs.get(1));
302 currentPartOfSpeech.formOfs.add(new FormOf(grammarForm, token));
306 // The fallback plan: append the template!
307 if (wikiBuilder != null) {
308 wikiBuilder.append("{");
309 boolean first = true;
310 for (final String arg : positionalArgs) {
312 wikiBuilder.append(", ");
315 wikiBuilder.append(arg);
317 // This one isn't so useful.
318 for (final Map.Entry<String, String> entry : namedArgs.entrySet()) {
320 wikiBuilder.append(", ");
323 wikiBuilder.append(entry.getKey()).append("=").append(entry.getValue());
325 wikiBuilder.append("}");
328 //System.err.println("Unhandled part of speech template: " + positionalArgs + namedArgs);
334 if (name.equals("trans-top")) {
335 assertTrue(positionalArgs.size() >= 1 && namedArgs.isEmpty(), positionalArgs.toString() + namedArgs + title);
337 if (currentPartOfSpeech == null) {
338 assertTrue(currentWord != null && !currentWord.partsOfSpeech.isEmpty(), title);
339 System.err.println("Assuming last part of speech for non-nested translation section: " + title);
340 currentPartOfSpeech = ListUtil.getLast(currentWord.partsOfSpeech);
343 currentTranslationSense = new WikiWord.TranslationSense();
344 currentPartOfSpeech.translationSenses.add(currentTranslationSense);
345 if (positionalArgs.size() > 1) {
346 currentTranslationSense.sense = positionalArgs.get(1);
351 if (wikiBuilder == null) {
354 if (name.equals("m") || name.equals("f") || name.equals("n") || name.equals("c")) {
355 assertTrue(positionalArgs.size() >= 1 && namedArgs.isEmpty(), positionalArgs.toString() + namedArgs.toString());
356 wikiBuilder.append("{");
357 for (int i = 1; i < positionalArgs.size(); ++i) {
358 wikiBuilder.append(i > 1 ? "," : "");
359 wikiBuilder.append(positionalArgs.get(i));
361 wikiBuilder.append(name).append("}");
363 } else if (name.equals("p")) {
364 assertTrue(positionalArgs.size() == 1 && namedArgs.isEmpty());
365 wikiBuilder.append("pl.");
367 } else if (name.equals("s")) {
368 assertTrue(positionalArgs.size() == 1 && namedArgs.isEmpty() || title.equals("dobra"), title);
369 wikiBuilder.append("sg.");
371 } else if (grammarTemplates.contains(name)) {
372 assert positionalArgs.size() == 1 && namedArgs.isEmpty() : positionalArgs.toString() + namedArgs;
373 wikiBuilder.append(name).append(".");
375 } else if (name.equals("l")) {
376 // This template is designed to generate a link to a specific language-section on the target page.
377 wikiBuilder.append(positionalArgs.size() >= 4 ? positionalArgs.get(3) : positionalArgs.get(2));
379 } else if (name.equals("t") || name.equals("t+") || name.equals("t-") || name.equals("tø")) {
380 if (positionalArgs.size() > 2) {
381 wikiBuilder.append(positionalArgs.get(2));
383 for (int i = 3; i < positionalArgs.size(); ++i) {
384 wikiBuilder.append(i == 3 ? " {" : ",");
385 wikiBuilder.append(positionalArgs.get(i));
386 wikiBuilder.append(i == positionalArgs.size() - 1 ? "}" : "");
388 final String transliteration = namedArgs.remove("tr");
389 if (transliteration != null) {
390 wikiBuilder.append(" (").append(transliteration).append(")");
393 } else if (name.equals("trreq")) {
394 wikiBuilder.append("{{trreq}}");
396 } else if (name.equals("qualifier")) {
397 //assert positionalArgs.size() == 2 && namedArgs.isEmpty() : positionalArgs.toString() + namedArgs.toString();
398 wikiBuilder.append(" (").append(positionalArgs.get(1)).append(")");
400 } else if (useRemainingArgTemplates.contains(name)) {
401 for (int i = 1; i < positionalArgs.size(); ++i) {
403 wikiBuilder.append(", ");
405 wikiBuilder.append(positionalArgs.get(i));
407 } else if (ignoreTemplates.contains(name)) {
410 } else if (name.equals("initialism")) {
411 assert positionalArgs.size() <= 2 && namedArgs.isEmpty() : positionalArgs.toString() + namedArgs;
412 wikiBuilder.append("Initialism");
413 } else if (name.equals("abbreviation")) {
414 assert positionalArgs.size() <= 2 && namedArgs.isEmpty() : positionalArgs.toString() + namedArgs;
415 wikiBuilder.append("Abbreviation");
416 } else if (name.equals("acronym")) {
417 assert positionalArgs.size() <= 2 && namedArgs.isEmpty() : positionalArgs.toString() + namedArgs;
418 wikiBuilder.append("Acronym");
420 if (currentTranslationSense != null) {
421 System.err.println("Unhandled template: " + positionalArgs.toString() + namedArgs);
427 public void onText(String text) {
428 if (wikiBuilder != null) {
429 wikiBuilder.append(text);
435 public void onHeadingStart(int depth) {
436 wikiBuilder = new StringBuilder();
437 currentDepth = depth;
438 if (currentPartOfSpeech != null && depth <= currentPartOfSpeech.depth) {
439 currentPartOfSpeech = null;
440 insidePartOfSpeech = false;
442 if (currentWord != null && depth <= currentWord.depth) {
446 currentHeading = null;
450 public void onHeadingEnd(int depth) {
451 final String name = wikiBuilder.toString().trim();
453 currentTranslationSense = null;
454 currentHeading = name;
456 final boolean lang0 = langPatterns[0].matcher(name).matches();
457 final boolean lang1 = langPatterns[1].matcher(name).matches();
458 if (name.equalsIgnoreCase("English") || lang0 || lang1 || name.equalsIgnoreCase("Translingual")) {
459 currentWord = new WikiWord(title, depth);
460 if (lang0 && lang1) {
461 System.err.println("Word is indexed in both index1 and index2: " + title);
463 currentWord.language = name;
464 currentWord.index = lang0 ? 0 : (lang1 ? 1 : -1);
465 words.add(currentWord);
469 if (currentWord == null) {
473 if (currentPartOfSpeech != null && depth <= currentPartOfSpeech.depth) {
474 currentPartOfSpeech = null;
477 insidePartOfSpeech = false;
478 if (currentPartOfSpeech == null && partOfSpeechHeader.matcher(name).matches()) {
479 currentPartOfSpeech = new WikiWord.PartOfSpeech(depth, name);
480 currentWord.partsOfSpeech.add(currentPartOfSpeech);
481 insidePartOfSpeech = true;
485 if (name.equals("Translations")) {
486 if (currentWord == null ||
487 !currentWord.language.equals("English") ||
488 currentPartOfSpeech == null) {
489 System.err.println("Unexpected Translations section: " + title);
492 currentTranslationSense = new WikiWord.TranslationSense();
498 public void onListItemStart(String header, int[] section) {
499 wikiBuilder = new StringBuilder();
500 if (currentWord != null) {
501 currentWord.currentPronunciation = null;
507 public void onListItemEnd(String header, int[] section) {
508 String item = wikiBuilder.toString().trim();
509 if (item.length() == 0) {
512 item = WikiParser.simpleParse(item);
516 if (insidePartOfSpeech) {
517 assert currentPartOfSpeech != null : title + item;
518 if (header.equals("#") ||
519 header.equals("##") ||
520 header.equals("###") ||
521 header.equals("####") ||
522 header.equals(":#") ||
523 header.equals("::") ||
524 header.equals(":::*")) {
526 // :: should append, probably.
527 currentPartOfSpeech.newMeaning().meaning = item;
530 } else if (header.equals("#*") ||
531 header.equals("##*") ||
532 header.equals("###*")) {
533 currentPartOfSpeech.lastMeaning().newExample().source = item;
536 } else if (header.equals("#:") ||
537 header.equals("#*:") ||
538 header.equals("#:*") ||
539 header.equals("##:") ||
540 header.equals("##*:") ||
541 header.equals("#:*:") ||
542 header.equals("#:*#") ||
543 header.equals("#*:") ||
544 header.equals("*:") ||
545 header.equals("#:::") ||
546 header.equals("#**") ||
547 header.equals("#*:::") ||
548 header.equals("#:#") ||
549 header.equals(":::") ||
550 header.equals("##:*") ||
551 header.equals("###*:")) {
552 StringUtil.appendLine(currentPartOfSpeech.lastMeaning().newExample().example, item);
554 // Example in English
555 } else if (header.equals("#::") ||
556 header.equals("#*::") ||
557 header.equals("#:**") ||
558 header.equals("#*#") ||
559 header.equals("##*::")) {
560 StringUtil.appendLine(currentPartOfSpeech.lastMeaning().lastExample().exampleInEnglish, item);
563 } else if (header.equals("*") ||
564 header.equals("**") ||
565 header.equals("***") ||
566 header.equals("*#") ||
567 header.equals(":") ||
568 header.equals("::*") ||
569 header.equals("#**") ||
570 header.equals(":*") ||
571 header.equals("#*:*") ||
572 header.equals("#*:**") ||
573 header.equals("#*:#") ||
574 header.equals("#*:*:") ||
575 header.equals("#*:*") ||
576 header.equals(";")) {
577 // might have: * {{seeCites}}
578 // * [[w:Arabic numerals|Arabic numerals]]: 2
579 //assert item.trim().length() == 0;
580 System.err.println("Skipping meaning: " + header + " " + item);
582 if (title.equals("Yellowknife")) {
585 System.err.println("Busted heading: " + title + " "+ header + " " + item);
592 if (currentTranslationSense != null) {
593 if (item.indexOf("{{[trreq]{}}}") != -1) {
597 if (currentPartOfSpeech.translationSenses.isEmpty()) {
598 currentPartOfSpeech.translationSenses.add(currentTranslationSense);
601 final int colonPos = item.indexOf(':');
602 if (colonPos == -1) {
603 System.err.println("Invalid translation: title=" + title + ", item=" + item);
606 final String lang = item.substring(0, colonPos);
607 final String trans = item.substring(colonPos + 1).trim();
608 for (int i = 0; i < 2; ++i) {
609 if (langPatterns[i].matcher(lang).find()) {
610 currentTranslationSense.translations.get(i).add(new Translation(lang, trans));
617 public void onNewLine() {
621 public void onNewParagraph() {
624 // ----------------------------------------------------------------------
627 public void onComment(String text) {
631 public void onFormatBold(boolean boldOn) {
635 public void onFormatItalic(boolean italicOn) {
639 public void onUnterminated(String start, String rest) {
640 System.err.printf("OnUnterminated: %s %s %s\n", title, start, rest);
643 public void onInvalidHeaderEnd(String rest) {
644 throw new RuntimeException(rest);