- if (selectors.isEmpty()) {
- selectors.addAll(Arrays.asList(
- new Selector("../DictionaryData/inputs/enWikiSplit/AF.data", ".*[Aa]frikaans.*"),
- new Selector("../DictionaryData/inputs/enWikiSplit/AR.data", ".*[Aa]rabic.*"),
- new Selector("../DictionaryData/inputs/enWikiSplit/HY.data", ".*[Aa]rmenian.*"),
- new Selector("../DictionaryData/inputs/enWikiSplit/HR.data", ".*[Cc]roation.*"),
- new Selector("../DictionaryData/inputs/enWikiSplit/CS.data", ".*[Cc]zech.*"),
- new Selector("../DictionaryData/inputs/enWikiSplit/ZH.data", ".*[Mm]andarin|[Cc]hinese.*"),
- new Selector("../DictionaryData/inputs/enWikiSplit/NL.data", ".*[Du]utch.*"),
- new Selector("../DictionaryData/inputs/enWikiSplit/EN.data", ".*[Ee]nglish.*"),
- new Selector("../DictionaryData/inputs/enWikiSplit/FI.data", ".*[Ff]inish.*"),
- new Selector("../DictionaryData/inputs/enWikiSplit/FR.data", ".*[Ff]rench.*"),
- new Selector("../DictionaryData/inputs/enWikiSplit/DE.data", ".*[Gg]erman.*"),
- new Selector("../DictionaryData/inputs/enWikiSplit/EL.data", ".*[Gg]reek.*"),
- new Selector("../DictionaryData/inputs/enWikiSplit/HE.data", ".*[Hh]ewbrew.*"),
- new Selector("../DictionaryData/inputs/enWikiSplit/HI.data", ".*[Hh]indi.*"),
- new Selector("../DictionaryData/inputs/enWikiSplit/IS.data", ".*[Ii]celandic.*"),
- new Selector("../DictionaryData/inputs/enWikiSplit/GA.data", ".*[Ii]rish.*"),
- new Selector("../DictionaryData/inputs/enWikiSplit/IT.data", ".*[Ii]talian.*"),
- new Selector("../DictionaryData/inputs/enWikiSplit/LT.data", ".*[Ll]ithuanian.*"),
- new Selector("../DictionaryData/inputs/enWikiSplit/JA.data", ".*[Jj]apanese.*"),
- new Selector("../DictionaryData/inputs/enWikiSplit/KO.data", ".*[Kk]orean.*"),
- new Selector("../DictionaryData/inputs/enWikiSplit/KU.data", ".*[Kk]urdish.*"),
- new Selector("../DictionaryData/inputs/enWikiSplit/MS.data", ".*[Mm]alay.*"),
- new Selector("../DictionaryData/inputs/enWikiSplit/MI.data", ".*[Mm]aori.*"),
- new Selector("../DictionaryData/inputs/enWikiSplit/MN.data", ".*[Mm]ongolian.*"),
- new Selector("../DictionaryData/inputs/enWikiSplit/NO.data", ".*[Nn]orwegian.*"),
- new Selector("../DictionaryData/inputs/enWikiSplit/FA.data", ".*[Pp]ersian.*"),
- new Selector("../DictionaryData/inputs/enWikiSplit/PT.data", ".*[Pp]ortuguese.*"),
- new Selector("../DictionaryData/inputs/enWikiSplit/RO.data", ".*[Rr]omanian.*"),
- new Selector("../DictionaryData/inputs/enWikiSplit/RU.data", ".*[Rr]ussian.*"),
- new Selector("../DictionaryData/inputs/enWikiSplit/SA.data", ".*[Ss]anskrit.*"),
- new Selector("../DictionaryData/inputs/enWikiSplit/SR.data", ".*[Ss]erbian.*"),
- new Selector("../DictionaryData/inputs/enWikiSplit/SO.data", ".*[Ss]omali.*"),
- new Selector("../DictionaryData/inputs/enWikiSplit/ES.data", ".*[Ss]panish.*"),
- new Selector("../DictionaryData/inputs/enWikiSplit/SU.data", ".*[Ss]udanese.*"),
- new Selector("../DictionaryData/inputs/enWikiSplit/SV.data", ".*[Ss]wedish.*"),
- new Selector("../DictionaryData/inputs/enWikiSplit/TG.data", ".*[Tt]ajik.*"),
- new Selector("../DictionaryData/inputs/enWikiSplit/TH.data", ".*[Tt]hai.*"),
- new Selector("../DictionaryData/inputs/enWikiSplit/BO.data", ".*[Tt]ibetan.*"),
- new Selector("../DictionaryData/inputs/enWikiSplit/TR.data", ".*[Tt]urkish.*"),
- new Selector("../DictionaryData/inputs/enWikiSplit/UK.data", ".*[Uu]kranian.*"),
- new Selector("../DictionaryData/inputs/enWikiSplit/VI.data", ".*[Vv]ietnamese.*"),
- new Selector("../DictionaryData/inputs/enWikiSplit/CI.data", ".*[Ww]elsh.*"),
- new Selector("../DictionaryData/inputs/enWikiSplit/YI.data", ".*[Yy]iddish.*"),
- new Selector("../DictionaryData/inputs/enWikiSplit/ZU.data", ".*[Zz]ulu.*")
- ));
+ String lastPageTitle = null;
+ int pageCount = 0;
+ private void endPage() {
+ final String title = titleBuilder.toString();
+ lastPageTitle = title;
+ if (++pageCount % 1000 == 0) {
+ System.out.println("endPage: " + title + ", count=" + pageCount);
+ }
+ if (title.startsWith("Wiktionary:") ||
+ title.startsWith("Appendix:") ||
+ title.startsWith("Help:") ||
+ title.startsWith("Index:") ||
+ title.startsWith("MediaWiki:") ||
+ title.startsWith("Citations:") ||
+ title.startsWith("Concordance:") ||
+ title.startsWith("Glossary:") ||
+ title.startsWith("Rhymes:") ||
+ title.startsWith("Category:") ||
+ title.startsWith("Wikisaurus:") ||
+ title.startsWith("Unsupported titles/") ||
+ title.startsWith("Transwiki:") ||
+ title.startsWith("File:") ||
+ title.startsWith("Thread:") ||
+ title.startsWith("Template:") ||
+ title.startsWith("Summary:") ||
+ title.startsWith("Module:") ||
+ // DE
+ title.startsWith("Datei:") ||
+ title.startsWith("Verzeichnis:") ||
+ title.startsWith("Vorlage:") ||
+ title.startsWith("Thesaurus:") ||
+ title.startsWith("Kategorie:") ||
+ title.startsWith("Hilfe:") ||
+ title.startsWith("Reim:") ||
+ // FR:
+ title.startsWith("Annexe:") ||
+ title.startsWith("Catégori:") ||
+ title.startsWith("Modèle:") ||
+ title.startsWith("Thésaurus:") ||
+ title.startsWith("Projet:") ||
+ title.startsWith("Aide:") ||
+ title.startsWith("Fichier:") ||
+ title.startsWith("Wiktionnaire:") ||
+ title.startsWith("Catégorie:") ||
+ title.startsWith("Portail:") ||
+ title.startsWith("utiliusateur:") ||
+ title.startsWith("Kategorio:") ||
+ // IT
+ title.startsWith("Wikizionario:") ||
+ title.startsWith("Appendice:") ||
+ title.startsWith("Categoria:") ||
+ title.startsWith("Aiuto:") ||
+ title.startsWith("Portail:") ||
+ // ES
+ title.startsWith("Apéndice:") ||
+ title.startsWith("Archivo:") ||
+ title.startsWith("Ayuda:") ||
+ title.startsWith("Categoría:") ||
+ title.startsWith("Plantilla:") ||
+ title.startsWith("Wikcionario:") ||
+
+ // sentinel
+ false
+ ) {
+ return;
+ }
+ if (title.contains(":")) {
+ if (!title.startsWith("Sign gloss:")) {
+ System.err.println("title with colon: " + title);
+ }
+ }
+
+ String text = textBuilder.toString();
+ String translingual = "";
+
+ while (text.length() > 0) {
+ // Find start.
+ final Matcher startMatcher = headingStart.matcher(text);
+ if (!startMatcher.find()) {
+ return;
+ }
+ text = text.substring(startMatcher.end());
+
+ final String heading = startMatcher.group();
+ for (final Selector selector : currentSelectors) {
+ if (heading.indexOf("Translingual") != -1) {
+ // Find end.
+ final int depth = startMatcher.group(1).length();
+ final Pattern endPattern = Pattern.compile(String.format("^={1,%d}[^=].*$", depth), Pattern.MULTILINE);
+
+ final Matcher endMatcher = endPattern.matcher(text);
+ if (endMatcher.find()) {
+ int end = endMatcher.start();
+ translingual = text.substring(0, endMatcher.start());
+ text = text.substring(end);
+ break;
+ }
+ }
+ if (selector.pattern.matcher(heading).find()) {
+
+ // Find end.
+ final int depth = startMatcher.group(1).length();
+ final Pattern endPattern = Pattern.compile(String.format("^={1,%d}[^=].*$", depth), Pattern.MULTILINE);
+
+ final Matcher endMatcher = endPattern.matcher(text);
+ final int end;
+ if (endMatcher.find()) {
+ end = endMatcher.start();
+ } else {
+ end = text.length();
+ }
+
+ String sectionText = text.substring(0, end);
+ // Hack to remove empty dummy section from French
+ if (sectionText.startsWith("\n=== {{S|étymologie}} ===\n: {{ébauche-étym")) {
+ int dummy_end = sectionText.indexOf("}}", 41) + 2;
+ while (dummy_end + 1 < sectionText.length() &&
+ sectionText.charAt(dummy_end) == '\n' &&
+ sectionText.charAt(dummy_end + 1) == '\n') ++dummy_end;
+ sectionText = sectionText.substring(dummy_end);
+ }
+ if (heading.indexOf("Japanese") == -1) sectionText += translingual;
+ final Section section = new Section(title, heading, sectionText);
+
+ try {
+ selector.out.writeUTF(section.title);
+ selector.out.writeUTF(section.heading);
+ final byte[] bytes = section.text.getBytes("UTF8");
+ selector.out.writeInt(bytes.length);
+ selector.out.write(bytes);
+ } catch (IOException e) {
+ throw new RuntimeException(e);
+ }
+
+ text = text.substring(end);
+ break;
+ }
+ }
+ }
+