- if (selectors.isEmpty()) {
- selectors.addAll(Arrays.asList(
- new Selector("wikiSplit/arabic.data", ".*[Ar]rabic.*"),
- new Selector("wikiSplit/croation.data", ".*[Cc]roation.*"),
- new Selector("wikiSplit/czech.data", ".*[Cc]zech.*"),
- new Selector("wikiSplit/mandarin.data", ".*[Mm]andarin|[Cc]hinese.*"),
- new Selector("wikiSplit/dutch.data", ".*[Du]utch.*"),
- new Selector("wikiSplit/english.data", ".*[Ee]nglish.*"),
- new Selector("wikiSplit/french.data", ".*[Ff]rench.*"),
- new Selector("wikiSplit/german.data", ".*[Gg]erman.*"),
- new Selector("wikiSplit/greek.data", ".*[Gg]reek.*"),
- new Selector("wikiSplit/hindi.data", ".*[Hh]indi.*"),
- new Selector("wikiSplit/italian.data", ".*[Ii]talian.*"),
- new Selector("wikiSplit/japanese.data", ".*[Jj]apanese.*"),
- new Selector("wikiSplit/korean.data", ".*[Kk]orean.*"),
- new Selector("wikiSplit/persian.data", ".*[Pp]ersian.*"),
- new Selector("wikiSplit/portuguese.data", ".*[Pp]ortuguese.*"),
- new Selector("wikiSplit/romanian.data", ".*[Rr]omanian.*"),
- new Selector("wikiSplit/russian.data", ".*[Rr]ussian.*"),
- new Selector("wikiSplit/spanish.data", ".*[Ss]panish.*"),
- new Selector("wikiSplit/swedish.data", ".*[Ss]wedish.*"),
- new Selector("wikiSplit/thai.data", ".*[Tt]hai.*"),
- new Selector("wikiSplit/vietnamese.data", ".*[Vv]ietnamese.*")
- ));
+ private void endPage() {
+ final String title = titleBuilder.toString();
+ lastPageTitle = title;
+ if (++pageCount % 1000 == 0) {
+ System.out.println("endPage: " + title + ", count=" + pageCount);
+ }
+ if (title.startsWith("Wiktionary:") ||
+ title.startsWith("Appendix:") ||
+ title.startsWith("Help:") ||
+ title.startsWith("Index:") ||
+ title.startsWith("MediaWiki:") ||
+ title.startsWith("Citations:") ||
+ title.startsWith("Concordance:") ||
+ title.startsWith("Glossary:") ||
+ title.startsWith("Rhymes:") ||
+ title.startsWith("Category:") ||
+ title.startsWith("Wikisaurus:") ||
+ title.startsWith("Unsupported titles/") ||
+ title.startsWith("Transwiki:") ||
+ title.startsWith("File:") ||
+ title.startsWith("Thread:") ||
+ title.startsWith("Template:") ||
+ title.startsWith("Summary:") ||
+ title.startsWith("Module:") ||
+ // DE
+ title.startsWith("Datei:") ||
+ title.startsWith("Verzeichnis:") ||
+ title.startsWith("Vorlage:") ||
+ title.startsWith("Thesaurus:") ||
+ title.startsWith("Kategorie:") ||
+ title.startsWith("Hilfe:") ||
+ title.startsWith("Reim:") ||
+ // FR:
+ title.startsWith("Annexe:") ||
+ title.startsWith("Catégori:") ||
+ title.startsWith("Modèle:") ||
+ title.startsWith("Thésaurus:") ||
+ title.startsWith("Projet:") ||
+ title.startsWith("Aide:") ||
+ title.startsWith("Fichier:") ||
+ title.startsWith("Wiktionnaire:") ||
+ title.startsWith("Catégorie:") ||
+ title.startsWith("Portail:") ||
+ title.startsWith("utiliusateur:") ||
+ title.startsWith("Kategorio:") ||
+ // IT
+ title.startsWith("Wikizionario:") ||
+ title.startsWith("Appendice:") ||
+ title.startsWith("Categoria:") ||
+ title.startsWith("Aiuto:") ||
+ title.startsWith("Portail:") ||
+ // ES
+ title.startsWith("Apéndice:") ||
+ title.startsWith("Archivo:") ||
+ title.startsWith("Ayuda:") ||
+ title.startsWith("Categoría:") ||
+ title.startsWith("Plantilla:") ||
+ title.startsWith("Wikcionario:") ||
+
+ // sentinel
+ false
+ ) {
+ return;
+ }
+ if (title.contains(":")) {
+ if (!title.startsWith("Sign gloss:")) {
+ System.err.println("title with colon: " + title);
+ }
+ }
+
+ String text = textBuilder.toString();
+ String translingual = "";
+
+ while (text.length() > 0) {
+ // Find start.
+ final Matcher startMatcher = headingStart.matcher(text);
+ if (!startMatcher.find()) {
+ return;
+ }
+ text = text.substring(startMatcher.end());
+
+ final String heading = startMatcher.group();
+ for (final Selector selector : currentSelectors) {
+ if (heading.indexOf("Translingual") != -1) {
+ // Find end.
+ final int depth = startMatcher.group(1).length();
+ final Pattern endPattern = getEndPattern(depth);
+
+ final Matcher endMatcher = endPattern.matcher(text);
+ if (endMatcher.find()) {
+ int end = endMatcher.start();
+ translingual = text.substring(0, endMatcher.start());
+ text = text.substring(end);
+ break;
+ }
+ }
+ if (selector.pattern.matcher(heading).find()) {
+
+ // Find end.
+ final int depth = startMatcher.group(1).length();
+ final Pattern endPattern = getEndPattern(depth);
+
+ final Matcher endMatcher = endPattern.matcher(text);
+ final int end;
+ if (endMatcher.find()) {
+ end = endMatcher.start();
+ } else {
+ end = text.length();
+ }
+
+ String sectionText = text.substring(0, end);
+ // Hack to remove empty dummy section from French
+ if (sectionText.startsWith("\n=== {{S|étymologie}} ===\n: {{ébauche-étym")) {
+ int dummy_end = sectionText.indexOf("}}", 41) + 2;
+ while (dummy_end + 1 < sectionText.length() &&
+ sectionText.charAt(dummy_end) == '\n' &&
+ sectionText.charAt(dummy_end + 1) == '\n') ++dummy_end;
+ sectionText = sectionText.substring(dummy_end);
+ }
+ if (heading.indexOf("Japanese") == -1) sectionText += translingual;
+ final Section section = new Section(title, heading, sectionText);
+
+ try {
+ selector.out.writeUTF(section.title);
+ selector.out.writeUTF(section.heading);
+ final byte[] bytes = section.text.getBytes("UTF8");
+ selector.out.writeInt(bytes.length);
+ selector.out.write(bytes);
+ } catch (IOException e) {
+ throw new RuntimeException(e);
+ }
+
+ text = text.substring(end);
+ break;
+ }
+ }
+ }
+