// The matches the whole line, otherwise regexes don't work well on French:
// {{=uk=}}
+ // Spanish has no initial headings, tried to also detect {{ES as such
+ // with "^(\\{\\{ES|(=+)[^=]).*$" but that broke English.
static final Pattern headingStart = Pattern.compile("^(=+)[^=].*$", Pattern.MULTILINE);
final Map<String,List<Selector>> pathToSelectors = new LinkedHashMap<String, List<Selector>>();
private WiktionarySplitter() {
List<Selector> selectors;
for (final String code : WiktionaryLangs.wikiCodeToIsoCodeToWikiName.keySet()) {
- //if (code.equals("en") || code.equals("de") || code.equals("fr")) {continue;}
+ //if (!code.equals("fr")) {continue;}
selectors = new ArrayList<WiktionarySplitter.Selector>();
pathToSelectors.put(String.format("data/inputs/%swiktionary-pages-articles.xml", code), selectors);
for (final Map.Entry<String, String> entry : WiktionaryLangs.wikiCodeToIsoCodeToWikiName.get(code).entrySet()) {
title.startsWith("Thread:") ||
title.startsWith("Template:") ||
title.startsWith("Summary:") ||
+ title.startsWith("Module:") ||
// DE
title.startsWith("Datei:") ||
title.startsWith("Verzeichnis:") ||
title.startsWith("Thesaurus:") ||
title.startsWith("Kategorie:") ||
title.startsWith("Hilfe:") ||
+ title.startsWith("Reim:") ||
// FR:
title.startsWith("Annexe:") ||
title.startsWith("Catégori:") ||
title.startsWith("Categoria:") ||
title.startsWith("Aiuto:") ||
title.startsWith("Portail:") ||
+ // ES
+ title.startsWith("Apéndice:") ||
+ title.startsWith("Archivo:") ||
+ title.startsWith("Ayuda:") ||
+ title.startsWith("Categoría:") ||
+ title.startsWith("Plantilla:") ||
+ title.startsWith("Wikcionario:") ||
// sentinel
false
end = text.length();
}
- final String sectionText = text.substring(0, end);
+ String sectionText = text.substring(0, end);
+ // Hack to remove empty dummy section from French
+ if (sectionText.startsWith("\n=== {{S|étymologie}} ===\n: {{ébauche-étym"))
+ {
+ int dummy_end = sectionText.indexOf("}}", 41) + 2;
+ while (dummy_end + 1 < sectionText.length() &&
+ sectionText.charAt(dummy_end) == '\n' &&
+ sectionText.charAt(dummy_end + 1) == '\n') ++dummy_end;
+ sectionText = sectionText.substring(dummy_end);
+ }
final Section section = new Section(title, heading, sectionText);
try {
}
text = text.substring(end);
+ break;
}
}
}
}
}
-
public void parse(final File file) throws ParserConfigurationException,
SAXException, IOException {
final SAXParser parser = SAXParserFactoryImpl.newInstance().newSAXParser();
parser.parse(file, this);
}
-
-
}