import javax.xml.parsers.ParserConfigurationException;
import javax.xml.parsers.SAXParser;
-import javax.xml.parsers.SAXParserFactory;
+import org.apache.xerces.jaxp.SAXParserFactoryImpl;
import org.xml.sax.Attributes;
import org.xml.sax.SAXException;
-import com.hughes.android.dictionary.parser.enwiktionary.WiktionaryLangs;
+import com.hughes.android.dictionary.parser.wiktionary.WiktionaryLangs;
public class WiktionarySplitter extends org.xml.sax.helpers.DefaultHandler {
// The matches the whole line, otherwise regexes don't work well on French:
// {{=uk=}}
+ // Spanish has no initial headings, tried to also detect {{ES as such
+ // with "^(\\{\\{ES|(=+)[^=]).*$" but that broke English.
static final Pattern headingStart = Pattern.compile("^(=+)[^=].*$", Pattern.MULTILINE);
final Map<String,List<Selector>> pathToSelectors = new LinkedHashMap<String, List<Selector>>();
StringBuilder textBuilder;
StringBuilder currentBuilder = null;
- public static void main(final String[] args) throws SAXException, IOException, ParserConfigurationException {
+ public static void main(final String[] args) throws Exception {
final WiktionarySplitter wiktionarySplitter = new WiktionarySplitter();
wiktionarySplitter.go();
}
}
}
- private void go() throws ParserConfigurationException, SAXException, IOException {
- final SAXParser parser = SAXParserFactory.newInstance().newSAXParser();
+ private void go() throws Exception {
+ final SAXParser parser = SAXParserFactoryImpl.newInstance().newSAXParser();
// Configure things.
for (final Map.Entry<String, List<Selector>> pathToSelectorsEntry : pathToSelectors.entrySet()) {
}
// Do it.
- parser.parse(new File(pathToSelectorsEntry.getKey()), this);
+ try {
+ parser.parse(new File(pathToSelectorsEntry.getKey()), this);
+ } catch (Exception e) {
+ System.err.println("Exception during parse, lastPageTitle=" + lastPageTitle + ", titleBuilder=" + titleBuilder.toString());
+ throw e;
+ }
// Shutdown.
for (final Selector selector : currentSelectors) {
}
}
+ String lastPageTitle = null;
int pageCount = 0;
private void endPage() {
final String title = titleBuilder.toString();
+ lastPageTitle = title;
if (++pageCount % 1000 == 0) {
System.out.println("endPage: " + title + ", count=" + pageCount);
}
+ if (title.startsWith("Wiktionary:") ||
+ title.startsWith("Appendix:") ||
+ title.startsWith("Help:") ||
+ title.startsWith("Index:") ||
+ title.startsWith("MediaWiki:") ||
+ title.startsWith("Citations:") ||
+ title.startsWith("Concordance:") ||
+ title.startsWith("Glossary:") ||
+ title.startsWith("Rhymes:") ||
+ title.startsWith("Category:") ||
+ title.startsWith("Wikisaurus:") ||
+ title.startsWith("Unsupported titles/") ||
+ title.startsWith("Transwiki:") ||
+ title.startsWith("File:") ||
+ title.startsWith("Thread:") ||
+ title.startsWith("Template:") ||
+ title.startsWith("Summary:") ||
+ title.startsWith("Module:") ||
+ // DE
+ title.startsWith("Datei:") ||
+ title.startsWith("Verzeichnis:") ||
+ title.startsWith("Vorlage:") ||
+ title.startsWith("Thesaurus:") ||
+ title.startsWith("Kategorie:") ||
+ title.startsWith("Hilfe:") ||
+ title.startsWith("Reim:") ||
+ // FR:
+ title.startsWith("Annexe:") ||
+ title.startsWith("Catégori:") ||
+ title.startsWith("Modèle:") ||
+ title.startsWith("Thésaurus:") ||
+ title.startsWith("Projet:") ||
+ title.startsWith("Aide:") ||
+ title.startsWith("Fichier:") ||
+ title.startsWith("Wiktionnaire:") ||
+ title.startsWith("Catégorie:") ||
+ title.startsWith("Portail:") ||
+ title.startsWith("utiliusateur:") ||
+ title.startsWith("Kategorio:") ||
+ // IT
+ title.startsWith("Wikizionario:") ||
+ title.startsWith("Appendice:") ||
+ title.startsWith("Categoria:") ||
+ title.startsWith("Aiuto:") ||
+ title.startsWith("Portail:") ||
+ // ES
+ title.startsWith("Apéndice:") ||
+ title.startsWith("Archivo:") ||
+ title.startsWith("Ayuda:") ||
+ title.startsWith("Categoría:") ||
+ title.startsWith("Plantilla:") ||
+ title.startsWith("Wikcionario:") ||
+
+ // sentinel
+ false
+ ) {
+ return;
+ }
+ if (title.contains(":")) {
+ if (!title.startsWith("Sign gloss:")) {
+ System.err.println("title with colon: " + title);
+ }
+ }
String text = textBuilder.toString();
end = text.length();
}
- final String sectionText = text.substring(0, end);
+ String sectionText = text.substring(0, end);
+ // Hack to remove empty dummy section from French
+ if (sectionText.startsWith("\n=== {{S|étymologie}} ===\n: {{ébauche-étym"))
+ {
+ int dummy_end = sectionText.indexOf("}}", 41) + 2;
+ while (dummy_end + 1 < sectionText.length() &&
+ sectionText.charAt(dummy_end) == '\n' &&
+ sectionText.charAt(dummy_end + 1) == '\n') ++dummy_end;
+ sectionText = sectionText.substring(dummy_end);
+ }
final Section section = new Section(title, heading, sectionText);
try {
}
}
-
public void parse(final File file) throws ParserConfigurationException,
SAXException, IOException {
- final SAXParser parser = SAXParserFactory.newInstance().newSAXParser();
+ final SAXParser parser = SAXParserFactoryImpl.newInstance().newSAXParser();
parser.parse(file, this);
}
-
-
}