import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collection;
-import java.util.Collections;
import java.util.LinkedHashSet;
import java.util.List;
import java.util.Map;
static final Logger LOG = Logger.getLogger(EnWiktionaryXmlParser.class.getName());
- // TODO: look for {{ and [[ and <adf> <!-- in output.
// TODO: process {{ttbc}} lines
static final Pattern partOfSpeechHeader = Pattern.compile(
// -------------------------------------------------------------------------
- String pos = null;
- int posDepth = -1;
-
private void doEnglishWord(String title, String text) {
+
+ String pos = null;
+ int posDepth = -1;
+
final WikiTokenizer wikiTokenizer = new WikiTokenizer(text);
while (wikiTokenizer.nextToken() != null) {
posDepth = wikiTokenizer.headingDepth();
pos = wikiTokenizer.headingWikiText();
} else if (headerName.equals("Translations")) {
- doTranslations(title, wikiTokenizer);
+ if (pos == null) {
+ LOG.warning("Translations without POS: " + title);
+ }
+ doTranslations(title, wikiTokenizer, pos);
} else if (headerName.equals("Pronunciation")) {
//doPronunciation(wikiLineReader);
}
"Jpan", "Kore", "Hebr", "rfscript", "Beng", "Mong", "Knda", "Cyrs",
"yue-tsj", "Mlym", "Tfng", "Grek", "yue-yue-j"));
- private void doTranslations(final String title, final WikiTokenizer wikiTokenizer) {
+ private void doTranslations(final String title, final WikiTokenizer wikiTokenizer, final String pos) {
if (title.equals("absolutely")) {
System.out.println();
}
sense = null;
} else if (functionName.equals("trans-mid")) {
} else if (functionName.equals("trans-see")) {
- // TODO
+ // TODO: would also be nice...
} else if (functionName.startsWith("picdic")) {
} else if (functionName.startsWith("checktrans")) {
} else if (functionName.startsWith("ttbc")) {
wikiTokenizer.nextLine();
- // TODO: would be great to handle
- //TODO: Check this: done = true;
+ // TODO: would be great to handle ttbc
+ // TODO: Check this: done = true;
} else {
LOG.warning("Unexpected translation wikifunction: " + wikiTokenizer.token() + ", title=" + title);
}
- } else if (wikiTokenizer.isListItem() && wikiTokenizer.listItemPrefix().startsWith("*")) {
+ } else if (wikiTokenizer.isListItem()) {
final String line = wikiTokenizer.listItemWikiText();
// This line could produce an output...
String rest = line.substring(colonIndex + 1).trim();
if (rest.length() > 0) {
- doTranslationLine(line, title, sense, rest);
+ doTranslationLine(line, title, pos, sense, rest);
} else {
- // TODO: do lines that are like Greek:
+ // TODO: do lines that are like "Greek:"
}
} else if (wikiTokenizer.remainderStartsWith("''See''")) {
return index < list.size() ? list.get(index) : null;
}
- private void doTranslationLine(final String line, final String title, final String sense, final String rest) {
+ private void doTranslationLine(final String line, final String title, final String pos, final String sense, final String rest) {
// Good chance we'll actually file this one...
final PairEntry pairEntry = new PairEntry();
final IndexedEntry indexedEntry = new IndexedEntry(pairEntry);
}
if (currentHeadingDepth > posDepth) {
- // TODO
+ // TODO: deal with other neat info sections
continue;
}
} finally {
// Here's where we exit.
- // TODO: Should we make an entry even if there are no foreign list items?
- if (foreignBuilder.indexOf(title) == -1) {
- foreignBuilder.insert(0, title + " ");
+ // Should we make an entry even if there are no foreign list items?
+ String foreign = foreignBuilder.toString().trim();
+ if (!foreign.toLowerCase().startsWith(title.toLowerCase())) {
+ foreign = title + " " + foreign;
}
for (final ListSection listSection : listSections) {
- doForeignListItem(foreignBuilder.toString(), title, wordForms, listSection);
+ doForeignListItem(foreign, title, wordForms, listSection);
}
}
}
englishBuilder.append(text);
otherIndexBuilder.addEntryWithString(indexedEntry, text, EntryTypeName.WIKTIONARY_ENGLISH_DEF_OTHER_LANG);
} else if (link.equals("plural")) {
- englishBuilder.append(englishTokenizer.wikiLinkText());
+ englishBuilder.append(text);
} else {
//LOG.warning("Special link: " + englishTokenizer.token());
- englishBuilder.append(englishTokenizer.wikiLinkText());
+ enIndexBuilder.addEntryWithString(indexedEntry, text, EntryTypeName.WIKTIONARY_ENGLISH_DEF_WIKI_LINK);
+ englishBuilder.append(text);
}
} else {
// link == null
}
}
}
+
final String english = trim(englishBuilder.toString());
if (english.length() > 0) {
final Pair pair = new Pair(english, trim(foreignText), this.swap);
otherIndexBuilder.addEntryWithString(indexedEntry, form, EntryTypeName.WIKTIONARY_FORM_SINGLE, EntryTypeName.WIKTIONARY_FORM_MULTI);
}
}
+
+ // Do examples.
+ String lastForeign = null;
+ for (int i = 0; i < listSection.nextPrefixes.size(); ++i) {
+ final String nextPrefix = listSection.nextPrefixes.get(i);
+ final String nextLine = listSection.nextLines.get(i);
+ int dash = nextLine.indexOf("—");
+ int mdashLen = 7;
+ if (dash == -1) {
+ dash = nextLine.indexOf("—");
+ mdashLen = 1;
+ }
+ if (dash == -1) {
+ dash = nextLine.indexOf(" - ");
+ mdashLen = 3;
+ }
+
+ if ((nextPrefix.equals("#:") || nextPrefix.equals("##:")) && dash != -1) {
+ final String foreignEx = nextLine.substring(0, dash);
+ final String englishEx = nextLine.substring(dash + mdashLen);
+ final Pair pair = new Pair(formatAndIndexExampleString(englishEx, enIndexBuilder, indexedEntry), formatAndIndexExampleString(foreignEx, otherIndexBuilder, indexedEntry), swap);
+ if (pair.lang1 != "--" && pair.lang1 != "--") {
+ pairEntry.pairs.add(pair);
+ }
+ lastForeign = null;
+ } else if (nextPrefix.equals("#:") || nextPrefix.equals("##:")){
+ final Pair pair = new Pair("--", formatAndIndexExampleString(nextLine, null, indexedEntry), swap);
+ lastForeign = nextLine;
+ if (pair.lang1 != "--" && pair.lang1 != "--") {
+ pairEntry.pairs.add(pair);
+ }
+ } else if (nextPrefix.equals("#::") || nextPrefix.equals("#**")) {
+ if (lastForeign != null) {
+ pairEntry.pairs.remove(pairEntry.pairs.size() - 1);
+ final Pair pair = new Pair(formatAndIndexExampleString(nextLine, enIndexBuilder, indexedEntry), formatAndIndexExampleString(lastForeign, otherIndexBuilder, indexedEntry), swap);
+ if (pair.lang1 != "--" && pair.lang1 != "--") {
+ pairEntry.pairs.add(pair);
+ }
+ } else {
+ LOG.warning("English example with no foreign: " + title + ", " + nextLine);
+ }
+ } else if (nextPrefix.equals("#*")) {
+ // Can't really index these.
+ final Pair pair = new Pair("--", formatAndIndexExampleString(nextLine, null, indexedEntry), swap);
+ lastForeign = nextLine;
+ if (pair.lang1 != "--" && pair.lang1 != "--") {
+ pairEntry.pairs.add(pair);
+ }
+ } else if (nextPrefix.equals("#::*") || nextPrefix.equals("##") || nextPrefix.equals("#*:") || nextPrefix.equals("#:*") || true) {
+ final Pair pair = new Pair("--", formatAndIndexExampleString(nextLine, null, indexedEntry), swap);
+ if (pair.lang1 != "--" && pair.lang1 != "--") {
+ pairEntry.pairs.add(pair);
+ }
+// } else {
+// assert false;
+ }
+ }
+ }
+
+ private String formatAndIndexExampleString(final String example, final IndexBuilder indexBuilder, final IndexedEntry indexedEntry) {
+ final WikiTokenizer wikiTokenizer = new WikiTokenizer(example, false);
+ final StringBuilder builder = new StringBuilder();
+ boolean insideTripleQuotes = false;
+ while (wikiTokenizer.nextToken() != null) {
+ if (wikiTokenizer.isPlainText()) {
+ builder.append(wikiTokenizer.token());
+ if (indexBuilder != null) {
+ indexBuilder.addEntryWithString(indexedEntry, wikiTokenizer.token(), EntryTypeName.WIKTIONARY_EXAMPLE);
+ }
+ } else if (wikiTokenizer.isWikiLink()) {
+ final String text = wikiTokenizer.wikiLinkText().replaceAll("'", "");
+ builder.append(text);
+ if (indexBuilder != null) {
+ indexBuilder.addEntryWithString(indexedEntry, text, EntryTypeName.WIKTIONARY_EXAMPLE);
+ }
+ } else if (wikiTokenizer.isFunction()) {
+ builder.append(wikiTokenizer.token());
+ } else if (wikiTokenizer.isMarkup()) {
+ if (wikiTokenizer.token().equals("'''")) {
+ insideTripleQuotes = !insideTripleQuotes;
+ }
+ } else if (wikiTokenizer.isComment() || wikiTokenizer.isNewline()) {
+ // Do nothing.
+ } else {
+ LOG.warning("unexpected token: " + wikiTokenizer.token());
+ }
+ }
+ final String result = trim(builder.toString());
+ return result.length() > 0 ? result : "--";
}
projects / DictionaryPC.git / blobdiff