initialism and acronym.

[DictionaryPC.git] / src / com / hughes / android / dictionary / parser / EnWiktionaryXmlParser.java

diff --git a/src/com/hughes/android/dictionary/parser/EnWiktionaryXmlParser.java b/src/com/hughes/android/dictionary/parser/EnWiktionaryXmlParser.java
index 600c6e7b8056fa92bcf0635857530e5f5b60f9b3..f985084e9b031b6e8dee8a88f69042cfb2f50b3b 100644 (file)
--- a/src/com/hughes/android/dictionary/parser/EnWiktionaryXmlParser.java
+++ b/src/com/hughes/android/dictionary/parser/EnWiktionaryXmlParser.java
@@ -46,7 +46,7 @@ public class EnWiktionaryXmlParser {
       "Noun|Verb|Adjective|Adverb|Pronoun|Conjunction|Interjection|" +
       "Preposition|Proper noun|Article|Prepositional phrase|Acronym|" +
       "Abbreviation|Initialism|Contraction|Prefix|Suffix|Symbol|Letter|" +
-      "Ligature|Idiom|Phrase|" +
+      "Ligature|Idiom|Phrase|\\{\\{acronym\\}\\}|\\{\\{initialism\\}\\}|" +
       // These are @deprecated:
       "Noun form|Verb form|Adjective form|Nominal phrase|Noun phrase|" +
       "Verb phrase|Transitive verb|Intransitive verb|Reflexive verb|" +
@@ -165,9 +165,10 @@ public class EnWiktionaryXmlParser {
   
   private void doTranslations(final String title, final WikiTokenizer wikiTokenizer, final String pos) {
     if (title.equals("absolutely")) {
-      System.out.println();
+      //System.out.println();
     }
     
+    String topLevelLang = null;
     String sense = null;
     boolean done = false;
     while (wikiTokenizer.nextToken() != null) {
@@ -181,8 +182,6 @@ public class EnWiktionaryXmlParser {
       
       // Check whether we care about this line:
       
-      //line = WikiLineReader.removeSquareBrackets(line);
-      
       if (wikiTokenizer.isFunction()) {
         final String functionName = wikiTokenizer.functionName();
         final List<String> positionArgs = wikiTokenizer.functionPositionArgs();
@@ -202,6 +201,7 @@ public class EnWiktionaryXmlParser {
           // TODO: would also be nice...
         } else if (functionName.startsWith("picdic")) {
         } else if (functionName.startsWith("checktrans")) {
+          done = true;
         } else if (functionName.startsWith("ttbc")) {
           wikiTokenizer.nextLine();
           // TODO: would be great to handle ttbc
@@ -213,6 +213,10 @@ public class EnWiktionaryXmlParser {
         final String line = wikiTokenizer.listItemWikiText();
         // This line could produce an output...
         
+        if (line.contains("ich hoan dich gear")) {
+          //System.out.println();
+        }
+        
         // First strip the language and check whether it matches.
         // And hold onto it for sub-lines.
         final int colonIndex = line.indexOf(":");
@@ -220,16 +224,28 @@ public class EnWiktionaryXmlParser {
           continue;
         }
         
-        final String lang = line.substring(0, colonIndex);
-        if (!this.langPattern.matcher(lang).find()) {
+        final String lang = trim(WikiTokenizer.toPlainText(line.substring(0, colonIndex)));
+        final boolean appendLang;
+        if (wikiTokenizer.listItemPrefix().length() == 1) {
+          topLevelLang = lang;
+          final boolean thisFind = langPattern.matcher(lang).find();
+          if (!thisFind) {
+            continue;
+          }
+          appendLang = !langPattern.matcher(lang).matches();
+        } else if (topLevelLang == null) {
           continue;
+        } else {
+          // Two-level -- the only way we won't append is if this second level matches exactly.
+          if (!langPattern.matcher(lang).matches() && !langPattern.matcher(topLevelLang).find()) {
+            continue;
+          }
+          appendLang = !langPattern.matcher(lang).matches();
         }
         
         String rest = line.substring(colonIndex + 1).trim();
         if (rest.length() > 0) {
-          doTranslationLine(line, title, pos, sense, rest);
-        } else {
-          // TODO: do lines that are like "Greek:"
+          doTranslationLine(line, appendLang ? lang : null, title, pos, sense, rest);
         }
         
       } else if (wikiTokenizer.remainderStartsWith("''See''")) {
@@ -258,7 +274,7 @@ public class EnWiktionaryXmlParser {
     return index < list.size() ? list.get(index) : null;
   }
   
-  private void doTranslationLine(final String line, final String title, final String pos, final String sense, final String rest) {
+  private void doTranslationLine(final String line, final String lang, final String title, final String pos, final String sense, final String rest) {
     // Good chance we'll actually file this one...
     final PairEntry pairEntry = new PairEntry();
     final IndexedEntry indexedEntry = new IndexedEntry(pairEntry);
@@ -306,12 +322,16 @@ public class EnWiktionaryXmlParser {
             }
           //}
         } else if (functionName.equals("qualifier")) {
-          String qualifier = args.get(0);
-          if (!namedArgs.isEmpty() || args.size() > 1) {
-            LOG.warning("weird qualifier: " + line);
+          if (args.size() == 0) {
+           otherText.append(wikiTokenizer.token()); 
+          } else { 
+            String qualifier = args.get(0);
+            if (!namedArgs.isEmpty() || args.size() > 1) {
+              LOG.warning("weird qualifier: " + line);
+            }
+            // Unindexed!
+            otherText.append("(").append(qualifier).append(")");
           }
-          // Unindexed!
-          otherText.append("(").append(qualifier).append(")");
         } else if (encodings.contains(functionName)) {
           otherText.append("").append(args.get(0));
           otherIndexBuilder.addEntryWithString(indexedEntry, args.get(0), EntryTypeName.WIKTIONARY_TRANSLATION_OTHER_TEXT);
@@ -359,6 +379,10 @@ public class EnWiktionaryXmlParser {
       return;
     }
     
+    if (lang != null) {
+      otherText.insert(0, "(" + lang + ") ");
+    }
+    
     StringBuilder englishText = new StringBuilder();
     
     englishText.append(title);
@@ -578,6 +602,7 @@ public class EnWiktionaryXmlParser {
     
     final String prefix = listSection.firstPrefix;
     if (prefix.length() > 1) {
+      // Could just get looser and say that any prefix longer than first is a sublist.
       LOG.warning("Prefix too long: " + listSection);
       return;
     }
@@ -680,14 +705,16 @@ public class EnWiktionaryXmlParser {
           pairEntry.pairs.add(pair);
         }
       } else if (nextPrefix.equals("#::") || nextPrefix.equals("#**")) {
-        if (lastForeign != null) {
+        if (lastForeign != null && pairEntry.pairs.size() > 0) {
           pairEntry.pairs.remove(pairEntry.pairs.size() - 1);
           final Pair pair = new Pair(formatAndIndexExampleString(nextLine, enIndexBuilder, indexedEntry), formatAndIndexExampleString(lastForeign, otherIndexBuilder, indexedEntry), swap);
           if (pair.lang1 != "--" && pair.lang1 != "--") {
             pairEntry.pairs.add(pair);
           }
+          lastForeign = null;
         } else {
-          LOG.warning("English example with no foreign: " + title + ", " + nextLine);
+          LOG.warning("TODO: English example with no foreign: " + title + ", " + nextLine);
+          // TODO: add something.
         }
       } else if (nextPrefix.equals("#*")) {
         // Can't really index these.