X-Git-Url: http://gitweb.fperrin.net/?a=blobdiff_plain;f=src%2Fcom%2Fhughes%2Fandroid%2Fdictionary%2Fengine%2FDictionaryBuilderMain.java;h=825e7f99e3b01b75ecf6c9ad28058ac63cc5cff1;hb=58fd4402729f38bf4408e8fef487a9bc359e45a0;hp=ea9e1a72f1e6446264057eb9746f315103deb751;hpb=67780552c216010c96a4823a2f2f44e6f206c8f2;p=DictionaryPC.git

diff --git a/src/com/hughes/android/dictionary/engine/DictionaryBuilderMain.java b/src/com/hughes/android/dictionary/engine/DictionaryBuilderMain.java
index ea9e1a7..825e7f9 100644
--- a/src/com/hughes/android/dictionary/engine/DictionaryBuilderMain.java
+++ b/src/com/hughes/android/dictionary/engine/DictionaryBuilderMain.java
@@ -34,6 +34,108 @@ public class DictionaryBuilderMain extends TestCase {
   static final String INPUTS = "data/inputs/";
   static final String STOPLISTS = "data/inputs/stoplists/";
   static final String OUTPUTS = "data/outputs/";  
+  
+  // Build the non EN ones.
+  static final String[][] nonEnPairs = new String[][] {
+      {"EN"},
+      {"DE"},
+      {"IT"},
+      // This one takes a really long time:
+      // {"FR"},
+          
+      // The 3 I use most:
+      {"IT", "EN" },
+      {"DE", "EN" },
+      {"DE", "IT" },
+
+      {"AR", "DE" },
+      {"AR", "ES" },
+      {"AR", "FR" },
+      {"AR", "HE" },
+      {"AR", "IT" },
+      {"AR", "JA" },
+      {"AR", "RU" },
+      {"AR", "TR" },  // Turkish
+      {"AR", "cmn" },
+      
+      {"DE", "AR" },
+      {"DE", "FR" },
+      {"DE", "CA" },  // Catalan
+      {"DE", "CS" },  // Czech
+      {"DE", "EO" },  // Esperanto
+      {"DE", "ES" },
+      {"DE", "FR" },
+      {"DE", "HE" },
+      {"DE", "HU" },  // Hungarian
+      {"DE", "IT" },
+      {"DE", "JA" },
+      {"DE", "LA" },  // Latin
+      {"DE", "NL" },  // Dutch
+      {"DE", "PL" },  // Polish
+      {"DE", "RU" },
+      {"DE", "SV" },  // Swedish
+      {"DE", "TR" },  // Turkish
+      {"DE", "cmn" },
+      {"DE", "TA" },  // Tamil
+      
+      {"ES", "RU" },  // Spanish-Russian
+      
+      {"FR", "BG" },  // Bulgarian
+      {"FR", "CS" },  // Czech
+      {"FR", "DE" },
+      {"FR", "ES" },
+      {"FR", "IT" },
+      {"FR", "JA" },
+      {"FR", "LA" },
+      {"FR", "NL" },  // Dutch
+      {"FR", "RU" },
+      {"FR", "TR" },  // Turkish
+      {"FR", "cmn" },
+      {"FR", "EL" },  
+
+      {"IT", "DE" },
+      {"IT", "EL" },  // Greek
+      {"IT", "ES" },
+      {"IT", "FR" },
+      {"IT", "HU" },
+      {"IT", "JA" },
+      {"IT", "LA" },  // Latin
+      {"IT", "LV" },  // Latvian
+      {"IT", "NL" },
+      {"IT", "PL" },
+      {"IT", "RU" },
+      {"IT", "SV" },
+      {"IT", "TR" },  // Turkish
+      {"IT", "cmn" },
+
+      {"JA", "cmn" },
+      {"JA", "AR" },
+      {"JA", "KO" },
+
+      {"cmn", "AR" },
+      {"cmn", "DE" },
+      {"cmn", "ES" },
+      {"cmn", "FR" },
+      {"cmn", "IT" },
+      {"cmn", "KO" },
+
+      
+      {"NO", "SV" },
+      {"NO", "FI" },
+      {"FI", "SV" },
+      
+      {"PL", "FR" },  // Polish
+      {"PL", "RU" },  // Polish
+      {"PL", "HU" },  // Polish
+      {"PL", "ES" },  // Polish
+      
+      {"TR", "EL" },  // Turkish, Greek
+
+      {"FA", "HY" },  // Persian, Armenian, by request.
+      {"FA", "SV" },  // Persian, Swedish, by request.
+      {"NL", "PL" },  // Dutch, Polish, by request.
+  };
+
 
   
   static final Map<String,String>  isoToDedication = new LinkedHashMap<String, String>();
@@ -75,7 +177,50 @@ public class DictionaryBuilderMain extends TestCase {
   }
   
   static List<String> getMainArgs(final String[] pair) {
-    final List<String> result = new ArrayList<String>();
+      final List<String> result = new ArrayList<String>();
+      
+    int i = 1;
+    
+    if (pair.length == 1) {
+        final String lang1 = pair[0];
+        final String dictFile = String.format("%s/%s.quickdic", OUTPUTS, lang1);
+        result.add(String.format("--dictOut=%s", dictFile));
+        result.add(String.format("--lang1=%s", lang1));
+        result.add(String.format("--lang1Stoplist=%s", STOPLISTS + getStoplist(lang1)));
+        result.add(String.format("--dictInfo=Wikitionary-based %s dictionary.", lang1));
+
+        
+        final String wikiSplitFile = String.format("%s/wikiSplit/%s/%s.data", INPUTS, lang1.toLowerCase(), lang1);
+        if (new File(wikiSplitFile).canRead()) {
+            result.add(String.format("--input%d=%s", i, wikiSplitFile));
+            result.add(String.format("--input%dName=%s.wiktionary.org", i, lang1.toLowerCase()));
+            result.add(String.format("--input%dFormat=%s", i, WholeSectionToHtmlParser.NAME));
+            result.add(String.format("--input%dTitleIndex=%d", i, 1));
+            result.add(String.format("--input%dWiktionaryLang=%s", i, lang1));
+            result.add(String.format("--input%dSkipLang=%s", i, lang1));
+            result.add(String.format("--input%dWebUrlTemplate=http://%s.wiktionary.org/wiki/%%s", i, lang1.toLowerCase()));
+            //result.add(String.format("--input%dPageLimit=100", i));
+            ++i;
+        } else {
+            System.err.println("Can't read file: " + wikiSplitFile);
+        }
+        
+        if (lang1.equals("EN") && !lang1.equals("EN")) {
+            // Add a parser that tries to use the definitions.  This is
+            // not very pretty yet.
+            result.add(String.format("--input%d=%s/wikiSplit/en/%s.data", i, INPUTS, lang1));
+            result.add(String.format("--input%dName=ENWiktionary.%s", i, lang1)) ;
+            result.add(String.format("--input%dFormat=enwiktionary", i));
+            result.add(String.format("--input%dWiktionaryType=EnEnglish", i));
+            result.add(String.format("--input%dLangPattern=%s", i, "English"));
+            result.add(String.format("--input%dLangCodePattern=%s", i, lang1.toLowerCase()));
+            result.add(String.format("--input%dEnIndex=%d", i, 1));
+            //result.add(String.format("--input%dPageLimit=100", i));
+            ++i;
+        }
+        
+        return result;
+    }  // Single-lang dictionaries.
     
     final String lang1 = pair[0];
     final String lang2 = pair[1];
@@ -87,8 +232,6 @@ public class DictionaryBuilderMain extends TestCase {
     result.add(String.format("--lang1Stoplist=%s", STOPLISTS + getStoplist(lang1)));
     result.add(String.format("--lang2Stoplist=%s", STOPLISTS + getStoplist(lang2)));
 
-    int i = 1;
-    
     // For a few langs, put the defs of the other language in DE/IT/FR using WholeSection.
     for (final String wikitionaryLang : Arrays.asList("EN", "DE", "IT", "FR")) {
         if (!Arrays.asList(pair).contains(wikitionaryLang)) {
@@ -97,32 +240,29 @@ public class DictionaryBuilderMain extends TestCase {
         final String foreignIso = getOtherLang(pair, wikitionaryLang);
         final String wikiSplitFile = String.format("%s/wikiSplit/%s/%s.data", INPUTS, wikitionaryLang.toLowerCase(), foreignIso);
         if (!new File(wikiSplitFile).canRead()) {
-            System.err.println("Can't read file: " + wikiSplitFile);
+            System.err.println("WARNING: Can't read file: " + wikiSplitFile);
             continue;
         }
         result.add(String.format("--input%d=%s", i, wikiSplitFile));
-        result.add(String.format("--input%dName=%s.wiktionary.org (%s)", i, wikitionaryLang, foreignIso));
+        result.add(String.format("--input%dName=%s.wiktionary.org", i, wikitionaryLang.toLowerCase()));
         result.add(String.format("--input%dFormat=%s", i, WholeSectionToHtmlParser.NAME));
         result.add(String.format("--input%dTitleIndex=%d", i, Arrays.asList(pair).indexOf(foreignIso) + 1));
         result.add(String.format("--input%dWiktionaryLang=%s", i, wikitionaryLang));
         result.add(String.format("--input%dSkipLang=%s", i, foreignIso));
+        result.add(String.format("--input%dWebUrlTemplate=http://%s.wiktionary.org/wiki/%%s", i, wikitionaryLang.toLowerCase()));
         ++i;
     }
     
     // Deal with the pairs where one is English.
     if (Arrays.asList(pair).contains("EN")) {
       final String foreignIso = getOtherLang(pair, "EN");
-      
       String foreignRegex = WiktionaryLangs.isoCodeToEnWikiName.get(foreignIso);
-      if (foreignIso.equals("ZH")) {
-        // HACK: The missing "e" prevents a full match, causing "Cantonese" to be appended to the entries.
-        foreignRegex = "Chinese|Mandarin|Cantones";
-      }
       
       result.add(String.format("--lang1=%s", lang1));
       result.add(String.format("--lang2=%s",  lang2));
       result.add(String.format("--dictInfo=(EN)Wikitionary-based EN-%s dictionary.%s", foreignIso, getDedication(foreignIso)));
       
+      // Foreign section.
       result.add(String.format("--input%d=%s/wikiSplit/en/%s.data", i, INPUTS, foreignIso));
       result.add(String.format("--input%dName=ENWiktionary.%s", i, foreignIso)) ;
       result.add(String.format("--input%dFormat=enwiktionary", i));
@@ -132,6 +272,7 @@ public class DictionaryBuilderMain extends TestCase {
       result.add(String.format("--input%dEnIndex=%d", i, Arrays.asList(pair).indexOf("EN") + 1));
       ++i;
 
+      // Translation section.
       result.add(String.format("--input%d=%swikiSplit/en/EN.data", i, INPUTS));
       result.add(String.format("--input%dName=enwiktionary.english", i));
       result.add(String.format("--input%dFormat=enwiktionary", i));
@@ -161,6 +302,8 @@ public class DictionaryBuilderMain extends TestCase {
       result.add(String.format("--input%dLangPattern1=%s", i, lang1));
       result.add(String.format("--input%dLangPattern2=%s", i, lang2));
       ++i;
+      
+      // TODO: Could use FR translation section here too.
     }
     
     return result;
@@ -170,111 +313,13 @@ public class DictionaryBuilderMain extends TestCase {
     
     final List<String[]> allPairs = new ArrayList<String[]>();
     
-    // Build the non EN ones.
-    final String[][] nonEnPairs = new String[][] {
-
-        // The 3 I use most:
-        {"IT", "EN" },
-        {"DE", "EN" },
-        {"DE", "IT" },
-            
-            
-        {"AR", "DE" },
-        {"AR", "ES" },
-        {"AR", "FR" },
-        {"AR", "HE" },
-        {"AR", "IT" },
-        {"AR", "JA" },
-        {"AR", "RU" },
-        {"AR", "TR" },  // Turkish
-        {"AR", "ZH" },
-        
-        {"DE", "AR" },
-        {"DE", "FR" },
-        {"DE", "CA" },  // Catalan
-        {"DE", "CS" },  // Czech
-        {"DE", "EO" },  // Esperanto
-        {"DE", "ES" },
-        {"DE", "FR" },
-        {"DE", "HE" },
-        {"DE", "HU" },  // Hungarian
-        {"DE", "IT" },
-        {"DE", "JA" },
-        {"DE", "LA" },  // Latin
-        {"DE", "NL" },  // Dutch
-        {"DE", "PL" },  // Polish
-        {"DE", "RU" },
-        {"DE", "SV" },  // Swedish
-        {"DE", "TR" },  // Turkish
-        {"DE", "ZH" },
-        {"DE", "TA" },  // Tamil
-        
-        {"ES", "RU" },  // Spanish-Russian
-        
-        {"FR", "BG" },  // Bulgarian
-        {"FR", "CS" },  // Czech
-        {"FR", "DE" },
-        {"FR", "ES" },
-        {"FR", "IT" },
-        {"FR", "JA" },
-        {"FR", "LA" },
-        {"FR", "NL" },  // Dutch
-        {"FR", "RU" },
-        {"FR", "TR" },  // Turkish
-        {"FR", "ZH" },
-
-        {"IT", "DE" },
-        {"IT", "EL" },  // Greek
-        {"IT", "ES" },
-        {"IT", "FR" },
-        {"IT", "HU" },
-        {"IT", "JA" },
-        {"IT", "LA" },  // Latin
-        {"IT", "LV" },  // Latvian
-        {"IT", "NL" },
-        {"IT", "PL" },
-        {"IT", "RU" },
-        {"IT", "SV" },
-        {"IT", "TR" },  // Turkish
-        {"IT", "ZH" },
-
-        {"JA", "ZH" },
-        {"JA", "AR" },
-        {"JA", "KO" },
-
-        {"ZH", "AR" },
-        {"ZH", "DE" },
-        {"ZH", "ES" },
-        {"ZH", "FR" },
-        {"ZH", "IT" },
-        {"ZH", "KO" },
-
-        
-        {"NO", "SV" },
-        {"NO", "FI" },
-        {"FI", "SV" },
-        
-        {"PL", "FR" },  // Polish
-        {"PL", "RU" },  // Polish
-        {"PL", "HU" },  // Polish
-        {"PL", "ES" },  // Polish
-        
-        {"TR", "EL" },  // Turkish, Greek
-
-        {"FA", "HY" },  // Persian, Armenian, by request.
-        {"FA", "SV" },  // Persian, Swedish, by request.
-
-    };
     allPairs.addAll(Arrays.asList(nonEnPairs));
-    
     // Add all the EN-XX pairs.
     for (final String isoCode : WiktionaryLangs.isoCodeToEnWikiName.keySet()) {
-      if (isoCode.equals("EN") || isoCode.equals("DE")) {
-        continue;
+      if (!isoCode.equals("EN")) {
+          allPairs.add(new String[] {"EN", isoCode});
       }
-      allPairs.add(new String[] {"EN", isoCode});
     }
-    allPairs.add(new String[] {"EN", "DE"});
     
         
     final Set<List<String>> done = new LinkedHashSet<List<String>>();
@@ -286,7 +331,7 @@ public class DictionaryBuilderMain extends TestCase {
       }
       done.add(pairList);
       
-      if (!pairList.contains("IT") || !pairList.contains("EN")) {
+      if (!pairList.contains("EN") && !pairList.contains("EL")) {
         //continue;
       }