src/com/hughes/android/dictionary/engine/DictionaryBuilder.java

   1 // Copyright 2011 Google Inc. All Rights Reserved.
   2 //
   3 // Licensed under the Apache License, Version 2.0 (the "License");
   4 // you may not use this file except in compliance with the License.
   5 // You may obtain a copy of the License at
   6 //
   7 //     http://www.apache.org/licenses/LICENSE-2.0
   8 //
   9 // Unless required by applicable law or agreed to in writing, software
  10 // distributed under the License is distributed on an "AS IS" BASIS,
  11 // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  12 // See the License for the specific language governing permissions and
  13 // limitations under the License.
  14
  15 package com.hughes.android.dictionary.engine;
  16
  17 import java.io.File;
  18 import java.io.IOException;
  19 import java.io.PrintStream;
  20 import java.io.RandomAccessFile;
  21 import java.nio.charset.Charset;
  22 import java.util.ArrayList;
  23 import java.util.LinkedHashSet;
  24 import java.util.List;
  25 import java.util.Map;
  26 import java.util.Set;
  27 import java.util.regex.Pattern;
  28
  29 import javax.xml.parsers.ParserConfigurationException;
  30
  31 import org.xml.sax.SAXException;
  32
  33 import com.hughes.android.dictionary.parser.DictFileParser;
  34 import com.hughes.android.dictionary.parser.Parser;
  35 import com.hughes.android.dictionary.parser.wiktionary.EnForeignParser;
  36 import com.hughes.android.dictionary.parser.wiktionary.EnToTranslationParser;
  37 import com.hughes.android.dictionary.parser.wiktionary.EnTranslationToTranslationParser;
  38 import com.hughes.android.dictionary.parser.wiktionary.WholeSectionToHtmlParser;
  39 import com.hughes.util.Args;
  40 import com.hughes.util.FileUtil;
  41
  42 public class DictionaryBuilder {
  43
  44     public final Dictionary dictionary;
  45     public final List<IndexBuilder> indexBuilders = new ArrayList<>();
  46
  47     public DictionaryBuilder(final String dictInfoString, final Language lang0, final Language lang1, final String normalizerRules1, final String normalizerRules2, final Set<String> lang1Stoplist, final Set<String> lang2Stoplist) {
  48         dictionary = new Dictionary(dictInfoString);
  49         if (lang1 != null) {
  50             indexBuilders.add(new IndexBuilder(this, lang0.getIsoCode(), lang0.getIsoCode() + "->" + lang1.getIsoCode(), lang0, normalizerRules1, lang1Stoplist, false));
  51             indexBuilders.add(new IndexBuilder(this, lang1.getIsoCode(), lang1.getIsoCode() + "->" + lang0.getIsoCode(), lang1, normalizerRules2, lang2Stoplist, true));
  52         } else {
  53             indexBuilders.add(new IndexBuilder(this, lang0.getIsoCode(), lang0.getIsoCode(), lang0, normalizerRules1, lang1Stoplist, false));
  54         }
  55     }
  56
  57     void build() {
  58         for (final IndexBuilder indexBuilder : indexBuilders) {
  59             indexBuilder.build();
  60             dictionary.indices.add(indexBuilder.index);
  61         }
  62     }
  63
  64     public static void main(final String[] args) throws IOException {
  65         System.out.println("Running with arguments:");
  66         for (final String arg : args) {
  67             System.out.println(arg);
  68         }
  69
  70         final Map<String,String> keyValueArgs = Args.keyValueArgs(args);
  71
  72         if (!keyValueArgs.containsKey("lang1")) {
  73             fatalError("--lang1= must be specified.");
  74         }
  75         final Language lang1 = Language.lookup(keyValueArgs.remove("lang1"));
  76         final Language lang2;
  77         if (keyValueArgs.containsKey("lang2")) {
  78             lang2 = Language.lookup(keyValueArgs.remove("lang2"));
  79         } else {
  80             lang2 = null;
  81         }
  82
  83         final Set<String> lang1Stoplist = new LinkedHashSet<>();
  84         final Set<String> lang2Stoplist = new LinkedHashSet<>();
  85         final String lang1StoplistFile = keyValueArgs.remove("lang1Stoplist");
  86         final String lang2StoplistFile = keyValueArgs.remove("lang2Stoplist");
  87         if (lang1StoplistFile != null) {
  88             lang1Stoplist.addAll(FileUtil.readLines(new File(lang1StoplistFile)));
  89         }
  90         if (lang2StoplistFile != null) {
  91             lang2Stoplist.addAll(FileUtil.readLines(new File(lang2StoplistFile)));
  92         }
  93
  94         String normalizerRules1 = keyValueArgs.remove("normalizerRules1");
  95         String normalizerRules2 = keyValueArgs.remove("normalizerRules2");
  96         if (normalizerRules1 == null) {
  97             normalizerRules1 = lang1.getDefaultNormalizerRules();
  98         }
  99         if (normalizerRules2 == null) {
 100             normalizerRules2 = lang2 == null ? null : lang2.getDefaultNormalizerRules();
 101         }
 102
 103         final String dictOutFilename = keyValueArgs.remove("dictOut");
 104         if (dictOutFilename == null) {
 105             fatalError("--dictOut= must be specified.");
 106         }
 107
 108         String dictInfo = keyValueArgs.remove("dictInfo");
 109         if (dictInfo == null) {
 110             fatalError("--dictInfo= must be specified.");
 111         }
 112         if (dictInfo.startsWith("@")) {
 113             dictInfo = FileUtil.readToString(new File(dictInfo.substring(1)));
 114         }
 115
 116         final String printFile = keyValueArgs.remove("print");
 117
 118         System.out.println("lang1=" + lang1);
 119         System.out.println("lang2=" + lang2);
 120         System.out.println("normalizerRules1=" + normalizerRules1);
 121         System.out.println("normalizerRules2=" + normalizerRules2);
 122         System.out.println("dictInfo=" + dictInfo);
 123         System.out.println("dictOut=" + dictOutFilename);
 124
 125         final DictionaryBuilder dictionaryBuilder = new DictionaryBuilder(dictInfo, lang1, lang2, normalizerRules1, normalizerRules2, lang1Stoplist, lang2Stoplist);
 126
 127         for (int i = 0; i < 100; ++i) {
 128             final String prefix = "input" + i;
 129             if (keyValueArgs.containsKey(prefix)) {
 130                 final File file = new File(keyValueArgs.remove(prefix));
 131                 System.out.println("Processing: " + file);
 132                 String charsetName = keyValueArgs.remove(prefix + "Charset");
 133                 if (charsetName == null) {
 134                     charsetName = "UTF8";
 135                 }
 136                 final Charset charset = Charset.forName(charsetName);
 137                 String inputName = keyValueArgs.remove(prefix + "Name");
 138                 if (inputName == null) {
 139                     fatalError("Must specify human readable name for: " + prefix + "Name");
 140                 }
 141                 String pageLimitString = keyValueArgs.remove(prefix + "PageLimit");
 142                 if (pageLimitString == null) {
 143                     pageLimitString = "-1";
 144                 }
 145                 final int pageLimit = Integer.parseInt(pageLimitString);
 146
 147                 final EntrySource entrySource = new EntrySource(dictionaryBuilder.dictionary.sources.size(), inputName, 0);
 148                 System.out.println();
 149
 150                 String inputFormat = keyValueArgs.remove(prefix + "Format");
 151                 if ("tab_separated".equals(inputFormat)) {
 152                     final boolean flipColumns = "true".equals(keyValueArgs.remove(prefix + "FlipColumns"));
 153                     new DictFileParser(charset, flipColumns, DictFileParser.TAB, null, dictionaryBuilder, dictionaryBuilder.indexBuilders.toArray(new IndexBuilder[0]), null).parse(file, entrySource, pageLimit);
 154                 } else if ("chemnitz".equals(inputFormat)) {
 155                     final boolean flipColumns = "true".equals(keyValueArgs.remove(prefix + "FlipColumns"));
 156                     new DictFileParser(charset, flipColumns, DictFileParser.DOUBLE_COLON, DictFileParser.PIPE, dictionaryBuilder, dictionaryBuilder.indexBuilders.toArray(new IndexBuilder[0]), null).parse(file, entrySource, pageLimit);
 157                 } else if ("enwiktionary".equals(inputFormat)) {
 158                     final String type = keyValueArgs.remove(prefix + "WiktionaryType");
 159                     final Pattern langPattern = Pattern.compile(keyValueArgs.remove(prefix + "LangPattern"), Pattern.CASE_INSENSITIVE);
 160                     final Pattern langCodePattern = Pattern.compile(keyValueArgs.remove(prefix + "LangCodePattern"));
 161                     final int enIndex = Integer.parseInt(keyValueArgs.remove(prefix + "EnIndex")) - 1;
 162
 163                     if (enIndex < 0 || enIndex >= 2) {
 164                         fatalError("Must be 1 or 2: " + prefix + "EnIndex");
 165                     }
 166                     final Parser parser;
 167                     if ("EnToTranslation".equals(type)) {
 168                         parser = new EnToTranslationParser(dictionaryBuilder.indexBuilders.get(enIndex), dictionaryBuilder.indexBuilders.get(1-enIndex),
 169                                                            langPattern, langCodePattern, enIndex != 0);
 170                     } else if ("EnForeign".equals(type)) {
 171                         parser = new EnForeignParser(dictionaryBuilder.indexBuilders.get(enIndex), dictionaryBuilder.indexBuilders.get(1-enIndex),
 172                                                      langPattern, langCodePattern, enIndex != 0);
 173                     } else if ("EnEnglish".equals(type)) {
 174                         parser = new EnForeignParser(dictionaryBuilder.indexBuilders.get(enIndex), dictionaryBuilder.indexBuilders.get(enIndex),
 175                                                      langPattern, langCodePattern, true);
 176                     } else {
 177                         fatalError("Invalid WiktionaryType (use EnToTranslation or EnForeign or EnEnglish): " + type);
 178                         return;
 179                     }
 180                     parser.parse(file, entrySource, pageLimit);
 181                 } else if (EnTranslationToTranslationParser.NAME.equals(inputFormat)) {
 182                     final String code1 = keyValueArgs.remove(prefix + "LangPattern1");
 183                     final String code2 = keyValueArgs.remove(prefix + "LangPattern2");
 184                     if (code1 == null || code2 == null) {
 185                         fatalError("Must specify LangPattern1 and LangPattern2.");
 186                         return;
 187                     }
 188                     final Pattern codePattern1 = Pattern.compile(code1, Pattern.CASE_INSENSITIVE);
 189                     final Pattern codePattern2 = Pattern.compile(code2, Pattern.CASE_INSENSITIVE);
 190                     new EnTranslationToTranslationParser(dictionaryBuilder.indexBuilders, new Pattern[] {codePattern1, codePattern2}).parse(file, entrySource, pageLimit);
 191                 } else if (WholeSectionToHtmlParser.NAME.equals(inputFormat)) {
 192                     final int titleIndex = Integer.parseInt(keyValueArgs.remove(prefix + "TitleIndex")) - 1;
 193                     final String wiktionaryLang = keyValueArgs.remove(prefix + "WiktionaryLang");
 194                     final String webUrlTemplate = keyValueArgs.remove(prefix + "WebUrlTemplate");
 195                     String skipLang = keyValueArgs.remove(prefix + "SkipLang");
 196                     if (skipLang == null) skipLang = "";
 197                     new WholeSectionToHtmlParser(dictionaryBuilder.indexBuilders.get(titleIndex), null, wiktionaryLang, skipLang, webUrlTemplate).parse(file, entrySource, pageLimit);
 198                 } else {
 199                     fatalError("Invalid or missing input format: " + inputFormat);
 200                 }
 201
 202                 dictionaryBuilder.dictionary.sources.add(entrySource);
 203                 System.out.println("Done: " + file + "\n\n");
 204             }
 205         }
 206
 207         dictionaryBuilder.build();
 208         // Drop indexBuilders to free RAM
 209         dictionaryBuilder.indexBuilders.clear();
 210
 211         if (printFile != null) {
 212             final PrintStream out = new PrintStream(new File(printFile));
 213             dictionaryBuilder.dictionary.print(out);
 214             out.close();
 215         }
 216
 217         System.out.println("Writing dictionary to: " + dictOutFilename);
 218         final RandomAccessFile dictOut = new RandomAccessFile(dictOutFilename, "rw");
 219         dictOut.setLength(0);
 220         dictionaryBuilder.dictionary.write(dictOut);
 221         dictOut.close();
 222
 223         if (!keyValueArgs.isEmpty()) {
 224             System.err.println("WARNING: couldn't parse arguments: " + keyValueArgs);
 225             System.exit(1);
 226         }
 227
 228     }
 229
 230     private static void fatalError(String string) {
 231         System.err.println(string);
 232
 233
 234         System.exit(1);
 235     }
 236
 237 }