1 // Copyright 2011 Google Inc. All Rights Reserved.
3 // Licensed under the Apache License, Version 2.0 (the "License");
4 // you may not use this file except in compliance with the License.
5 // You may obtain a copy of the License at
7 // http://www.apache.org/licenses/LICENSE-2.0
9 // Unless required by applicable law or agreed to in writing, software
10 // distributed under the License is distributed on an "AS IS" BASIS,
11 // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 // See the License for the specific language governing permissions and
13 // limitations under the License.
15 package com.hughes.android.dictionary.engine;
18 import java.io.IOException;
19 import java.io.PrintStream;
20 import java.io.RandomAccessFile;
21 import java.nio.charset.Charset;
22 import java.util.ArrayList;
23 import java.util.LinkedHashSet;
24 import java.util.List;
27 import java.util.regex.Pattern;
29 import javax.xml.parsers.ParserConfigurationException;
31 import org.xml.sax.SAXException;
33 import com.hughes.android.dictionary.parser.DictFileParser;
34 import com.hughes.android.dictionary.parser.Parser;
35 import com.hughes.android.dictionary.parser.SingleDictFileParser;
36 import com.hughes.android.dictionary.parser.wiktionary.EnForeignParser;
37 import com.hughes.android.dictionary.parser.wiktionary.EnToTranslationParser;
38 import com.hughes.android.dictionary.parser.wiktionary.EnTranslationToTranslationParser;
39 import com.hughes.android.dictionary.parser.wiktionary.WholeSectionToHtmlParser;
40 import com.hughes.util.Args;
41 import com.hughes.util.FileUtil;
43 public class DictionaryBuilder {
45 public final Dictionary dictionary;
46 public final List<IndexBuilder> indexBuilders = new ArrayList<>();
48 public DictionaryBuilder(final String dictInfoString, final Language lang0, final Language lang1, final String normalizerRules1, final String normalizerRules2, final Set<String> lang1Stoplist, final Set<String> lang2Stoplist) {
49 dictionary = new Dictionary(dictInfoString);
51 indexBuilders.add(new IndexBuilder(this, lang0.getIsoCode(), lang0.getIsoCode() + "->" + lang1.getIsoCode(), lang0, normalizerRules1, lang1Stoplist, false));
52 indexBuilders.add(new IndexBuilder(this, lang1.getIsoCode(), lang1.getIsoCode() + "->" + lang0.getIsoCode(), lang1, normalizerRules2, lang2Stoplist, true));
54 indexBuilders.add(new IndexBuilder(this, lang0.getIsoCode(), lang0.getIsoCode(), lang0, normalizerRules1, lang1Stoplist, false));
59 for (final IndexBuilder indexBuilder : indexBuilders) {
61 dictionary.indices.add(indexBuilder.index);
65 public static void main(final String[] args) throws IOException {
66 System.out.println("Running with arguments:");
67 for (final String arg : args) {
68 System.out.println(arg);
71 final Map<String,String> keyValueArgs = Args.keyValueArgs(args);
73 if (!keyValueArgs.containsKey("lang1")) {
74 fatalError("--lang1= must be specified.");
76 final Language lang1 = Language.lookup(keyValueArgs.remove("lang1"));
78 if (keyValueArgs.containsKey("lang2")) {
79 lang2 = Language.lookup(keyValueArgs.remove("lang2"));
84 final Set<String> lang1Stoplist = new LinkedHashSet<>();
85 final Set<String> lang2Stoplist = new LinkedHashSet<>();
86 final String lang1StoplistFile = keyValueArgs.remove("lang1Stoplist");
87 final String lang2StoplistFile = keyValueArgs.remove("lang2Stoplist");
88 if (lang1StoplistFile != null) {
89 lang1Stoplist.addAll(FileUtil.readLines(new File(lang1StoplistFile)));
91 if (lang2StoplistFile != null) {
92 lang2Stoplist.addAll(FileUtil.readLines(new File(lang2StoplistFile)));
95 String normalizerRules1 = keyValueArgs.remove("normalizerRules1");
96 String normalizerRules2 = keyValueArgs.remove("normalizerRules2");
97 if (normalizerRules1 == null) {
98 normalizerRules1 = lang1.getDefaultNormalizerRules();
100 if (normalizerRules2 == null) {
101 normalizerRules2 = lang2 == null ? null : lang2.getDefaultNormalizerRules();
104 final String dictOutFilename = keyValueArgs.remove("dictOut");
105 if (dictOutFilename == null) {
106 fatalError("--dictOut= must be specified.");
109 String dictInfo = keyValueArgs.remove("dictInfo");
110 if (dictInfo == null) {
111 fatalError("--dictInfo= must be specified.");
113 if (dictInfo.startsWith("@")) {
114 dictInfo = FileUtil.readToString(new File(dictInfo.substring(1)));
117 final String printFile = keyValueArgs.remove("print");
119 System.out.println("lang1=" + lang1);
120 System.out.println("lang2=" + lang2);
121 System.out.println("normalizerRules1=" + normalizerRules1);
122 System.out.println("normalizerRules2=" + normalizerRules2);
123 System.out.println("dictInfo=" + dictInfo);
124 System.out.println("dictOut=" + dictOutFilename);
126 final DictionaryBuilder dictionaryBuilder = new DictionaryBuilder(dictInfo, lang1, lang2, normalizerRules1, normalizerRules2, lang1Stoplist, lang2Stoplist);
128 for (int i = 0; i < 100; ++i) {
129 final String prefix = "input" + i;
130 if (keyValueArgs.containsKey(prefix)) {
131 final File file = new File(keyValueArgs.remove(prefix));
132 System.out.println("Processing: " + file);
133 String charsetName = keyValueArgs.remove(prefix + "Charset");
134 if (charsetName == null) {
135 charsetName = "UTF8";
137 final Charset charset = Charset.forName(charsetName);
138 String inputName = keyValueArgs.remove(prefix + "Name");
139 if (inputName == null) {
140 fatalError("Must specify human readable name for: " + prefix + "Name");
142 String pageLimitString = keyValueArgs.remove(prefix + "PageLimit");
143 if (pageLimitString == null) {
144 pageLimitString = "-1";
146 final int pageLimit = Integer.parseInt(pageLimitString);
148 final EntrySource entrySource = new EntrySource(dictionaryBuilder.dictionary.sources.size(), inputName, 0);
149 System.out.println();
151 String inputFormat = keyValueArgs.remove(prefix + "Format");
152 if ("tab_separated".equals(inputFormat)) {
153 final boolean flipColumns = "true".equals(keyValueArgs.remove(prefix + "FlipColumns"));
154 new DictFileParser(charset, flipColumns, DictFileParser.TAB, null, (lang2 == null), dictionaryBuilder).parse(file, entrySource, pageLimit);
155 } else if ("html_tab_separated".equals(inputFormat)) {
156 new SingleDictFileParser(charset, DictFileParser.TAB, dictionaryBuilder).parse(file, entrySource, pageLimit);
158 } else if ("chemnitz".equals(inputFormat)) {
159 final boolean flipColumns = "true".equals(keyValueArgs.remove(prefix + "FlipColumns"));
160 new DictFileParser(charset, flipColumns, DictFileParser.DOUBLE_COLON, DictFileParser.PIPE, (lang2 == null), dictionaryBuilder).parse(file, entrySource, pageLimit);
161 } else if ("enwiktionary".equals(inputFormat)) {
162 final String type = keyValueArgs.remove(prefix + "WiktionaryType");
163 final Pattern langPattern = Pattern.compile(keyValueArgs.remove(prefix + "LangPattern"), Pattern.CASE_INSENSITIVE);
164 final Pattern langCodePattern = Pattern.compile(keyValueArgs.remove(prefix + "LangCodePattern"));
165 final int enIndex = Integer.parseInt(keyValueArgs.remove(prefix + "EnIndex")) - 1;
167 if (enIndex < 0 || enIndex >= 2) {
168 fatalError("Must be 1 or 2: " + prefix + "EnIndex");
171 if ("EnToTranslation".equals(type)) {
172 parser = new EnToTranslationParser(dictionaryBuilder.indexBuilders.get(enIndex), dictionaryBuilder.indexBuilders.get(1-enIndex),
173 langPattern, langCodePattern, enIndex != 0);
174 } else if ("EnForeign".equals(type)) {
175 parser = new EnForeignParser(dictionaryBuilder.indexBuilders.get(enIndex), dictionaryBuilder.indexBuilders.get(1-enIndex),
176 langPattern, langCodePattern, enIndex != 0);
177 } else if ("EnEnglish".equals(type)) {
178 parser = new EnForeignParser(dictionaryBuilder.indexBuilders.get(enIndex), dictionaryBuilder.indexBuilders.get(enIndex),
179 langPattern, langCodePattern, true);
181 fatalError("Invalid WiktionaryType (use EnToTranslation or EnForeign or EnEnglish): " + type);
184 parser.parse(file, entrySource, pageLimit);
185 } else if (EnTranslationToTranslationParser.NAME.equals(inputFormat)) {
186 final String code1 = keyValueArgs.remove(prefix + "LangPattern1");
187 final String code2 = keyValueArgs.remove(prefix + "LangPattern2");
188 if (code1 == null || code2 == null) {
189 fatalError("Must specify LangPattern1 and LangPattern2.");
192 final Pattern codePattern1 = Pattern.compile(code1, Pattern.CASE_INSENSITIVE);
193 final Pattern codePattern2 = Pattern.compile(code2, Pattern.CASE_INSENSITIVE);
194 new EnTranslationToTranslationParser(dictionaryBuilder.indexBuilders, new Pattern[] {codePattern1, codePattern2}).parse(file, entrySource, pageLimit);
195 } else if (WholeSectionToHtmlParser.NAME.equals(inputFormat)) {
196 final int titleIndex = Integer.parseInt(keyValueArgs.remove(prefix + "TitleIndex")) - 1;
197 final String wiktionaryLang = keyValueArgs.remove(prefix + "WiktionaryLang");
198 final String webUrlTemplate = keyValueArgs.remove(prefix + "WebUrlTemplate");
199 String skipLang = keyValueArgs.remove(prefix + "SkipLang");
200 if (skipLang == null) skipLang = "";
201 new WholeSectionToHtmlParser(dictionaryBuilder.indexBuilders.get(titleIndex), null, wiktionaryLang, skipLang, webUrlTemplate).parse(file, entrySource, pageLimit);
203 fatalError("Invalid or missing input format: " + inputFormat);
206 dictionaryBuilder.dictionary.sources.add(entrySource);
207 System.out.println("Done: " + file + "\n\n");
211 dictionaryBuilder.build();
212 // Drop indexBuilders to free RAM
213 dictionaryBuilder.indexBuilders.clear();
215 if (printFile != null) {
216 final PrintStream out = new PrintStream(new File(printFile));
217 dictionaryBuilder.dictionary.print(out);
221 System.out.println("Writing dictionary to: " + dictOutFilename);
222 final RandomAccessFile dictOut = new RandomAccessFile(dictOutFilename, "rw");
223 dictOut.setLength(0);
224 dictionaryBuilder.dictionary.write(dictOut);
227 if (!keyValueArgs.isEmpty()) {
228 System.err.println("WARNING: couldn't parse arguments: " + keyValueArgs);
234 private static void fatalError(String string) {
235 System.err.println(string);