1 // Copyright 2011 Google Inc. All Rights Reserved.
3 // Licensed under the Apache License, Version 2.0 (the "License");
4 // you may not use this file except in compliance with the License.
5 // You may obtain a copy of the License at
7 // http://www.apache.org/licenses/LICENSE-2.0
9 // Unless required by applicable law or agreed to in writing, software
10 // distributed under the License is distributed on an "AS IS" BASIS,
11 // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 // See the License for the specific language governing permissions and
13 // limitations under the License.
15 package com.hughes.android.dictionary.engine;
18 import java.io.IOException;
19 import java.io.PrintStream;
20 import java.io.RandomAccessFile;
21 import java.nio.charset.Charset;
22 import java.util.ArrayList;
23 import java.util.Arrays;
24 import java.util.LinkedHashSet;
25 import java.util.List;
28 import java.util.regex.Pattern;
30 import javax.xml.parsers.ParserConfigurationException;
32 import org.xml.sax.SAXException;
34 import com.hughes.android.dictionary.parser.DictFileParser;
35 import com.hughes.android.dictionary.parser.enwiktionary.EnWiktionaryXmlParser;
36 import com.hughes.util.Args;
37 import com.hughes.util.FileUtil;
40 public class DictionaryBuilder {
42 public final Dictionary dictionary;
43 public final List<IndexBuilder> indexBuilders = new ArrayList<IndexBuilder>();
45 public DictionaryBuilder(final String dictInfo, final Language lang0, final Language lang1, final String normalizerRules1, final String normalizerRules2, final Set<String> lang1Stoplist, final Set<String> lang2Stoplist) {
46 dictionary = new Dictionary(dictInfo);
47 indexBuilders.add(new IndexBuilder(this, lang0.getSymbol(), lang0.getSymbol() + "->" + lang1.getSymbol(), lang0, normalizerRules1, lang1Stoplist, false));
48 indexBuilders.add(new IndexBuilder(this, lang1.getSymbol(), lang1.getSymbol() + "->" + lang0.getSymbol(), lang1, normalizerRules2, lang2Stoplist, true));
52 for (final IndexBuilder indexBuilder : indexBuilders) {
54 dictionary.indices.add(indexBuilder.index);
58 public static void main(final String[] args) throws IOException, ParserConfigurationException, SAXException {
59 System.out.println("Running with arguments:");
60 for (final String arg : args) {
61 System.out.println(arg);
64 final Map<String,String> keyValueArgs = Args.keyValueArgs(args);
66 if (!keyValueArgs.containsKey("lang1") || !keyValueArgs.containsKey("lang2")) {
67 fatalError("--lang1= and --lang2= must both be specified.");
69 final Language lang1 = Language.lookup(keyValueArgs.remove("lang1"));
70 final Language lang2 = Language.lookup(keyValueArgs.remove("lang2"));
72 final Set<String> lang1Stoplist = new LinkedHashSet<String>();
73 final Set<String> lang2Stoplist = new LinkedHashSet<String>();
74 final String lang1StoplistFile = keyValueArgs.remove("lang1Stoplist");
75 final String lang2StoplistFile = keyValueArgs.remove("lang2Stoplist");
76 if (lang1StoplistFile != null) {
77 lang1Stoplist.addAll(FileUtil.readLines(new File(lang1StoplistFile)));
79 if (lang2StoplistFile != null) {
80 lang2Stoplist.addAll(FileUtil.readLines(new File(lang2StoplistFile)));
83 String normalizerRules1 = keyValueArgs.remove("normalizerRules1");
84 String normalizerRules2 = keyValueArgs.remove("normalizerRules2");
85 if (normalizerRules1 == null) {
86 normalizerRules1 = lang1.getDefaultNormalizerRules();
88 if (normalizerRules2 == null) {
89 normalizerRules2 = lang2.getDefaultNormalizerRules();
92 final String dictOutFilename = keyValueArgs.remove("dictOut");
93 if (dictOutFilename == null) {
94 fatalError("--dictOut= must be specified.");
97 String dictInfo = keyValueArgs.remove("dictInfo");
98 if (dictInfo == null) {
99 fatalError("--dictInfo= must be specified.");
101 if (dictInfo.startsWith("@")) {
102 dictInfo = FileUtil.readToString(new File(dictInfo.substring(1)));
105 final String printFile = keyValueArgs.remove("print");
107 System.out.println("lang1=" + lang1);
108 System.out.println("lang2=" + lang2);
109 System.out.println("normalizerRules1=" + normalizerRules1);
110 System.out.println("normalizerRules2=" + normalizerRules2);
111 System.out.println("dictInfo=" + dictInfo);
112 System.out.println("dictOut=" + dictOutFilename);
114 final DictionaryBuilder dictionaryBuilder = new DictionaryBuilder(dictInfo, lang1, lang2, normalizerRules1, normalizerRules2, lang1Stoplist, lang2Stoplist);
116 for (int i = 0; i < 100; ++i) {
117 final String prefix = "input" + i;
118 if (keyValueArgs.containsKey(prefix)) {
119 final File file = new File(keyValueArgs.remove(prefix));
120 System.out.println("Processing: " + file);
121 String charsetName = keyValueArgs.remove(prefix + "Charset");
122 if (charsetName == null) {
123 charsetName = "UTF8";
125 final Charset charset = Charset.forName(charsetName);
126 String inputName = keyValueArgs.remove(prefix + "Name");
127 if (inputName == null) {
128 fatalError("Must specify human readable name for: " + prefix + "Name");
131 final EntrySource entrySource = new EntrySource(dictionaryBuilder.dictionary.sources.size(), inputName, dictionaryBuilder.dictionary.pairEntries.size());
132 System.out.println("");
134 String inputFormat = keyValueArgs.remove(prefix + "Format");
135 if ("tab_separated".equals(inputFormat)) {
136 final boolean flipColumns = "true".equals(keyValueArgs.remove(prefix + "FlipColumns"));
137 new DictFileParser(charset, flipColumns, DictFileParser.TAB, null, dictionaryBuilder, dictionaryBuilder.indexBuilders.toArray(new IndexBuilder[0]), null).parseFile(file);
138 } else if ("chemnitz".equals(inputFormat)) {
139 final boolean flipColumns = "true".equals(keyValueArgs.remove(prefix + "FlipColumns"));
140 new DictFileParser(charset, flipColumns, DictFileParser.DOUBLE_COLON, DictFileParser.PIPE, dictionaryBuilder, dictionaryBuilder.indexBuilders.toArray(new IndexBuilder[0]), null).parseFile(file);
141 } else if ("enwiktionary".equals(inputFormat)) {
142 final Pattern langPattern = Pattern.compile(keyValueArgs.remove(prefix + "LangPattern"), Pattern.CASE_INSENSITIVE);
143 final Pattern langCodePattern = Pattern.compile(keyValueArgs.remove(prefix + "LangCodePattern"));
144 final int enIndex = Integer.parseInt(keyValueArgs.remove(prefix + "EnIndex")) - 1;
145 String pageLimit = keyValueArgs.remove(prefix + "PageLimit");
146 if (pageLimit == null) {
150 if (enIndex < 0 || enIndex >= 2) {
151 fatalError("Must be 1 or 2: " + prefix + "EnIndex");
153 new EnWiktionaryXmlParser(dictionaryBuilder.indexBuilders.get(enIndex), dictionaryBuilder.indexBuilders.get(1-enIndex),
154 langPattern, langCodePattern, enIndex != 0).parse(file, Integer.parseInt(pageLimit));
156 fatalError("Invalid or missing input format: " + inputFormat);
159 dictionaryBuilder.dictionary.sources.add(entrySource);
160 System.out.println("Done: " + file + "\n\n");
164 dictionaryBuilder.build();
166 if (printFile != null) {
167 final PrintStream out = new PrintStream(new File(printFile));
168 dictionaryBuilder.dictionary.print(out);
172 System.out.println("Writing dictionary to: " + dictOutFilename);
173 final RandomAccessFile dictOut = new RandomAccessFile(dictOutFilename, "rw");
174 dictOut.setLength(0);
175 dictionaryBuilder.dictionary.write(dictOut);
178 if (!keyValueArgs.isEmpty()) {
179 System.err.println("WARNING: couldn't parse arguments: " + keyValueArgs);
185 private static void fatalError(String string) {
186 System.err.println(string);