1 package com.hughes.android.dictionary.engine;
4 import java.io.IOException;
5 import java.io.PrintStream;
6 import java.io.RandomAccessFile;
7 import java.nio.charset.Charset;
8 import java.util.ArrayList;
11 import java.util.regex.Pattern;
13 import javax.xml.parsers.ParserConfigurationException;
15 import org.xml.sax.SAXException;
17 import com.hughes.android.dictionary.parser.DictFileParser;
18 import com.hughes.android.dictionary.parser.EnWiktionaryXmlParser;
19 import com.hughes.util.Args;
20 import com.hughes.util.FileUtil;
28 --dictInfo=@dictInfo.txt
30 --input0=/Users/thadh/personal/quickDic/de-en-chemnitz.txt
33 --input0Format=chemnitz
35 --input1=/Users/thadh/personal/quickDic/dewiktionary-20100326-pages-articles.xml
36 --input1Name=wiktionary
37 --input1Format=wiktionary
39 --input2=/Users/thadh/personal/quickDic/de-en-dictcc.txt
41 --input2Charset=Cp1252
45 public class DictionaryBuilder {
47 public final Dictionary dictionary;
48 public final List<IndexBuilder> indexBuilders = new ArrayList<IndexBuilder>();
50 public DictionaryBuilder(final String dictInfo, final Language lang0, final Language lang1, final String normalizerRules1, final String normalizerRules2) {
51 dictionary = new Dictionary(dictInfo);
52 indexBuilders.add(new IndexBuilder(this, lang0.getSymbol(), lang0.getSymbol() + "->" + lang1.getSymbol(), lang0, normalizerRules1, false));
53 indexBuilders.add(new IndexBuilder(this, lang1.getSymbol(), lang1.getSymbol() + "->" + lang0.getSymbol(), lang1, normalizerRules2, true));
57 for (final IndexBuilder indexBuilder : indexBuilders) {
59 dictionary.indices.add(indexBuilder.index);
63 public static void main(final String[] args) throws IOException, ParserConfigurationException, SAXException {
64 final Map<String,String> keyValueArgs = Args.keyValueArgs(args);
66 final Language lang1 = Language.lookup(keyValueArgs.remove("lang1"));
67 final Language lang2 = Language.lookup(keyValueArgs.remove("lang2"));
68 if (lang1 == null || lang2 == null) {
69 fatalError("--lang1= and --lang2= must both be specified.");
72 String normalizerRules1 = keyValueArgs.remove("normalizerRules1");
73 String normalizerRules2 = keyValueArgs.remove("normalizerRules2");
74 if (normalizerRules1 == null) {
75 normalizerRules1 = lang1.getDefaultNormalizerRules();
77 if (normalizerRules2 == null) {
78 normalizerRules2 = lang2.getDefaultNormalizerRules();
81 final String dictOutFilename = keyValueArgs.remove("dictOut");
82 if (dictOutFilename == null) {
83 fatalError("--dictOut= must be specified.");
86 String dictInfo = keyValueArgs.remove("dictInfo");
87 if (dictInfo == null) {
88 fatalError("--dictInfo= must be specified.");
90 if (dictInfo.startsWith("@")) {
91 dictInfo = FileUtil.readToString(new File(dictInfo.substring(1)));
94 final String printFile = keyValueArgs.remove("print");
96 System.out.println("lang1=" + lang1);
97 System.out.println("lang2=" + lang2);
98 System.out.println("normalizerRules1=" + normalizerRules1);
99 System.out.println("normalizerRules2=" + normalizerRules2);
100 System.out.println("dictInfo=" + dictInfo);
101 System.out.println("dictOut=" + dictOutFilename);
103 final DictionaryBuilder dictionaryBuilder = new DictionaryBuilder(dictInfo, lang1, lang2, normalizerRules1, normalizerRules2);
105 for (int i = 0; i < 100; ++i) {
106 final String prefix = "input" + i;
107 if (keyValueArgs.containsKey(prefix)) {
108 final File file = new File(keyValueArgs.remove(prefix));
109 System.out.println("Processing: " + file);
110 String charsetName = keyValueArgs.remove(prefix + "Charset");
111 if (charsetName == null) {
112 charsetName = "UTF8";
114 final Charset charset = Charset.forName(charsetName);
115 String inputName = keyValueArgs.remove(prefix + "Name");
116 if (inputName == null) {
117 fatalError("Must specify human readable name for: " + prefix + "Name");
120 String inputFormat = keyValueArgs.remove(prefix + "Format");
121 if ("dictcc".equals(inputFormat)) {
122 new DictFileParser(charset, false, DictFileParser.TAB, null, dictionaryBuilder, dictionaryBuilder.indexBuilders.toArray(new IndexBuilder[0]), null).parseFile(file);
123 } else if ("chemnitz".equals(inputFormat)) {
124 new DictFileParser(charset, false, DictFileParser.DOUBLE_COLON, DictFileParser.PIPE, dictionaryBuilder, dictionaryBuilder.indexBuilders.toArray(new IndexBuilder[0]), null).parseFile(file);
125 } else if ("enwiktionary".equals(inputFormat)) {
126 final Pattern[] translationPatterns = new Pattern[2];
127 translationPatterns[0] = Pattern.compile(keyValueArgs.remove(prefix + "TranslationPattern1"));
128 translationPatterns[1] = Pattern.compile(keyValueArgs.remove(prefix + "TranslationPattern2"));
129 final int enIndex = Integer.parseInt(keyValueArgs.remove(prefix + "EnIndex")) - 1;
130 if (enIndex < 0 || enIndex >= 2) {
131 fatalError("Must be 1 or 2: " + prefix + "EnIndex");
133 new EnWiktionaryXmlParser(dictionaryBuilder, translationPatterns, enIndex).parse(file);
135 fatalError("Invalid or missing input format: " + inputFormat);
138 final EntrySource entrySource = new EntrySource(dictionaryBuilder.dictionary.sources.size(), inputName);
139 dictionaryBuilder.dictionary.sources.add(entrySource);
140 System.out.println("Done: " + file + "\n\n");
144 dictionaryBuilder.build();
146 if (printFile != null) {
147 final PrintStream out = new PrintStream(new File(printFile));
148 dictionaryBuilder.dictionary.print(out);
152 System.out.println("Writing dictionary to: " + dictOutFilename);
153 final RandomAccessFile dictOut = new RandomAccessFile(dictOutFilename, "rw");
154 dictOut.setLength(0);
155 dictionaryBuilder.dictionary.write(dictOut);
158 if (!keyValueArgs.isEmpty()) {
159 System.err.println("WARNING: couldn't parse arguments: " + keyValueArgs);
165 private static void fatalError(String string) {
166 System.err.println(string);