From 020aa910526ece05ee8514e55a9a951b45ce1fea Mon Sep 17 00:00:00 2001 From: =?utf8?q?Fr=C3=A9d=C3=A9ric=20Perrin?= Date: Sat, 6 Feb 2021 10:57:12 +0000 Subject: [PATCH] Enable single-language dictionary --- .../dictionary/engine/DictionaryBuilder.java | 8 +- .../dictionary/engine/WiktionarySplitter.java | 2 +- .../dictionary/parser/DictFileParser.java | 74 ++++-- .../parser/SingleDictFileParser.java | 134 +++++++++++ .../parser/SingleDictFileParser.java~ | 211 ++++++++++++++++++ 5 files changed, 409 insertions(+), 20 deletions(-) create mode 100644 src/com/hughes/android/dictionary/parser/SingleDictFileParser.java create mode 100644 src/com/hughes/android/dictionary/parser/SingleDictFileParser.java~ diff --git a/src/com/hughes/android/dictionary/engine/DictionaryBuilder.java b/src/com/hughes/android/dictionary/engine/DictionaryBuilder.java index 1d47448..5d5eb5e 100644 --- a/src/com/hughes/android/dictionary/engine/DictionaryBuilder.java +++ b/src/com/hughes/android/dictionary/engine/DictionaryBuilder.java @@ -32,6 +32,7 @@ import org.xml.sax.SAXException; import com.hughes.android.dictionary.parser.DictFileParser; import com.hughes.android.dictionary.parser.Parser; +import com.hughes.android.dictionary.parser.SingleDictFileParser; import com.hughes.android.dictionary.parser.wiktionary.EnForeignParser; import com.hughes.android.dictionary.parser.wiktionary.EnToTranslationParser; import com.hughes.android.dictionary.parser.wiktionary.EnTranslationToTranslationParser; @@ -150,10 +151,13 @@ public class DictionaryBuilder { String inputFormat = keyValueArgs.remove(prefix + "Format"); if ("tab_separated".equals(inputFormat)) { final boolean flipColumns = "true".equals(keyValueArgs.remove(prefix + "FlipColumns")); - new DictFileParser(charset, flipColumns, DictFileParser.TAB, null, dictionaryBuilder).parse(file, entrySource, pageLimit); + new DictFileParser(charset, flipColumns, DictFileParser.TAB, null, (lang2 == null), dictionaryBuilder).parse(file, entrySource, pageLimit); + } else if ("html_tab_separated".equals(inputFormat)) { + new SingleDictFileParser(charset, DictFileParser.TAB, dictionaryBuilder).parse(file, entrySource, pageLimit); + } else if ("chemnitz".equals(inputFormat)) { final boolean flipColumns = "true".equals(keyValueArgs.remove(prefix + "FlipColumns")); - new DictFileParser(charset, flipColumns, DictFileParser.DOUBLE_COLON, DictFileParser.PIPE, dictionaryBuilder).parse(file, entrySource, pageLimit); + new DictFileParser(charset, flipColumns, DictFileParser.DOUBLE_COLON, DictFileParser.PIPE, (lang2 == null), dictionaryBuilder).parse(file, entrySource, pageLimit); } else if ("enwiktionary".equals(inputFormat)) { final String type = keyValueArgs.remove(prefix + "WiktionaryType"); final Pattern langPattern = Pattern.compile(keyValueArgs.remove(prefix + "LangPattern"), Pattern.CASE_INSENSITIVE); diff --git a/src/com/hughes/android/dictionary/engine/WiktionarySplitter.java b/src/com/hughes/android/dictionary/engine/WiktionarySplitter.java index 9d51b78..5e24608 100644 --- a/src/com/hughes/android/dictionary/engine/WiktionarySplitter.java +++ b/src/com/hughes/android/dictionary/engine/WiktionarySplitter.java @@ -80,7 +80,7 @@ public class WiktionarySplitter extends org.xml.sax.helpers.DefaultHandler imple final Map> pathToSelectors = new LinkedHashMap<>(); List selectors; for (final String code : WiktionaryLangs.wikiCodeToIsoCodeToWikiName.keySet()) { - //if (!code.equals("fr")) {continue;} + if (!code.equals("pt")) {continue;} selectors = new ArrayList<>(); pathToSelectors.put(String.format("data/inputs/%swiktionary-pages-articles.xml", code), selectors); for (final Map.Entry entry : WiktionaryLangs.wikiCodeToIsoCodeToWikiName.get(code).entrySet()) { diff --git a/src/com/hughes/android/dictionary/parser/DictFileParser.java b/src/com/hughes/android/dictionary/parser/DictFileParser.java index 8c6424b..1f08428 100644 --- a/src/com/hughes/android/dictionary/parser/DictFileParser.java +++ b/src/com/hughes/android/dictionary/parser/DictFileParser.java @@ -15,8 +15,11 @@ package com.hughes.android.dictionary.parser; import java.io.BufferedReader; +import java.io.BufferedInputStream; +import java.io.DataInputStream; import java.io.File; import java.io.FileInputStream; +import java.io.FileWriter; import java.io.IOException; import java.io.InputStreamReader; import java.nio.charset.Charset; @@ -28,10 +31,13 @@ import java.util.logging.Logger; import java.util.regex.Matcher; import java.util.regex.Pattern; +import com.hughes.android.dictionary.engine.AbstractEntry; import com.hughes.android.dictionary.engine.DictionaryBuilder; import com.hughes.android.dictionary.engine.EntrySource; import com.hughes.android.dictionary.engine.EntryTypeName; +import com.hughes.android.dictionary.engine.HtmlEntry; import com.hughes.android.dictionary.engine.IndexBuilder; +import com.hughes.android.dictionary.engine.IndexBuilder.TokenData; import com.hughes.android.dictionary.engine.IndexedEntry; import com.hughes.android.dictionary.engine.Language; import com.hughes.android.dictionary.engine.PairEntry; @@ -64,6 +70,8 @@ public class DictFileParser implements Parser { final String fieldSplit; final String subfieldSplit; + final boolean singleLang; + final DictionaryBuilder dictBuilder; EntrySource entrySource; @@ -72,18 +80,20 @@ public class DictFileParser implements Parser { public DictFileParser(final Charset charset, boolean flipCols, final String fieldSplit, final String subfieldSplit, + final boolean singleLang, final DictionaryBuilder dictBuilder) { this.charset = charset; this.flipCols = flipCols; this.fieldSplit = fieldSplit; this.subfieldSplit = subfieldSplit; + this.singleLang = singleLang; this.dictBuilder = dictBuilder; } @Override public void parse(final File file, final EntrySource entrySouce, final int pageLimit) throws IOException { this.entrySource = entrySouce; - final BufferedReader reader = new BufferedReader(new InputStreamReader(new FileInputStream(file), charset)); + final BufferedReader reader = new BufferedReader(new InputStreamReader(new FileInputStream(file))); String line; int count = 0; while ((line = reader.readLine()) != null) { @@ -130,7 +140,6 @@ public class DictFileParser implements Parser { subfields[1] = new String[] { fields[1] }; } - final PairEntry pairEntry = new PairEntry(entrySource); for (int i = 0; i < subfields[0].length; ++i) { subfields[0][i] = subfields[0][i].trim(); subfields[1][i] = subfields[1][i].trim(); @@ -144,25 +153,56 @@ public class DictFileParser implements Parser { if (subfields[1][i].isEmpty()) { subfields[1][i] = "__"; } - pairEntry.pairs.add(new PairEntry.Pair(subfields[0][i], subfields[1][i])); } - final IndexedEntry entryData = new IndexedEntry(pairEntry); - entryData.isValid = true; - - for (int l = 0; l < 2; ++l) { - // alreadyDone.clear(); - - final IndexBuilder indexBuilder = dictBuilder.indexBuilders.get(l); - for (int j = 0; j < subfields[l].length; ++j) { - String subfield = subfields[l][j]; - if (indexBuilder.index.sortLanguage == Language.de) { - subfield = parseField_DE(indexBuilder, subfield, entryData, j); - } else if (indexBuilder.index.sortLanguage == Language.en) { - subfield = parseField_EN(indexBuilder, subfield, entryData, j); + + if (singleLang) { + HtmlEntry htmlEntry = new HtmlEntry(entrySource, fields[0]); + htmlEntry.html = StringUtil.escapeUnicodeToPureHtml(fields[1]); + + final IndexBuilder titleIndexBuilder = dictBuilder.indexBuilders.get(0); + htmlEntry.addToDictionary(titleIndexBuilder.index.dict); + + TokenData tokenData = titleIndexBuilder.getOrCreateTokenData(fields[0]); + tokenData.hasMainEntry = true; + tokenData.htmlEntries.add(0, htmlEntry); + + final String[] tokens = NON_CHAR.split(fields[0]); + if (tokens.length > 1) { + for (final String token : tokens) { + assert token.length() >= 1; + assert token.indexOf("-") == -1; + if (/*!alreadyDone.contains(token) && */!token.isEmpty()) { + tokenData = titleIndexBuilder.getOrCreateTokenData(token); + tokenData.htmlEntries.add(htmlEntry); + } + } + } + final IndexedEntry entryData = new IndexedEntry(htmlEntry); + entryData.isValid = true; + + } else { + PairEntry pairEntry = new PairEntry(entrySource); + for (int i = 0; i < subfields[0].length; ++i) { + pairEntry.pairs.add(new PairEntry.Pair(subfields[0][i], subfields[1][i])); + } + final IndexedEntry entryData = new IndexedEntry(pairEntry); + entryData.isValid = true; + for (int l = 0; l < 2; ++l) { + // alreadyDone.clear(); + + final IndexBuilder indexBuilder = dictBuilder.indexBuilders.get(l); + for (int j = 0; j < subfields[l].length; ++j) { + String subfield = subfields[l][j]; + if (indexBuilder.index.sortLanguage == Language.de) { + subfield = parseField_DE(indexBuilder, subfield, entryData, j); + } else if (indexBuilder.index.sortLanguage == Language.en) { + subfield = parseField_EN(indexBuilder, subfield, entryData, j); + } + parseFieldGeneric(indexBuilder, subfield, entryData, j, subfields[l].length); } - parseFieldGeneric(indexBuilder, subfield, entryData, j, subfields[l].length); } } + } private StringBuilder extractParenthesized(StringBuilder in, String startChar, String endChar) { diff --git a/src/com/hughes/android/dictionary/parser/SingleDictFileParser.java b/src/com/hughes/android/dictionary/parser/SingleDictFileParser.java new file mode 100644 index 0000000..0f6f95f --- /dev/null +++ b/src/com/hughes/android/dictionary/parser/SingleDictFileParser.java @@ -0,0 +1,134 @@ +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.hughes.android.dictionary.parser; + +import java.io.BufferedReader; +import java.io.File; +import java.io.FileInputStream; +import java.io.IOException; +import java.io.InputStreamReader; +import java.nio.charset.Charset; +import java.util.Arrays; +import java.util.Collections; +import java.util.LinkedHashSet; +import java.util.Set; +import java.util.logging.Logger; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +import com.hughes.android.dictionary.engine.DictionaryBuilder; +import com.hughes.android.dictionary.engine.EntrySource; +import com.hughes.android.dictionary.engine.EntryTypeName; +import com.hughes.android.dictionary.engine.HtmlEntry; +import com.hughes.android.dictionary.engine.IndexBuilder; +import com.hughes.android.dictionary.engine.IndexBuilder.TokenData; +import com.hughes.android.dictionary.engine.IndexedEntry; +import com.hughes.android.dictionary.engine.Language; +import com.hughes.android.dictionary.engine.PairEntry; +import com.hughes.util.StringUtil; + +public class SingleDictFileParser implements Parser { + + static final Logger logger = Logger.getLogger(SingleDictFileParser.class.getName()); + + // Dictcc + public static final String TAB = "\t"; + + // Chemnitz + public static final String DOUBLE_COLON = " :: "; + public static final String PIPE = "|"; + + static final Pattern SPACES = Pattern.compile("\\s+"); + + static final Pattern CURLY_BRACED = Pattern.compile("\\{([^}]+)\\}"); + + // http://www.regular-expressions.info/unicode.html + static final Pattern NON_CHAR_DASH = Pattern.compile("[^-'\\p{L}\\p{M}\\p{N}]+"); + public static final Pattern NON_CHAR = Pattern.compile("[^\\p{L}\\p{M}\\p{N}]+"); + + static final Pattern TRIM_PUNC = Pattern.compile("^[^\\p{L}\\p{M}\\p{N}]+|[^\\p{L}\\p{M}\\p{N}]+$"); + + final Charset charset; + + final String fieldSplit; + + final DictionaryBuilder dictBuilder; + + EntrySource entrySource; + + // final Set alreadyDone = new HashSet(); + + public SingleDictFileParser(final Charset charset, + final String fieldSplit, + final DictionaryBuilder dictBuilder) { + this.charset = charset; + this.fieldSplit = fieldSplit; + this.dictBuilder = dictBuilder; + } + + @Override + public void parse(final File file, final EntrySource entrySouce, final int pageLimit) throws IOException { + this.entrySource = entrySouce; + final BufferedReader reader = new BufferedReader(new InputStreamReader(new FileInputStream(file), charset)); + String line; + int count = 0; + while ((line = reader.readLine()) != null) { + if (pageLimit >= 0 && count >= pageLimit) { + return; + } + if (count % 10000 == 0) { + logger.info("count=" + count + ", line=" + line); + } + parseLine(line); + ++count; + } + } + + private void parseLine(final String line) { + if (line.startsWith("#") || line.isEmpty()) { + logger.info("Skipping comment line: " + line); + return; + } + final String[] fields = StringUtil.split(line, fieldSplit); + if (fields.length < 2 || fields.length > 4) { + logger.warning("Malformed line, expected 3 or 4 fields, got " + fields.length + ": " + line); + return; + } + + String headword = SPACES.matcher(fields[0]).replaceAll(" ").trim(); + String definition = SPACES.matcher(fields[1]).replaceAll(" ").trim(); + + final HtmlEntry htmlEntry = new HtmlEntry(entrySource, headword); + htmlEntry.html = definition; + IndexedEntry entryData = new IndexedEntry(htmlEntry); + entryData.isValid = true; + + final IndexBuilder titleIndexBuilder = dictBuilder.indexBuilders.get(0); + final TokenData tokenData = titleIndexBuilder.getOrCreateTokenData(headword); + tokenData.hasMainEntry = true; + + htmlEntry.addToDictionary(titleIndexBuilder.index.dict); + tokenData.htmlEntries.add(htmlEntry); + + entryData = null; + } + + public static Set tokenize(final String text, final Pattern pattern) { + final String[] split = pattern.split(text); + final Set result = new LinkedHashSet<>(Arrays.asList(split)); + result.remove(""); + return result; + } + + +} diff --git a/src/com/hughes/android/dictionary/parser/SingleDictFileParser.java~ b/src/com/hughes/android/dictionary/parser/SingleDictFileParser.java~ new file mode 100644 index 0000000..fad6595 --- /dev/null +++ b/src/com/hughes/android/dictionary/parser/SingleDictFileParser.java~ @@ -0,0 +1,211 @@ +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.hughes.android.dictionary.parser; + +import java.io.BufferedReader; +import java.io.File; +import java.io.FileInputStream; +import java.io.IOException; +import java.io.InputStreamReader; +import java.nio.charset.Charset; +import java.util.Arrays; +import java.util.Collections; +import java.util.LinkedHashSet; +import java.util.Set; +import java.util.logging.Logger; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +import com.hughes.android.dictionary.engine.DictionaryBuilder; +import com.hughes.android.dictionary.engine.EntrySource; +import com.hughes.android.dictionary.engine.EntryTypeName; +import com.hughes.android.dictionary.engine.HtmlEntry; +import com.hughes.android.dictionary.engine.IndexBuilder; +import com.hughes.android.dictionary.engine.IndexBuilder.TokenData; +import com.hughes.android.dictionary.engine.IndexedEntry; +import com.hughes.android.dictionary.engine.Language; +import com.hughes.android.dictionary.engine.PairEntry; +import com.hughes.util.StringUtil; + +public class SingleDictFileParser implements Parser { + + static final Logger logger = Logger.getLogger(SingleDictFileParser.class.getName()); + + // Dictcc + public static final String TAB = "\t"; + + // Chemnitz + public static final String DOUBLE_COLON = " :: "; + public static final String PIPE = "|"; + + static final Pattern SPACES = Pattern.compile("\\s+"); + + static final Pattern CURLY_BRACED = Pattern.compile("\\{([^}]+)\\}"); + + // http://www.regular-expressions.info/unicode.html + static final Pattern NON_CHAR_DASH = Pattern.compile("[^-'\\p{L}\\p{M}\\p{N}]+"); + public static final Pattern NON_CHAR = Pattern.compile("[^\\p{L}\\p{M}\\p{N}]+"); + + static final Pattern TRIM_PUNC = Pattern.compile("^[^\\p{L}\\p{M}\\p{N}]+|[^\\p{L}\\p{M}\\p{N}]+$"); + + final Charset charset; + + final String fieldSplit; + + final DictionaryBuilder dictBuilder; + + EntrySource entrySource; + + // final Set alreadyDone = new HashSet(); + + public SingleDictFileParser(final Charset charset, + final String fieldSplit, + final DictionaryBuilder dictBuilder) { + this.charset = charset; + this.fieldSplit = fieldSplit; + this.dictBuilder = dictBuilder; + } + + @Override + public void parse(final File file, final EntrySource entrySouce, final int pageLimit) throws IOException { + this.entrySource = entrySouce; + final BufferedReader reader = new BufferedReader(new InputStreamReader(new FileInputStream(file), charset)); + String line; + int count = 0; + while ((line = reader.readLine()) != null) { + if (pageLimit >= 0 && count >= pageLimit) { + return; + } + if (count % 10000 == 0) { + logger.info("count=" + count + ", line=" + line); + } + parseLine(line); + ++count; + } + } + + private void parseLine(final String line) { + if (line.startsWith("#") || line.isEmpty()) { + logger.info("Skipping comment line: " + line); + return; + } + final String[] fields = StringUtil.split(line, fieldSplit); + if (fields.length < 2 || fields.length > 4) { + logger.warning("Malformed line, expected 3 or 4 fields, got " + fields.length + ": " + line); + return; + } + + String headword = SPACES.matcher(fields[0]).replaceAll(" ").trim(); + String definition = SPACES.matcher(fields[1]).replaceAll(" ").trim(); + + final HtmlEntry htmlEntry = new HtmlEntry(entrySource, headword); + htmlEntry.html = definition; + IndexedEntry entryData = new IndexedEntry(htmlEntry); + entryData.isValid = true; + + final IndexBuilder titleIndexBuilder = dictBuilder.indexBuilders.get(0); + final TokenData tokenData = titleIndexBuilder.getOrCreateTokenData(headword); + tokenData.hasMainEntry = true; + + htmlEntry.addToDictionary(titleIndexBuilder.index.dict); + tokenData.htmlEntries.add(htmlEntry); + + entryData = null; + } + + private StringBuilder extractParenthesized(StringBuilder in, String startChar, String endChar) { + StringBuilder res = new StringBuilder(); + int pos = 0; + while ((pos = in.indexOf(startChar, pos)) != -1) { + int end = in.indexOf(endChar, pos + 1); + if (end == -1) break; + res.append(in, pos + 1, end).append(" "); + in.replace(pos, end + 1, " "); + pos++; // skip the just appended space + } + return res; + } + + private void parseFieldGeneric(final IndexBuilder indexBuilder, String field, + final IndexedEntry entryData, final int subfieldIdx, final int numSubFields) { + final StringBuilder fieldsb = new StringBuilder(field); + // remove bracketed and parenthesized stuff. + final StringBuilder bracketed = extractParenthesized(fieldsb, "[", "]"); + final StringBuilder parenthesized = extractParenthesized(fieldsb, "(", ")"); + + field = fieldsb.toString().trim(); + + // split words on non -A-z0-9, do them. + final String[] tokens = NON_CHAR_DASH.split(field); + + final EntryTypeName entryTypeName; + if (numSubFields == 1) { + assert subfieldIdx == 0; + if (tokens.length == 1) { + entryTypeName = EntryTypeName.ONE_WORD; + } else if (tokens.length == 2) { + entryTypeName = EntryTypeName.TWO_WORDS; + } else if (tokens.length == 3) { + entryTypeName = EntryTypeName.THREE_WORDS; + } else if (tokens.length == 4) { + entryTypeName = EntryTypeName.FOUR_WORDS; + } else { + entryTypeName = EntryTypeName.FIVE_OR_MORE_WORDS; + } + } else { + assert numSubFields > 1; + if (subfieldIdx == 0) { + if (tokens.length == 1) { + entryTypeName = EntryTypeName.MULTIROW_HEAD_ONE_WORD; + } else { + entryTypeName = EntryTypeName.MULTIROW_HEAD_MANY_WORDS; + } + } else { + assert subfieldIdx > 0; + if (tokens.length == 1) { + entryTypeName = EntryTypeName.MULTIROW_TAIL_ONE_WORD; + } else { + entryTypeName = EntryTypeName.MULTIROW_TAIL_MANY_WORDS; + } + } + } + + for (String token : tokens) { + token = TRIM_PUNC.matcher(token).replaceAll(""); + if (/*!alreadyDone.contains(token) && */!token.isEmpty()) { + indexBuilder.addEntryWithTokens(entryData, Collections.singleton(token), entryTypeName); + // alreadyDone.add(token); + + // also split words on dashes, do them, too. + if (token.indexOf('-') != -1) { + final String[] dashed = StringUtil.split(token, "-"); + for (final String dashedToken : dashed) { + if (/*!alreadyDone.contains(dashedToken) && */!dashedToken.isEmpty()) { + indexBuilder.addEntryWithTokens(entryData, Collections.singleton(dashedToken), EntryTypeName.PART_OF_HYPHENATED); + } + } + } + + } // if (!alreadyDone.contains(token)) { + } // for (final String token : tokens) { + } + + public static Set tokenize(final String text, final Pattern pattern) { + final String[] split = pattern.split(text); + final Set result = new LinkedHashSet<>(Arrays.asList(split)); + result.remove(""); + return result; + } + + +} -- 2.43.0