src/com/hughes/android/dictionary/engine/Dictionary.java

   1 // Copyright 2011 Google Inc. All Rights Reserved.
   2 //
   3 // Licensed under the Apache License, Version 2.0 (the "License");
   4 // you may not use this file except in compliance with the License.
   5 // You may obtain a copy of the License at
   6 //
   7 //     http://www.apache.org/licenses/LICENSE-2.0
   8 //
   9 // Unless required by applicable law or agreed to in writing, software
  10 // distributed under the License is distributed on an "AS IS" BASIS,
  11 // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  12 // See the License for the specific language governing permissions and
  13 // limitations under the License.
  14
  15 package com.hughes.android.dictionary.engine;
  16
  17 import java.io.BufferedOutputStream;
  18 import java.io.ByteArrayOutputStream;
  19 import java.io.DataInput;
  20 import java.io.DataInputStream;
  21 import java.io.DataOutput;
  22 import java.io.DataOutputStream;
  23 import java.io.File;
  24 import java.io.FileOutputStream;
  25 import java.io.IOException;
  26 import java.io.ObjectOutputStream;
  27 import java.io.PrintStream;
  28 import java.io.RandomAccessFile;
  29 import java.nio.channels.Channels;
  30 import java.nio.channels.FileChannel;
  31 import java.nio.charset.StandardCharsets;
  32 import java.util.ArrayList;
  33 import java.util.Collections;
  34 import java.util.List;
  35 import java.util.zip.GZIPOutputStream;
  36
  37 import com.hughes.android.dictionary.DictionaryInfo;
  38 import com.hughes.util.CachingList;
  39 import com.hughes.util.raf.RAFList;
  40 import com.hughes.util.raf.RAFListSerializer;
  41 import com.hughes.util.raf.RAFSerializable;
  42
  43 public class Dictionary implements RAFSerializable<Dictionary> {
  44
  45     private static final int CACHE_SIZE = 5000;
  46
  47     private static final int CURRENT_DICT_VERSION = 7;
  48     private static final String END_OF_DICTIONARY = "END OF DICTIONARY";
  49
  50     // persisted
  51     final int dictFileVersion;
  52     private final long creationMillis;
  53     public final String dictInfo;
  54     public final List<PairEntry> pairEntries;
  55     public final List<TextEntry> textEntries;
  56     public final List<HtmlEntry> htmlEntries;
  57     public final List<byte[]> htmlData;
  58     public final List<EntrySource> sources;
  59     public final List<Index> indices;
  60
  61     /**
  62      * dictFileVersion 1 adds: <li>links to sources? dictFileVersion 2 adds: <li>
  63      * counts of tokens in indices.
  64      */
  65
  66     public Dictionary(final String dictInfo) {
  67         this.dictFileVersion = CURRENT_DICT_VERSION;
  68         this.creationMillis = System.currentTimeMillis();
  69         this.dictInfo = dictInfo;
  70         pairEntries = new ArrayList<>();
  71         textEntries = new ArrayList<>();
  72         htmlEntries = new ArrayList<>();
  73         htmlData = null;
  74         sources = new ArrayList<>();
  75         indices = new ArrayList<>();
  76     }
  77
  78     public Dictionary(final FileChannel ch) throws IOException {
  79         DataInput raf = new DataInputStream(Channels.newInputStream(ch));
  80         dictFileVersion = raf.readInt();
  81         if (dictFileVersion < 0 || dictFileVersion > CURRENT_DICT_VERSION) {
  82             throw new IOException("Invalid dictionary version: " + dictFileVersion);
  83         }
  84         creationMillis = raf.readLong();
  85         dictInfo = raf.readUTF();
  86
  87         // Load the sources, then seek past them, because reading them later
  88         // disrupts the offset.
  89         try {
  90             final RAFList<EntrySource> rafSources = RAFList.create(ch, new EntrySource.Serializer(
  91                     this), ch.position(), dictFileVersion, dictInfo + " sources: ");
  92             sources = new ArrayList<>(rafSources);
  93             ch.position(rafSources.getEndOffset());
  94
  95             pairEntries = CachingList.create(
  96                               RAFList.create(ch, new PairEntry.Serializer(this), ch.position(), dictFileVersion, dictInfo + " pairs: "),
  97                               CACHE_SIZE, false);
  98             textEntries = CachingList.create(
  99                               RAFList.create(ch, new TextEntry.Serializer(this), ch.position(), dictFileVersion, dictInfo + " text: "),
 100                               CACHE_SIZE, true);
 101             if (dictFileVersion >= 5) {
 102                 htmlEntries = CachingList.create(
 103                                   RAFList.create(ch, new HtmlEntry.Serializer(this, ch), ch.position(), dictFileVersion, dictInfo + " html: "),
 104                                   CACHE_SIZE, false);
 105             } else {
 106                 htmlEntries = Collections.emptyList();
 107             }
 108             if (dictFileVersion >= 7) {
 109                 htmlData = RAFList.create(ch, new HtmlEntry.DataDeserializer(), ch.position(), dictFileVersion, dictInfo + " html: ");
 110             } else {
 111                 htmlData = null;
 112             }
 113             indices = CachingList.createFullyCached(RAFList.create(ch, new IndexSerializer(ch),
 114                                                     ch.position(), dictFileVersion, dictInfo + " index: "));
 115         } catch (RuntimeException e) {
 116             throw new IOException("RuntimeException loading dictionary", e);
 117         }
 118         final String end = raf.readUTF();
 119         if (!end.equals(END_OF_DICTIONARY)) {
 120             throw new IOException("Dictionary seems corrupt: " + end);
 121         }
 122     }
 123
 124     @Override
 125     public void write(DataOutput out) throws IOException {
 126         RandomAccessFile raf = (RandomAccessFile)out;
 127         if (dictFileVersion < 7) throw new RuntimeException("write function cannot write formats older than v7!");
 128         raf.writeInt(dictFileVersion);
 129         raf.writeLong(creationMillis);
 130         raf.writeUTF(dictInfo);
 131         System.out.println("sources start: " + raf.getFilePointer());
 132         RAFList.write(raf, sources, new EntrySource.Serializer(this));
 133         System.out.println("pair start: " + raf.getFilePointer());
 134         RAFList.write(raf, pairEntries, new PairEntry.Serializer(this), 64, true);
 135         System.out.println("text start: " + raf.getFilePointer());
 136         RAFList.write(raf, textEntries, new TextEntry.Serializer(this));
 137         System.out.println("html index start: " + raf.getFilePointer());
 138         RAFList.write(raf, htmlEntries, new HtmlEntry.Serializer(this, null), 64, true);
 139         System.out.println("html data start: " + raf.getFilePointer());
 140         assert htmlData == null;
 141         RAFList.write(raf, htmlEntries, new HtmlEntry.DataSerializer(), 128, true);
 142         System.out.println("indices start: " + raf.getFilePointer());
 143         RAFList.write(raf, indices, new IndexSerializer(null));
 144         System.out.println("end: " + raf.getFilePointer());
 145         raf.writeUTF(END_OF_DICTIONARY);
 146     }
 147
 148     private void writev6Sources(RandomAccessFile out) throws IOException {
 149         ByteArrayOutputStream toc = new ByteArrayOutputStream();
 150         DataOutputStream tocout = new DataOutputStream(toc);
 151
 152         out.writeInt(sources.size());
 153         long tocPos = out.getFilePointer();
 154         out.seek(tocPos + sources.size() * 8 + 8);
 155         for (EntrySource s : sources) {
 156             long dataPos = out.getFilePointer();
 157             tocout.writeLong(dataPos);
 158
 159             out.writeUTF(s.getName());
 160             out.writeInt(s.getNumEntries());
 161         }
 162         long dataPos = out.getFilePointer();
 163         tocout.writeLong(dataPos);
 164         tocout.close();
 165
 166         out.seek(tocPos);
 167         out.write(toc.toByteArray());
 168         out.seek(dataPos);
 169     }
 170
 171     private void writev6PairEntries(RandomAccessFile out) throws IOException {
 172         ByteArrayOutputStream toc = new ByteArrayOutputStream();
 173         DataOutputStream tocout = new DataOutputStream(toc);
 174
 175         long tocPos = out.getFilePointer();
 176         long dataPos = tocPos + 4 + pairEntries.size() * 8 + 8;
 177
 178         out.seek(dataPos);
 179         DataOutputStream outb = new DataOutputStream(new BufferedOutputStream(new FileOutputStream(out.getFD())));
 180
 181         tocout.writeInt(pairEntries.size());
 182         for (PairEntry pe : pairEntries) {
 183             tocout.writeLong(dataPos + outb.size());
 184
 185             outb.writeShort(pe.entrySource.index());
 186             outb.writeInt(pe.pairs.size());
 187             for (PairEntry.Pair p : pe.pairs) {
 188                 outb.writeUTF(p.lang1);
 189                 outb.writeUTF(p.lang2);
 190             }
 191         }
 192         dataPos += outb.size();
 193         outb.flush();
 194         tocout.writeLong(dataPos);
 195         tocout.close();
 196
 197         out.seek(tocPos);
 198         out.write(toc.toByteArray());
 199         out.seek(dataPos);
 200     }
 201
 202     private void writev6TextEntries(RandomAccessFile out) throws IOException {
 203         ByteArrayOutputStream toc = new ByteArrayOutputStream();
 204         DataOutputStream tocout = new DataOutputStream(toc);
 205
 206         out.writeInt(textEntries.size());
 207         long tocPos = out.getFilePointer();
 208         out.seek(tocPos + textEntries.size() * 8 + 8);
 209         for (TextEntry t : textEntries) {
 210             long dataPos = out.getFilePointer();
 211             tocout.writeLong(dataPos);
 212
 213             out.writeShort(t.entrySource.index());
 214             out.writeUTF(t.text);
 215         }
 216         long dataPos = out.getFilePointer();
 217         tocout.writeLong(dataPos);
 218         tocout.close();
 219
 220         out.seek(tocPos);
 221         out.write(toc.toByteArray());
 222         out.seek(dataPos);
 223     }
 224
 225     private void writev6EmptyList(RandomAccessFile out) throws IOException {
 226         out.writeInt(0);
 227         out.writeLong(out.getFilePointer() + 8);
 228     }
 229
 230     private void writev6HtmlEntries(RandomAccessFile out) throws IOException {
 231         ByteArrayOutputStream toc = new ByteArrayOutputStream();
 232         DataOutputStream tocout = new DataOutputStream(toc);
 233
 234         long tocPos = out.getFilePointer();
 235         long dataPos = tocPos + 4 + htmlEntries.size() * 8 + 8;
 236
 237         out.seek(dataPos);
 238         DataOutputStream outb = new DataOutputStream(new BufferedOutputStream(new FileOutputStream(out.getFD())));
 239
 240         tocout.writeInt(htmlEntries.size());
 241         for (HtmlEntry h : htmlEntries) {
 242             tocout.writeLong(dataPos + outb.size());
 243
 244             outb.writeShort(h.entrySource.index());
 245             outb.writeUTF(h.title);
 246             byte[] data = h.getHtml().getBytes(StandardCharsets.UTF_8);
 247             outb.writeInt(data.length);
 248             ByteArrayOutputStream baos = new ByteArrayOutputStream();
 249             GZIPOutputStream gzout = new GZIPOutputStream(baos);
 250             gzout.write(data);
 251             gzout.close();
 252             outb.writeInt(baos.size());
 253             outb.write(baos.toByteArray());
 254         }
 255         dataPos += outb.size();
 256         outb.flush();
 257         tocout.writeLong(dataPos);
 258         tocout.close();
 259
 260         out.seek(tocPos);
 261         out.write(toc.toByteArray());
 262         out.seek(dataPos);
 263     }
 264
 265     private void writev6HtmlIndices(DataOutputStream out, long pos, List<HtmlEntry> entries) throws IOException {
 266         long dataPos = pos + 4 + entries.size() * 8 + 8;
 267
 268         out.writeInt(entries.size());
 269
 270         // TOC is trivial, so optimize writing it
 271         for (int i = 0; i < entries.size(); i++) {
 272             out.writeLong(dataPos);
 273             dataPos += 4;
 274         }
 275         out.writeLong(dataPos);
 276
 277         for (HtmlEntry e : entries) {
 278             out.writeInt(e.index());
 279         }
 280     }
 281
 282     private void writev6IndexEntries(RandomAccessFile out, List<Index.IndexEntry> entries, int[] prunedRowIdx) throws IOException {
 283         ByteArrayOutputStream toc = new ByteArrayOutputStream();
 284         DataOutputStream tocout = new DataOutputStream(toc);
 285
 286         long tocPos = out.getFilePointer();
 287         long dataPos = tocPos + 4 + entries.size() * 8 + 8;
 288
 289         out.seek(dataPos);
 290         DataOutputStream outb = new DataOutputStream(new BufferedOutputStream(new FileOutputStream(out.getFD())));
 291
 292         tocout.writeInt(entries.size());
 293         for (Index.IndexEntry e : entries) {
 294             tocout.writeLong(dataPos + outb.size());
 295
 296             outb.writeUTF(e.token);
 297
 298             int startRow = e.startRow;
 299             int numRows = e.numRows;
 300             if (prunedRowIdx != null) {
 301                 // note: the start row will always be a TokenRow
 302                 // and thus never be pruned
 303                 int newNumRows = 1;
 304                 for (int i = 1; i < numRows; i++) {
 305                     if (prunedRowIdx[startRow + i] >= 0) newNumRows++;
 306                 }
 307                 startRow = prunedRowIdx[startRow];
 308                 numRows = newNumRows;
 309             }
 310
 311             outb.writeInt(startRow);
 312             outb.writeInt(numRows);
 313             final boolean hasNormalizedForm = !e.token.equals(e.normalizedToken());
 314             outb.writeBoolean(hasNormalizedForm);
 315             if (hasNormalizedForm) outb.writeUTF(e.normalizedToken());
 316             writev6HtmlIndices(outb, dataPos + outb.size(),
 317                                prunedRowIdx == null ? e.htmlEntries : Collections.<HtmlEntry>emptyList());
 318         }
 319         dataPos += outb.size();
 320         outb.flush();
 321         tocout.writeLong(dataPos);
 322         tocout.close();
 323
 324         out.seek(tocPos);
 325         out.write(toc.toByteArray());
 326         out.seek(dataPos);
 327     }
 328
 329     private void writev6Index(RandomAccessFile out, boolean skipHtml) throws IOException {
 330         ByteArrayOutputStream toc = new ByteArrayOutputStream();
 331         DataOutputStream tocout = new DataOutputStream(toc);
 332
 333         out.writeInt(indices.size());
 334         long tocPos = out.getFilePointer();
 335         out.seek(tocPos + indices.size() * 8 + 8);
 336         for (Index idx : indices) {
 337             // create pruned index for skipHtml feature
 338             int[] prunedRowIdx = null;
 339             int prunedSize = 0;
 340             if (skipHtml) {
 341                 prunedRowIdx = new int[idx.rows.size()];
 342                 for (int i = 0; i < idx.rows.size(); i++) {
 343                     final RowBase r = idx.rows.get(i);
 344                     // prune Html entries
 345                     boolean pruned = r instanceof HtmlEntry.Row;
 346                     prunedRowIdx[i] = pruned ? -1 : prunedSize;
 347                     if (!pruned) prunedSize++;
 348                 }
 349             }
 350
 351             long dataPos = out.getFilePointer();
 352             tocout.writeLong(dataPos);
 353
 354             out.writeUTF(idx.shortName);
 355             out.writeUTF(idx.longName);
 356             out.writeUTF(idx.sortLanguage.getIsoCode());
 357             out.writeUTF(idx.normalizerRules);
 358             out.writeBoolean(idx.swapPairEntries);
 359             out.writeInt(idx.mainTokenCount);
 360             writev6IndexEntries(out, idx.sortedIndexEntries, prunedRowIdx);
 361
 362             // write stoplist, serializing the whole Set *shudder*
 363             final ByteArrayOutputStream baos = new ByteArrayOutputStream();
 364             final ObjectOutputStream oos = new ObjectOutputStream(baos);
 365             oos.writeObject(idx.stoplist);
 366             oos.close();
 367             final byte[] bytes = baos.toByteArray();
 368
 369
 370             DataOutputStream outb = new DataOutputStream(new BufferedOutputStream(new FileOutputStream(out.getFD())));
 371             outb.writeInt(bytes.length);
 372             outb.write(bytes);
 373
 374             outb.writeInt(skipHtml ? prunedSize : idx.rows.size());
 375             outb.writeInt(5);
 376             for (RowBase r : idx.rows) {
 377                 int type = 0;
 378                 if (r instanceof PairEntry.Row) {
 379                     type = 0;
 380                 } else if (r instanceof TokenRow) {
 381                     final TokenRow tokenRow = (TokenRow)r;
 382                     type = tokenRow.hasMainEntry ? 1 : 3;
 383                 } else if (r instanceof TextEntry.Row) {
 384                     type = 2;
 385                 } else if (r instanceof HtmlEntry.Row) {
 386                     type = 4;
 387                     if (skipHtml) continue;
 388                 } else {
 389                     throw new RuntimeException("Row type not supported for v6");
 390                 }
 391                 outb.writeByte(type);
 392                 outb.writeInt(r.referenceIndex);
 393             }
 394             outb.flush();
 395         }
 396         long dataPos = out.getFilePointer();
 397         tocout.writeLong(dataPos);
 398         tocout.close();
 399
 400         out.seek(tocPos);
 401         out.write(toc.toByteArray());
 402         out.seek(dataPos);
 403     }
 404
 405     public void writev6(DataOutput out, boolean skipHtml) throws IOException {
 406         RandomAccessFile raf = (RandomAccessFile)out;
 407         raf.writeInt(6);
 408         raf.writeLong(creationMillis);
 409         raf.writeUTF(dictInfo);
 410         System.out.println("sources start: " + raf.getFilePointer());
 411         writev6Sources(raf);
 412         System.out.println("pair start: " + raf.getFilePointer());
 413         writev6PairEntries(raf);
 414         System.out.println("text start: " + raf.getFilePointer());
 415         writev6TextEntries(raf);
 416         System.out.println("html index start: " + raf.getFilePointer());
 417         if (skipHtml) writev6EmptyList(raf);
 418         else writev6HtmlEntries(raf);
 419         System.out.println("indices start: " + raf.getFilePointer());
 420         writev6Index(raf, skipHtml);
 421         System.out.println("end: " + raf.getFilePointer());
 422         raf.writeUTF(END_OF_DICTIONARY);
 423     }
 424
 425     private final class IndexSerializer implements RAFListSerializer<Index> {
 426         private final FileChannel ch;
 427
 428         IndexSerializer(FileChannel ch) {
 429             this.ch = ch;
 430         }
 431
 432         @Override
 433         public Index read(DataInput raf, final int readIndex) throws IOException {
 434             return new Index(Dictionary.this, ch, raf);
 435         }
 436
 437         @Override
 438         public void write(DataOutput raf, Index t) throws IOException {
 439             t.write(raf);
 440         }
 441     }
 442
 443     final RAFListSerializer<HtmlEntry> htmlEntryIndexSerializer = new RAFListSerializer<HtmlEntry>() {
 444         @Override
 445         public void write(DataOutput raf, HtmlEntry t) {
 446             assert false;
 447         }
 448
 449         @Override
 450         public HtmlEntry read(DataInput raf, int readIndex) throws IOException {
 451             return htmlEntries.get(raf.readInt());
 452         }
 453     };
 454
 455     public void print(final PrintStream out) {
 456         out.println("dictInfo=" + dictInfo);
 457         for (final EntrySource entrySource : sources) {
 458             out.printf("EntrySource: %s %d\n", entrySource.name, entrySource.numEntries);
 459         }
 460         out.println();
 461         for (final Index index : indices) {
 462             out.printf("Index: %s %s\n", index.shortName, index.longName);
 463             index.print(out);
 464             out.println();
 465         }
 466     }
 467
 468     public DictionaryInfo getDictionaryInfo() {
 469         final DictionaryInfo result = new DictionaryInfo();
 470         result.creationMillis = this.creationMillis;
 471         result.dictInfo = this.dictInfo;
 472         for (final Index index : indices) {
 473             result.indexInfos.add(index.getIndexInfo());
 474         }
 475         return result;
 476     }
 477
 478     public static DictionaryInfo getDictionaryInfo(final File file) {
 479         RandomAccessFile raf = null;
 480         try {
 481             raf = new RandomAccessFile(file, "r");
 482             final Dictionary dict = new Dictionary(raf.getChannel());
 483             final DictionaryInfo dictionaryInfo = dict.getDictionaryInfo();
 484             dictionaryInfo.uncompressedFilename = file.getName();
 485             dictionaryInfo.uncompressedBytes = file.length();
 486             raf.close();
 487             return dictionaryInfo;
 488         } catch (IOException e) {
 489             final DictionaryInfo dictionaryInfo = new DictionaryInfo();
 490             dictionaryInfo.uncompressedFilename = file.getName();
 491             dictionaryInfo.uncompressedBytes = file.length();
 492             return dictionaryInfo;
 493         } finally {
 494             if (raf != null) {
 495                 try {
 496                     raf.close();
 497                 } catch (IOException e) {
 498                     e.printStackTrace();
 499                 }
 500             }
 501         }
 502     }
 503
 504 }