src/com/hughes/android/dictionary/engine/Dictionary.java

   1 // Copyright 2011 Google Inc. All Rights Reserved.
   2 //
   3 // Licensed under the Apache License, Version 2.0 (the "License");
   4 // you may not use this file except in compliance with the License.
   5 // You may obtain a copy of the License at
   6 //
   7 //     http://www.apache.org/licenses/LICENSE-2.0
   8 //
   9 // Unless required by applicable law or agreed to in writing, software
  10 // distributed under the License is distributed on an "AS IS" BASIS,
  11 // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  12 // See the License for the specific language governing permissions and
  13 // limitations under the License.
  14
  15 package com.hughes.android.dictionary.engine;
  16
  17 import com.hughes.android.dictionary.DictionaryInfo;
  18 import com.hughes.util.CachingList;
  19 import com.hughes.util.raf.RAFList;
  20 import com.hughes.util.raf.RAFListSerializer;
  21 import com.hughes.util.raf.RAFSerializable;
  22
  23 import java.io.ByteArrayOutputStream;
  24 import java.io.DataInput;
  25 import java.io.DataInputStream;
  26 import java.io.DataOutput;
  27 import java.io.File;
  28 import java.io.IOException;
  29 import java.io.ObjectOutputStream;
  30 import java.io.PrintStream;
  31 import java.io.RandomAccessFile;
  32 import java.nio.channels.Channels;
  33 import java.nio.channels.FileChannel;
  34 import java.nio.charset.StandardCharsets;
  35 import java.util.ArrayList;
  36 import java.util.Collections;
  37 import java.util.List;
  38 import java.util.zip.GZIPOutputStream;
  39
  40 public class Dictionary implements RAFSerializable<Dictionary> {
  41
  42     private static final int CACHE_SIZE = 5000;
  43
  44     private static final int CURRENT_DICT_VERSION = 7;
  45     private static final String END_OF_DICTIONARY = "END OF DICTIONARY";
  46
  47     // persisted
  48     final int dictFileVersion;
  49     private final long creationMillis;
  50     public final String dictInfo;
  51     public final List<PairEntry> pairEntries;
  52     public final List<TextEntry> textEntries;
  53     public final List<HtmlEntry> htmlEntries;
  54     public final List<byte[]> htmlData;
  55     public final List<EntrySource> sources;
  56     public final List<Index> indices;
  57
  58     /**
  59      * dictFileVersion 1 adds: <li>links to sources? dictFileVersion 2 adds: <li>
  60      * counts of tokens in indices.
  61      */
  62
  63     public Dictionary(final String dictInfo) {
  64         this.dictFileVersion = CURRENT_DICT_VERSION;
  65         this.creationMillis = System.currentTimeMillis();
  66         this.dictInfo = dictInfo;
  67         pairEntries = new ArrayList<>();
  68         textEntries = new ArrayList<>();
  69         htmlEntries = new ArrayList<>();
  70         htmlData = null;
  71         sources = new ArrayList<>();
  72         indices = new ArrayList<>();
  73     }
  74
  75     public Dictionary(final FileChannel ch) throws IOException {
  76         DataInput raf = new DataInputStream(Channels.newInputStream(ch));
  77         dictFileVersion = raf.readInt();
  78         if (dictFileVersion < 0 || dictFileVersion > CURRENT_DICT_VERSION) {
  79             throw new IOException("Invalid dictionary version: " + dictFileVersion);
  80         }
  81         creationMillis = raf.readLong();
  82         dictInfo = raf.readUTF();
  83
  84         // Load the sources, then seek past them, because reading them later
  85         // disrupts the offset.
  86         try {
  87             final RAFList<EntrySource> rafSources = RAFList.create(ch, new EntrySource.Serializer(
  88                     this), ch.position(), dictFileVersion, dictInfo + " sources: ");
  89             sources = new ArrayList<>(rafSources);
  90             ch.position(rafSources.getEndOffset());
  91
  92             pairEntries = CachingList.create(
  93                               RAFList.create(ch, new PairEntry.Serializer(this), ch.position(), dictFileVersion, dictInfo + " pairs: "),
  94                               CACHE_SIZE, false);
  95             textEntries = CachingList.create(
  96                               RAFList.create(ch, new TextEntry.Serializer(this), ch.position(), dictFileVersion, dictInfo + " text: "),
  97                               CACHE_SIZE, true);
  98             if (dictFileVersion >= 5) {
  99                 htmlEntries = CachingList.create(
 100                                   RAFList.create(ch, new HtmlEntry.Serializer(this, ch), ch.position(), dictFileVersion, dictInfo + " html: "),
 101                                   CACHE_SIZE, false);
 102             } else {
 103                 htmlEntries = Collections.emptyList();
 104             }
 105             if (dictFileVersion >= 7) {
 106                 htmlData = RAFList.create(ch, new HtmlEntry.DataDeserializer(), ch.position(), dictFileVersion, dictInfo + " html: ");
 107             } else {
 108                 htmlData = null;
 109             }
 110             indices = CachingList.createFullyCached(RAFList.create(ch, new IndexSerializer(ch),
 111                                                     ch.position(), dictFileVersion, dictInfo + " index: "));
 112         } catch (RuntimeException e) {
 113             throw new IOException("RuntimeException loading dictionary", e);
 114         }
 115         final String end = raf.readUTF();
 116         if (!end.equals(END_OF_DICTIONARY)) {
 117             throw new IOException("Dictionary seems corrupt: " + end);
 118         }
 119     }
 120
 121     @Override
 122     public void write(DataOutput out) throws IOException {
 123         RandomAccessFile raf = (RandomAccessFile)out;
 124         if (dictFileVersion < 7) throw new RuntimeException("write function cannot write formats older than v7!");
 125         raf.writeInt(dictFileVersion);
 126         raf.writeLong(creationMillis);
 127         raf.writeUTF(dictInfo);
 128         System.out.println("sources start: " + raf.getFilePointer());
 129         RAFList.write(raf, sources, new EntrySource.Serializer(this));
 130         System.out.println("pair start: " + raf.getFilePointer());
 131         RAFList.write(raf, pairEntries, new PairEntry.Serializer(this), 64, true);
 132         System.out.println("text start: " + raf.getFilePointer());
 133         RAFList.write(raf, textEntries, new TextEntry.Serializer(this));
 134         System.out.println("html index start: " + raf.getFilePointer());
 135         RAFList.write(raf, htmlEntries, new HtmlEntry.Serializer(this, null), 64, true);
 136         System.out.println("html data start: " + raf.getFilePointer());
 137         assert htmlData == null;
 138         RAFList.write(raf, htmlEntries, new HtmlEntry.DataSerializer(), 128, true);
 139         System.out.println("indices start: " + raf.getFilePointer());
 140         RAFList.write(raf, indices, new IndexSerializer(null));
 141         System.out.println("end: " + raf.getFilePointer());
 142         raf.writeUTF(END_OF_DICTIONARY);
 143     }
 144
 145     private void writev6Sources(RandomAccessFile out) throws IOException {
 146         out.writeInt(sources.size());
 147         long tocPos = out.getFilePointer();
 148         out.seek(tocPos + sources.size() * 8 + 8);
 149         for (EntrySource s : sources) {
 150             long dataPos = out.getFilePointer();
 151             out.seek(tocPos);
 152             out.writeLong(dataPos);
 153             tocPos += 8;
 154             out.seek(dataPos);
 155             out.writeUTF(s.getName());
 156             out.writeInt(s.getNumEntries());
 157         }
 158         long dataPos = out.getFilePointer();
 159         out.seek(tocPos);
 160         out.writeLong(dataPos);
 161         out.seek(dataPos);
 162     }
 163
 164     private void writev6PairEntries(RandomAccessFile out) throws IOException {
 165         out.writeInt(pairEntries.size());
 166         long tocPos = out.getFilePointer();
 167         out.seek(tocPos + pairEntries.size() * 8 + 8);
 168         for (PairEntry pe : pairEntries) {
 169             long dataPos = out.getFilePointer();
 170             out.seek(tocPos);
 171             out.writeLong(dataPos);
 172             tocPos += 8;
 173             out.seek(dataPos);
 174             out.writeShort(pe.entrySource.index());
 175             out.writeInt(pe.pairs.size());
 176             for (PairEntry.Pair p : pe.pairs) {
 177                 out.writeUTF(p.lang1);
 178                 out.writeUTF(p.lang2);
 179             }
 180         }
 181         long dataPos = out.getFilePointer();
 182         out.seek(tocPos);
 183         out.writeLong(dataPos);
 184         out.seek(dataPos);
 185     }
 186
 187     private void writev6TextEntries(RandomAccessFile out) throws IOException {
 188         out.writeInt(textEntries.size());
 189         long tocPos = out.getFilePointer();
 190         out.seek(tocPos + textEntries.size() * 8 + 8);
 191         for (TextEntry t : textEntries) {
 192             long dataPos = out.getFilePointer();
 193             out.seek(tocPos);
 194             out.writeLong(dataPos);
 195             tocPos += 8;
 196             out.seek(dataPos);
 197             out.writeShort(t.entrySource.index());
 198             out.writeUTF(t.text);
 199         }
 200         long dataPos = out.getFilePointer();
 201         out.seek(tocPos);
 202         out.writeLong(dataPos);
 203         out.seek(dataPos);
 204     }
 205
 206     private void writev6HtmlEntries(RandomAccessFile out) throws IOException {
 207         out.writeInt(htmlEntries.size());
 208         long tocPos = out.getFilePointer();
 209         out.seek(tocPos + htmlEntries.size() * 8 + 8);
 210         for (HtmlEntry h : htmlEntries) {
 211             long dataPos = out.getFilePointer();
 212             out.seek(tocPos);
 213             out.writeLong(dataPos);
 214             tocPos += 8;
 215             out.seek(dataPos);
 216             out.writeShort(h.entrySource.index());
 217             out.writeUTF(h.title);
 218             byte[] data = h.getHtml().getBytes(StandardCharsets.UTF_8);
 219             out.writeInt(data.length);
 220             ByteArrayOutputStream baos = new ByteArrayOutputStream();
 221             GZIPOutputStream gzout = new GZIPOutputStream(baos);
 222             gzout.write(data);
 223             gzout.close();
 224             out.writeInt(baos.size());
 225             out.write(baos.toByteArray());
 226         }
 227         long dataPos = out.getFilePointer();
 228         out.seek(tocPos);
 229         out.writeLong(dataPos);
 230         out.seek(dataPos);
 231     }
 232
 233     private void writev6HtmlIndices(RandomAccessFile out, List<HtmlEntry> entries) throws IOException {
 234         out.writeInt(entries.size());
 235         long tocPos = out.getFilePointer();
 236         out.seek(tocPos + entries.size() * 8 + 8);
 237         for (HtmlEntry e : entries) {
 238             long dataPos = out.getFilePointer();
 239             out.seek(tocPos);
 240             out.writeLong(dataPos);
 241             tocPos += 8;
 242             out.seek(dataPos);
 243             out.writeInt(e.index());
 244         }
 245         long dataPos = out.getFilePointer();
 246         out.seek(tocPos);
 247         out.writeLong(dataPos);
 248         out.seek(dataPos);
 249     }
 250
 251     private void writev6IndexEntries(RandomAccessFile out, List<Index.IndexEntry> entries) throws IOException {
 252         out.writeInt(entries.size());
 253         long tocPos = out.getFilePointer();
 254         out.seek(tocPos + entries.size() * 8 + 8);
 255         for (Index.IndexEntry e : entries) {
 256             long dataPos = out.getFilePointer();
 257             out.seek(tocPos);
 258             out.writeLong(dataPos);
 259             tocPos += 8;
 260             out.seek(dataPos);
 261             out.writeUTF(e.token);
 262             out.writeInt(e.startRow);
 263             out.writeInt(e.numRows);
 264             final boolean hasNormalizedForm = !e.token.equals(e.normalizedToken());
 265             out.writeBoolean(hasNormalizedForm);
 266             if (hasNormalizedForm) out.writeUTF(e.normalizedToken());
 267             writev6HtmlIndices(out, e.htmlEntries);
 268         }
 269         long dataPos = out.getFilePointer();
 270         out.seek(tocPos);
 271         out.writeLong(dataPos);
 272         out.seek(dataPos);
 273     }
 274
 275     private void writev6Index(RandomAccessFile out) throws IOException {
 276         out.writeInt(indices.size());
 277         long tocPos = out.getFilePointer();
 278         out.seek(tocPos + indices.size() * 8 + 8);
 279         for (Index idx : indices) {
 280             long dataPos = out.getFilePointer();
 281             out.seek(tocPos);
 282             out.writeLong(dataPos);
 283             tocPos += 8;
 284             out.seek(dataPos);
 285             out.writeUTF(idx.shortName);
 286             out.writeUTF(idx.longName);
 287             out.writeUTF(idx.sortLanguage.getIsoCode());
 288             out.writeUTF(idx.normalizerRules);
 289             out.writeBoolean(idx.swapPairEntries);
 290             out.writeInt(idx.mainTokenCount);
 291             writev6IndexEntries(out, idx.sortedIndexEntries);
 292
 293             // write stoplist, serializing the whole Set *shudder*
 294             final ByteArrayOutputStream baos = new ByteArrayOutputStream();
 295             final ObjectOutputStream oos = new ObjectOutputStream(baos);
 296             oos.writeObject(idx.stoplist);
 297             oos.close();
 298             final byte[] bytes = baos.toByteArray();
 299             out.writeInt(bytes.length);
 300             out.write(bytes);
 301
 302             out.writeInt(idx.rows.size());
 303             out.writeInt(5);
 304             for (RowBase r : idx.rows) {
 305                 int type = 0;
 306                 if (r instanceof PairEntry.Row) {
 307                     type = 0;
 308                 } else if (r instanceof TokenRow) {
 309                     final TokenRow tokenRow = (TokenRow)r;
 310                     type = tokenRow.hasMainEntry ? 1 : 3;
 311                 } else if (r instanceof TextEntry.Row) {
 312                     type = 2;
 313                 } else if (r instanceof HtmlEntry.Row) {
 314                     type = 4;
 315                 } else {
 316                     throw new RuntimeException("Row type not supported for v6");
 317                 }
 318                 out.writeByte(type);
 319                 out.writeInt(r.referenceIndex);
 320             }
 321         }
 322         long dataPos = out.getFilePointer();
 323         out.seek(tocPos);
 324         out.writeLong(dataPos);
 325         out.seek(dataPos);
 326     }
 327
 328     public void writev6(DataOutput out) throws IOException {
 329         RandomAccessFile raf = (RandomAccessFile)out;
 330         raf.writeInt(6);
 331         raf.writeLong(creationMillis);
 332         raf.writeUTF(dictInfo);
 333         System.out.println("sources start: " + raf.getFilePointer());
 334         writev6Sources(raf);
 335         System.out.println("pair start: " + raf.getFilePointer());
 336         writev6PairEntries(raf);
 337         System.out.println("text start: " + raf.getFilePointer());
 338         writev6TextEntries(raf);
 339         System.out.println("html index start: " + raf.getFilePointer());
 340         writev6HtmlEntries(raf);
 341         System.out.println("indices start: " + raf.getFilePointer());
 342         writev6Index(raf);
 343         System.out.println("end: " + raf.getFilePointer());
 344         raf.writeUTF(END_OF_DICTIONARY);
 345     }
 346
 347     private final class IndexSerializer implements RAFListSerializer<Index> {
 348         private final FileChannel ch;
 349
 350         IndexSerializer(FileChannel ch) {
 351             this.ch = ch;
 352         }
 353
 354         @Override
 355         public Index read(DataInput raf, final int readIndex) throws IOException {
 356             return new Index(Dictionary.this, ch, raf);
 357         }
 358
 359         @Override
 360         public void write(DataOutput raf, Index t) throws IOException {
 361             t.write(raf);
 362         }
 363     }
 364
 365     final RAFListSerializer<HtmlEntry> htmlEntryIndexSerializer = new RAFListSerializer<HtmlEntry>() {
 366         @Override
 367         public void write(DataOutput raf, HtmlEntry t) {
 368             assert false;
 369         }
 370
 371         @Override
 372         public HtmlEntry read(DataInput raf, int readIndex) throws IOException {
 373             return htmlEntries.get(raf.readInt());
 374         }
 375     };
 376
 377     public void print(final PrintStream out) {
 378         out.println("dictInfo=" + dictInfo);
 379         for (final EntrySource entrySource : sources) {
 380             out.printf("EntrySource: %s %d\n", entrySource.name, entrySource.numEntries);
 381         }
 382         out.println();
 383         for (final Index index : indices) {
 384             out.printf("Index: %s %s\n", index.shortName, index.longName);
 385             index.print(out);
 386             out.println();
 387         }
 388     }
 389
 390     public DictionaryInfo getDictionaryInfo() {
 391         final DictionaryInfo result = new DictionaryInfo();
 392         result.creationMillis = this.creationMillis;
 393         result.dictInfo = this.dictInfo;
 394         for (final Index index : indices) {
 395             result.indexInfos.add(index.getIndexInfo());
 396         }
 397         return result;
 398     }
 399
 400     public static DictionaryInfo getDictionaryInfo(final File file) {
 401         RandomAccessFile raf = null;
 402         try {
 403             raf = new RandomAccessFile(file, "r");
 404             final Dictionary dict = new Dictionary(raf.getChannel());
 405             final DictionaryInfo dictionaryInfo = dict.getDictionaryInfo();
 406             dictionaryInfo.uncompressedFilename = file.getName();
 407             dictionaryInfo.uncompressedBytes = file.length();
 408             raf.close();
 409             return dictionaryInfo;
 410         } catch (IOException e) {
 411             final DictionaryInfo dictionaryInfo = new DictionaryInfo();
 412             dictionaryInfo.uncompressedFilename = file.getName();
 413             dictionaryInfo.uncompressedBytes = file.length();
 414             return dictionaryInfo;
 415         } finally {
 416             if (raf != null) {
 417                 try {
 418                     raf.close();
 419                 } catch (IOException e) {
 420                     e.printStackTrace();
 421                 }
 422             }
 423         }
 424     }
 425
 426 }