2 *******************************************************************************
\r
3 * Copyright (C) 2006-2010, International Business Machines Corporation and *
\r
4 * others. All Rights Reserved. *
\r
5 *******************************************************************************
\r
8 package com.ibm.icu.charset;
\r
10 import java.io.DataInputStream;
\r
11 import java.io.IOException;
\r
12 import java.io.InputStream;
\r
13 import java.nio.ByteBuffer;
\r
15 import com.ibm.icu.impl.ICUBinary;
\r
20 * ICU conversion (.cnv) data file structure, following the usual UDataInfo
\r
23 * Format version: 6.2
\r
25 * struct UConverterStaticData -- struct containing the converter name, IBM CCSID,
\r
26 * min/max bytes per character, etc.
\r
29 * --------------------
\r
31 * The static data is followed by conversionType-specific data structures.
\r
32 * At the moment, there are only variations of MBCS converters. They all have
\r
33 * the same toUnicode structures, while the fromUnicode structures for SBCS
\r
34 * differ from those for other MBCS-style converters.
\r
36 * _MBCSHeader.version 4.2 adds an optional conversion extension data structure.
\r
37 * If it is present, then an ICU version reading header versions 4.0 or 4.1
\r
38 * will be able to use the base table and ignore the extension.
\r
40 * The unicodeMask in the static data is part of the base table data structure.
\r
41 * Especially, the UCNV_HAS_SUPPLEMENTARY flag determines the length of the
\r
42 * fromUnicode stage 1 array.
\r
43 * The static data unicodeMask refers only to the base table's properties if
\r
44 * a base table is included.
\r
45 * In an extension-only file, the static data unicodeMask is 0.
\r
46 * The extension data indexes have a separate field with the unicodeMask flags.
\r
48 * MBCS-style data structure following the static data.
\r
49 * Offsets are counted in bytes from the beginning of the MBCS header structure.
\r
50 * Details about usage in comments in ucnvmbcs.c.
\r
52 * struct _MBCSHeader (see the definition in this header file below)
\r
53 * contains 32-bit fields as follows:
\r
55 * 0 uint8_t[4] MBCS version in UVersionInfo format (currently 4.2.0.0)
\r
56 * 1 uint32_t countStates
\r
57 * 2 uint32_t countToUFallbacks
\r
58 * 3 uint32_t offsetToUCodeUnits
\r
59 * 4 uint32_t offsetFromUTable
\r
60 * 5 uint32_t offsetFromUBytes
\r
61 * 6 uint32_t flags, bits:
\r
62 * 31.. 8 offsetExtension -- _MBCSHeader.version 4.2 (ICU 2.8) and higher
\r
63 * 0 for older versions and if
\r
64 * there is not extension structure
\r
66 * 7 uint32_t fromUBytesLength -- _MBCSHeader.version 4.1 (ICU 2.4) and higher
\r
67 * counts bytes in fromUBytes[]
\r
69 * if(outputType==MBCS_OUTPUT_EXT_ONLY) {
\r
70 * -- base table name for extension-only table
\r
71 * char baseTableName[variable]; -- with NUL plus padding for 4-alignment
\r
73 * -- all _MBCSHeader fields except for version and flags are 0
\r
75 * -- normal base table with optional extension
\r
77 * int32_t stateTable[countStates][256];
\r
79 * struct _MBCSToUFallback { (fallbacks are sorted by offset)
\r
81 * UChar32 codePoint;
\r
82 * } toUFallbacks[countToUFallbacks];
\r
84 * uint16_t unicodeCodeUnits[(offsetFromUTable-offsetToUCodeUnits)/2];
\r
85 * (padded to an even number of units)
\r
88 * if(staticData.unicodeMask&UCNV_HAS_SUPPLEMENTARY) {
\r
89 * -- stage 1 table for all of Unicode
\r
90 * uint16_t fromUTable[0x440]; (32-bit-aligned)
\r
92 * -- BMP-only tables have a smaller stage 1 table
\r
93 * uint16_t fromUTable[0x40]; (32-bit-aligned)
\r
97 * length determined by top of stage 1 and bottom of stage 3 tables
\r
98 * if(outputType==MBCS_OUTPUT_1) {
\r
99 * -- SBCS: pure indexes
\r
100 * uint16_t stage 2 indexes[?];
\r
102 * -- DBCS, MBCS, EBCDIC_STATEFUL, ...: roundtrip flags and indexes
\r
103 * uint32_t stage 2 flags and indexes[?];
\r
106 * -- stage 3 tables with byte results
\r
107 * if(outputType==MBCS_OUTPUT_1) {
\r
108 * -- SBCS: each 16-bit result contains flags and the result byte, see ucnvmbcs.c
\r
109 * uint16_t fromUBytes[fromUBytesLength/2];
\r
111 * -- DBCS, MBCS, EBCDIC_STATEFUL, ... 2/3/4 bytes result, see ucnvmbcs.c
\r
112 * uint8_t fromUBytes[fromUBytesLength]; or
\r
113 * uint16_t fromUBytes[fromUBytesLength/2]; or
\r
114 * uint32_t fromUBytes[fromUBytesLength/4];
\r
118 * -- extension table, details see ucnv_ext.h
\r
119 * int32_t indexes[>=32]; ...
\r
124 * See icuhtml/design/conversion/conversion_extensions.html
\r
126 * Conversion extensions serve two purposes:
\r
127 * 1. They support m:n mappings.
\r
128 * 2. They support extension-only conversion files that are used together
\r
129 * with the regular conversion data in base files.
\r
131 * A base file may contain an extension table (explicitly requested or
\r
132 * implicitly generated for m:n mappings), but its extension table is not
\r
133 * used when an extension-only file is used.
\r
135 * It is an error if a base file contains any regular (not extension) mapping
\r
136 * from the same sequence as a mapping in the extension file
\r
137 * because the base mapping would hide the extension mapping.
\r
140 * Data for conversion extensions:
\r
142 * One set of data structures per conversion direction (to/from Unicode).
\r
143 * The data structures are sorted by input units to allow for binary search.
\r
144 * Input sequences of more than one unit are handled like contraction tables
\r
146 * The lookup value of a unit points to another table that is to be searched
\r
147 * for the next unit, recursively.
\r
149 * For conversion from Unicode, the initial code point is looked up in
\r
150 * a 3-stage trie for speed,
\r
151 * with an additional table of unique results to save space.
\r
153 * Long output strings are stored in separate arrays, with length and index
\r
154 * in the lookup tables.
\r
155 * Output results also include a flag distinguishing roundtrip from
\r
156 * (reverse) fallback mappings.
\r
158 * Input Unicode strings must not begin or end with unpaired surrogates
\r
159 * to avoid problems with matches on parts of surrogate pairs.
\r
161 * Mappings from multiple characters (code points or codepage state
\r
162 * table sequences) must be searched preferring the longest match.
\r
163 * For this to work and be efficient, the variable-width table must contain
\r
164 * all mappings that contain prefixes of the multiple characters.
\r
165 * If an extension table is built on top of a base table in another file
\r
166 * and a base table entry is a prefix of a multi-character mapping, then
\r
167 * this is an error.
\r
170 * Implementation note:
\r
172 * Currently, the parser and several checks in the code limit the number
\r
173 * of UChars or bytes in a mapping to
\r
174 * UCNV_EXT_MAX_UCHARS and UCNV_EXT_MAX_BYTES, respectively,
\r
175 * which are output value limits in the data structure.
\r
177 * For input, this is not strictly necessary - it is a hard limit only for the
\r
178 * buffers in UConverter that are used to store partial matches.
\r
180 * Input sequences could otherwise be arbitrarily long if partial matches
\r
181 * need not be stored (i.e., if a sequence does not span several buffers with too
\r
182 * many units before the last buffer), although then results would differ
\r
183 * depending on whether partial matches exceed the limits or not,
\r
184 * which depends on the pattern of buffer sizes.
\r
189 * int32_t indexes[>=32];
\r
191 * Array of indexes and lengths etc. The length of the array is at least 32.
\r
192 * The actual length is stored in indexes[0] to be forward compatible.
\r
194 * Each index to another array is the number of bytes from indexes[].
\r
195 * Each length of an array is the number of array base units in that array.
\r
197 * Some of the structures may not be present, in which case their indexes
\r
198 * and lengths are 0.
\r
200 * Usage of indexes[i]:
\r
201 * [0] length of indexes[]
\r
203 * // to Unicode table
\r
204 * [1] index of toUTable[] (array of uint32_t)
\r
205 * [2] length of toUTable[]
\r
206 * [3] index of toUUChars[] (array of UChar)
\r
207 * [4] length of toUUChars[]
\r
209 * // from Unicode table, not for the initial code point
\r
210 * [5] index of fromUTableUChars[] (array of UChar)
\r
211 * [6] index of fromUTableValues[] (array of uint32_t)
\r
212 * [7] length of fromUTableUChars[] and fromUTableValues[]
\r
213 * [8] index of fromUBytes[] (array of char)
\r
214 * [9] length of fromUBytes[]
\r
216 * // from Unicode trie for initial-code point lookup
\r
217 * [10] index of fromUStage12[] (combined array of uint16_t for stages 1 & 2)
\r
218 * [11] length of stage 1 portion of fromUStage12[]
\r
219 * [12] length of fromUStage12[]
\r
220 * [13] index of fromUStage3[] (array of uint16_t indexes into fromUStage3b[])
\r
221 * [14] length of fromUStage3[]
\r
222 * [15] index of fromUStage3b[] (array of uint32_t like fromUTableValues[])
\r
223 * [16] length of fromUStage3b[]
\r
225 * [17] Bit field containing numbers of bytes:
\r
226 * 31..24 reserved, 0
\r
227 * 23..16 maximum input bytes
\r
228 * 15.. 8 maximum output bytes
\r
229 * 7.. 0 maximum bytes per UChar
\r
231 * [18] Bit field containing numbers of UChars:
\r
232 * 31..24 reserved, 0
\r
233 * 23..16 maximum input UChars
\r
234 * 15.. 8 maximum output UChars
\r
235 * 7.. 0 maximum UChars per byte
\r
237 * [19] Bit field containing flags:
\r
238 * (extension table unicodeMask)
\r
239 * 1 UCNV_HAS_SURROGATES flag for the extension table
\r
240 * 0 UCNV_HAS_SUPPLEMENTARY flag for the extension table
\r
242 * [20]..[30] reserved, 0
\r
243 * [31] number of bytes for the entire extension structure
\r
244 * [>31] reserved; there are indexes[0] indexes
\r
247 * uint32_t toUTable[];
\r
249 * Array of byte/value pairs for lookups for toUnicode conversion.
\r
250 * The array is partitioned into sections like collation contraction tables.
\r
251 * Each section contains one word with the number of following words and
\r
252 * a default value for when the lookup in this section yields no match.
\r
254 * A section is sorted in ascending order of input bytes,
\r
255 * allowing for fast linear or binary searches.
\r
256 * The builder may store entries for a contiguous range of byte values
\r
257 * (compare difference between the first and last one with count),
\r
258 * which then allows for direct array access.
\r
259 * The builder should always do this for the initial table section.
\r
261 * Entries may have 0 values, see below.
\r
262 * No two entries in a section have the same byte values.
\r
264 * Each uint32_t contains an input byte value in bits 31..24 and the
\r
265 * corresponding lookup value in bits 23..0.
\r
266 * Interpret the value as follows:
\r
268 * no match, see below
\r
269 * } else if(value<0x1f0000) {
\r
270 * partial match - use value as index to the next toUTable section
\r
271 * and match the next unit; (value indexes toUTable[value])
\r
278 * unset value bit 23;
\r
279 * if(value<=0x2fffff) {
\r
280 * (value-0x1f0000) is a code point; (BMP: value<=0x1fffff)
\r
282 * bits 17..0 (value&0x3ffff) is an index to
\r
283 * the result UChars in toUUChars[]; (0 indexes toUUChars[0])
\r
284 * length of the result=((value>>18)-12); (length=0..19)
\r
288 * The first word in a section contains the number of following words in the
\r
289 * input byte position (bits 31..24, number=1..0xff).
\r
290 * The value of the initial word is used when the current byte is not found
\r
292 * If the value is not 0, then it represents a result as above.
\r
293 * If the value is 0, then the search has to return a shorter match with an
\r
294 * earlier default value as the result, or result in "unmappable" even for the
\r
296 * If the value is 0 for the initial toUTable entry, then the initial byte
\r
297 * does not start any mapping input.
\r
300 * UChar toUUChars[];
\r
302 * Contains toUnicode mapping results, stored as sequences of UChars.
\r
303 * Indexes and lengths stored in the toUTable[].
\r
306 * UChar fromUTableUChars[];
\r
307 * uint32_t fromUTableValues[];
\r
309 * The fromUTable is split into two arrays, but works otherwise much like
\r
310 * the toUTable. The array is partitioned into sections like collation
\r
311 * contraction tables and toUTable.
\r
312 * A row in the table consists of same-index entries in fromUTableUChars[]
\r
313 * and fromUTableValues[].
\r
315 * Interpret a value as follows:
\r
317 * no match, see below
\r
318 * } else if(value<=0xffffff) { (bits 31..24 are 0)
\r
319 * partial match - use value as index to the next fromUTable section
\r
320 * and match the next unit; (value indexes fromUTable[value])
\r
322 * if(value==0x80000001) {
\r
323 * return no mapping, but request for <subchar1>;
\r
330 * // bits 30..29 reserved, 0
\r
331 * length=(value>>24)&0x1f; (bits 28..24)
\r
332 * if(length==1..3) {
\r
333 * bits 23..0 contain 1..3 bytes, padded with 00s on the left;
\r
335 * bits 23..0 (value&0xffffff) is an index to
\r
336 * the result bytes in fromUBytes[]; (0 indexes fromUBytes[0])
\r
340 * The first pair in a section contains the number of following pairs in the
\r
341 * UChar position (16 bits, number=1..0xffff).
\r
342 * The value of the initial pair is used when the current UChar is not found
\r
344 * If the value is not 0, then it represents a result as above.
\r
345 * If the value is 0, then the search has to return a shorter match with an
\r
346 * earlier default value as the result, or result in "unmappable" even for the
\r
349 * If the from Unicode trie is present, then the from Unicode search tables
\r
350 * are not used for initial code points.
\r
351 * In this case, the first entries (index 0) in the tables are not used
\r
352 * (reserved, set to 0) because a value of 0 is used in trie results
\r
353 * to indicate no mapping.
\r
356 * uint16_t fromUStage12[];
\r
358 * Stages 1 & 2 of a trie that maps an initial code point.
\r
359 * Indexes in stage 1 are all offset by the length of stage 1 so that the
\r
360 * same array pointer can be used for both stages.
\r
361 * If (c>>10)>=(length of stage 1) then c does not start any mapping.
\r
362 * Same bit distribution as for regular conversion tries.
\r
365 * uint16_t fromUStage3[];
\r
366 * uint32_t fromUStage3b[];
\r
368 * Stage 3 of the trie. The first array simply contains indexes to the second,
\r
369 * which contains words in the same format as fromUTableValues[].
\r
370 * Use a stage 3 granularity of 4, which allows for 256k stage 3 entries,
\r
371 * and 16-bit entries in stage 3 allow for 64k stage 3b entries.
\r
372 * The stage 3 granularity means that the stage 2 entry needs to be left-shifted.
\r
374 * Two arrays are used because it is expected that more than half of the stage 3
\r
375 * entries will be zero. The 16-bit index stage 3 array saves space even
\r
376 * considering storing a total of 6 bytes per non-zero entry in both arrays
\r
378 * Using a stage 3 granularity of >1 diminishes the compactability in that stage
\r
379 * but provides a larger effective addressing space in stage 2.
\r
380 * All but the final result stage use 16-bit entries to save space.
\r
382 * fromUStage3b[] contains a zero for "no mapping" at its index 0,
\r
383 * and may contain UCNV_EXT_FROM_U_SUBCHAR1 at index 1 for "<subchar1> SUB mapping"
\r
384 * (i.e., "no mapping" with preference for <subchar1> rather than <subchar>),
\r
385 * and all other items are unique non-zero results.
\r
387 * The default value of a fromUTableValues[] section that is referenced
\r
388 * _directly_ from a fromUStage3b[] item may also be UCNV_EXT_FROM_U_SUBCHAR1,
\r
389 * but this value must not occur anywhere else in fromUTableValues[]
\r
390 * because "no mapping" is always a property of a single code point,
\r
391 * never of multiple.
\r
394 * char fromUBytes[];
\r
396 * Contains fromUnicode mapping results, stored as sequences of chars.
\r
397 * Indexes and lengths stored in the fromUTableValues[].
\r
400 final class UConverterDataReader implements ICUBinary.Authenticate {
\r
401 //private final static boolean debug = ICUDebug.enabled("UConverterDataReader");
\r
404 * UConverterDataReader(UConverterDataReader r)
\r
406 dataInputStream = new DataInputStream(r.dataInputStream);
\r
407 unicodeVersion = r.unicodeVersion;
\r
410 /* the number bytes read from the stream */
\r
412 /* the number of bytes read for static data */
\r
413 int staticDataBytesRead = 0;
\r
415 * <p>Protected constructor.</p>
\r
416 * @param inputStream ICU uprop.dat file input stream
\r
417 * @exception IOException throw if data file fails authentication
\r
419 protected UConverterDataReader(InputStream inputStream)
\r
420 throws IOException{
\r
421 //if(debug) System.out.println("Bytes in inputStream " + inputStream.available());
\r
423 /*unicodeVersion = */ICUBinary.readHeader(inputStream, DATA_FORMAT_ID, this);
\r
425 //if(debug) System.out.println("Bytes left in inputStream " +inputStream.available());
\r
427 dataInputStream = new DataInputStream(inputStream);
\r
429 //if(debug) System.out.println("Bytes left in dataInputStream " +dataInputStream.available());
\r
432 // protected methods -------------------------------------------------
\r
434 protected void readStaticData(UConverterStaticData sd) throws IOException
\r
437 sd.structSize = dataInputStream.readInt();
\r
439 byte[] name = new byte[UConverterConstants.MAX_CONVERTER_NAME_LENGTH];
\r
440 dataInputStream.readFully(name);
\r
441 bRead +=name.length;
\r
442 sd.name = new String(name, 0, name.length);
\r
443 sd.codepage = dataInputStream.readInt();
\r
445 sd.platform = dataInputStream.readByte();
\r
447 sd.conversionType = dataInputStream.readByte();
\r
449 sd.minBytesPerChar = dataInputStream.readByte();
\r
451 sd.maxBytesPerChar = dataInputStream.readByte();
\r
453 dataInputStream.readFully(sd.subChar);
\r
454 bRead += sd.subChar.length;
\r
455 sd.subCharLen = dataInputStream.readByte();
\r
457 sd.hasToUnicodeFallback = dataInputStream.readByte();
\r
459 sd.hasFromUnicodeFallback = dataInputStream.readByte();
\r
461 sd.unicodeMask = (short)dataInputStream.readUnsignedByte();
\r
463 sd.subChar1 = dataInputStream.readByte();
\r
465 dataInputStream.readFully(sd.reserved);
\r
466 bRead += sd.reserved.length;
\r
467 staticDataBytesRead = bRead;
\r
468 bytesRead += bRead;
\r
471 protected void readMBCSHeader(CharsetMBCS.MBCSHeader h) throws IOException
\r
473 dataInputStream.readFully(h.version);
\r
474 bytesRead += h.version.length;
\r
475 h.countStates = dataInputStream.readInt();
\r
477 h.countToUFallbacks = dataInputStream.readInt();
\r
479 h.offsetToUCodeUnits = dataInputStream.readInt();
\r
481 h.offsetFromUTable = dataInputStream.readInt();
\r
483 h.offsetFromUBytes = dataInputStream.readInt();
\r
485 h.flags = dataInputStream.readInt();
\r
487 h.fromUBytesLength = dataInputStream.readInt();
\r
489 if (h.version[0] == 5 && h.version[1] >= 3) {
\r
490 h.options = dataInputStream.readInt();
\r
492 if ((h.options & CharsetMBCS.MBCS_OPT_NO_FROM_U) != 0) {
\r
493 h.fullStage2Length = dataInputStream.readInt();
\r
499 protected void readMBCSTable(int[][] stateTableArray, CharsetMBCS.MBCSToUFallback[] toUFallbacksArray, char[] unicodeCodeUnitsArray, char[] fromUnicodeTableArray, byte[] fromUnicodeBytesArray) throws IOException
\r
502 for(i = 0; i < stateTableArray.length; ++i){
\r
503 for(j = 0; j < stateTableArray[i].length; ++j){
\r
504 stateTableArray[i][j] = dataInputStream.readInt();
\r
508 for(i = 0; i < toUFallbacksArray.length; ++i) {
\r
509 toUFallbacksArray[i].offset = dataInputStream.readInt();
\r
511 toUFallbacksArray[i].codePoint = dataInputStream.readInt();
\r
514 for(i = 0; i < unicodeCodeUnitsArray.length; ++i){
\r
515 unicodeCodeUnitsArray[i] = dataInputStream.readChar();
\r
518 for(i = 0; i < fromUnicodeTableArray.length; ++i){
\r
519 fromUnicodeTableArray[i] = dataInputStream.readChar();
\r
522 for(i = 0; i < fromUnicodeBytesArray.length; ++i){
\r
523 fromUnicodeBytesArray[i] = dataInputStream.readByte();
\r
528 protected String readBaseTableName() throws IOException
\r
531 StringBuilder name = new StringBuilder();
\r
532 while((c = (char)dataInputStream.readByte()) != 0){
\r
536 bytesRead++/*for null terminator*/;
\r
537 return name.toString();
\r
540 //protected int[] readExtIndexes(int skip) throws IOException
\r
541 protected ByteBuffer readExtIndexes(int skip) throws IOException
\r
543 int skipped = dataInputStream.skipBytes(skip);
\r
544 if(skipped != skip){
\r
545 throw new IOException("could not skip "+ skip +" bytes");
\r
547 int n = dataInputStream.readInt();
\r
549 int[] indexes = new int[n];
\r
551 for(int i = 1; i < n; ++i) {
\r
552 indexes[i] = dataInputStream.readInt();
\r
557 ByteBuffer b = ByteBuffer.allocate(indexes[31]);
\r
558 for(int i = 0; i < n; ++i) {
\r
559 b.putInt(indexes[i]);
\r
561 int len = dataInputStream.read(b.array(), b.position(), b.remaining());
\r
563 throw new IOException("Read failed");
\r
569 /*protected byte[] readExtTables(int n) throws IOException
\r
571 byte[] tables = new byte[n];
\r
572 int len =dataInputStream.read(tables);
\r
574 throw new IOException("Read failed");
\r
580 byte[] getDataFormatVersion(){
\r
581 return DATA_FORMAT_VERSION;
\r
586 public boolean isDataVersionAcceptable(byte version[]){
\r
587 return version[0] == DATA_FORMAT_VERSION[0];
\r
590 /* byte[] getUnicodeVersion(){
\r
591 return unicodeVersion;
\r
593 // private data members -------------------------------------------------
\r
596 * ICU data file input stream
\r
598 DataInputStream dataInputStream;
\r
600 // private byte[] unicodeVersion;
\r
603 * File format version that this class understands.
\r
604 * No guarantees are made if a older version is used
\r
605 * see store.c of gennorm for more information and values
\r
607 // DATA_FORMAT_ID_ values taken from icu4c isCnvAcceptable (ucnv_bld.c)
\r
608 private static final byte DATA_FORMAT_ID[] = {(byte)0x63, (byte)0x6e, (byte)0x76, (byte)0x74}; // dataFormat="cnvt"
\r
609 private static final byte DATA_FORMAT_VERSION[] = {(byte)0x6};
\r
projects / Dictionary.git / blob