2 *******************************************************************************
\r
3 * Copyright (C) 2008, International Business Machines Corporation and *
\r
4 * others. All Rights Reserved. *
\r
5 *******************************************************************************
\r
7 package com.ibm.icu.charset;
\r
9 import java.nio.ByteBuffer;
\r
10 import java.nio.CharBuffer;
\r
11 import java.nio.IntBuffer;
\r
12 import java.nio.charset.CharsetDecoder;
\r
13 import java.nio.charset.CharsetEncoder;
\r
14 import java.nio.charset.CoderResult;
\r
16 import com.ibm.icu.charset.CharsetMBCS.CharsetDecoderMBCS;
\r
17 import com.ibm.icu.charset.CharsetMBCS.CharsetEncoderMBCS;
\r
18 import com.ibm.icu.util.ULocale;
\r
19 import com.ibm.icu.text.UnicodeSet;
\r
21 * @author Michael Ow
\r
28 * (Lotus Multi-Byte Character Set)
\r
30 * LMBS was invented in the alte 1980's and is primarily used in Lotus Notes
\r
31 * databases and in Lotus 1-2-3 files. Programmers who work with the APIs
\r
32 * into these products will sometimes need to deal with strings in this format.
\r
34 * The code in this file provides an implementation for an ICU converter of
\r
35 * LMBCS to and from Unicode.
\r
37 * Since the LMBCS character set is only sparsely documented in existing
\r
38 * printed or online material, we have added extensive annotation to this
\r
39 * file to serve as a guide to understanding LMBCS.
\r
41 * LMBCS was originally designed with these four sometimes-competing design goals:
\r
42 * -Provide encodings for characters in 12 existing national standards
\r
43 * (plus a few other characters)
\r
44 * -Minimal memory footprint
\r
45 * -Maximal speed of conversion into the existing national character sets
\r
46 * -No need to track a changing state as you interpret a string.
\r
48 * All of the national character sets LMBCS was trying to encode are 'ANSI'
\r
49 * based, in that the bytes from 0x20 - 0x7F are almost exactly the
\r
50 * same common Latin unaccented characters and symbols in all character sets.
\r
52 * So, in order to help meet the speed & memory design goals, the common ANSI
\r
53 * bytes from 0x20-0x7F are represented by the same single-byte values in LMBCS.
\r
55 class CharsetLMBCS extends CharsetICU {
\r
57 * The general LMBCS code unit is from 1-3 bytes. We can describe the 3 bytes as
\r
60 * That is, a sometimes-optional 'group' byte, followed by 1 and sometimes 2
\r
61 * data bytes. The maximum size of a LMBCS character is 3 bytes:
\r
63 private static final short ULMBCS_CHARSIZE_MAX = 3;
\r
65 * The single-byte values from 0x20 to 0x7F are examples of single D1 bytes.
\r
66 * We often have to figure out if byte values are below or above this, so we
\r
67 * use the ANSI nomenclature 'C0' and 'C1' to refer to the range of control
\r
68 * characters just above & below the common lower-ANSI range.
\r
70 private static final short ULMBCS_C0END = 0x1F;
\r
71 private static final short ULMBCS_C1START = 0x80;
\r
73 * Most of the values less than 0x20 are reserved in LMBCS to announce
\r
74 * which national character standard is being used for the 'D' bytes.
\r
75 * In the comments we show that common name and the IBM character-set ID
\r
76 * for these character-set announcers:
\r
78 private static final short ULMBCS_GRP_L1 = 0x01; /* Latin-1 :ibm-850 */
\r
79 private static final short ULMBCS_GRP_GR = 0x02; /* Greek :ibm-851 */
\r
80 private static final short ULMBCS_GRP_HE = 0x03; /* Hebrew :ibm-1255 */
\r
81 private static final short ULMBCS_GRP_AR = 0x04; /* Arabic :ibm-1256 */
\r
82 private static final short ULMBCS_GRP_RU = 0x05; /* Cyrillic :ibm-1251 */
\r
83 private static final short ULMBCS_GRP_L2 = 0x06; /* Latin-2 :ibm-852 */
\r
84 private static final short ULMBCS_GRP_TR = 0x08; /* Turkish :ibm-1254 */
\r
85 private static final short ULMBCS_GRP_TH = 0x0B; /* Thai :ibm-874 */
\r
86 private static final short ULMBCS_GRP_JA = 0x10; /* Japanese :ibm-943 */
\r
87 private static final short ULMBCS_GRP_KO = 0x11; /* Korean :ibm-1261 */
\r
88 private static final short ULMBCS_GRP_TW = 0x12; /* Chinese SC :ibm-950 */
\r
89 private static final short ULMBCS_GRP_CN = 0x13; /* Chinese TC :ibm-1386 */
\r
91 * So, the beginnning of understanding LMBCS is that IF the first byte of a LMBCS
\r
92 * character is one of those 12 values, you can interpret the remaining bytes of
\r
93 * that character as coming from one of those character sets. Since the lower
\r
94 * ANSI bytes already are represented in singl bytes, using one of the chracter
\r
95 * set announcers is used to announce a character that starts with a byte of
\r
98 * The character sets are arranged so that the single byte sets all appear
\r
99 * before the multi-byte character sets. When we need to tell whether a
\r
100 * group byte is for a single byte char set or not we use this definition:
\r
102 private static final short ULMBCS_DOUBLEOPTGROUP_START = 0x10;
\r
104 * However, to fully understand LMBCS, you must also understand a series of
\r
105 * exceptions & optimizations made in service of the design goals.
\r
107 * First, those of you who are character set mavens may have noticed that
\r
108 * the 'double-byte' character sets are actually multi-byte chracter sets
\r
109 * that can have 1 or two bytes, even in upper-ascii range. To force
\r
110 * each group byte to introduce a fixed-width encoding (to make it faster to
\r
111 * count characters), we use a convention of doubling up on the group byte
\r
112 * to introduce any single-byte character > 0x80 in an otherwise double-byte
\r
113 * character set. So, for example, the LMBCS sequence x10 x10 xAE is the
\r
114 * same as '0xAE' in the Japanese code page 943.
\r
116 * Next, you will notice that the list of group bytes has some gaps.
\r
117 * These are used in various ways.
\r
119 * We reserve a few special single byte values for common control
\r
120 * characters. These are in the same place as their ANSI equivalents for speed.
\r
122 private static final short ULMBCS_HT = 0x09; /* Fixed control-char - Horizontal Tab */
\r
123 private static final short ULMBCS_LF = 0x0A; /* Fixed control-char - Line Feed */
\r
124 private static final short ULMBCS_CR = 0x0D; /* Fixed control-char - Carriage Return */
\r
126 * Then, 1-2-3 reserved a special single-byte character to put at the
\r
127 * beginning of internal 'system' range names:
\r
129 private static final short ULMBCS_123SYSTEMRANGE = 0x19;
\r
131 * Then we needed a place to put all the other ansi control characters
\r
132 * that must be moved to different values because LMBCS reserves those
\r
133 * values for other purposes. To represent the control characters, we start
\r
134 * with a first byte of 0x0F & add the control character value as the
\r
137 private static final short ULMBCS_GRP_CTRL = 0x0F;
\r
139 * For the C0 controls (less than 0x20), we add 0x20 to preserve the
\r
140 * useful doctrine that any byte less than 0x20 in a LMBCS char must be
\r
141 * the first byte of a character:
\r
143 private static final short ULMBCS_CTRLOFFSET = 0x20;
\r
145 * Where to put the characters that aren't part of any of the 12 national
\r
146 * character sets? The first thing that was done, in the earlier years of
\r
147 * LMBCS, was to use up the spaces of the form
\r
149 * where 'G' was one of the single-byte character groups, and
\r
150 * D1 was less than 0x80. These sequences are gathered together
\r
151 * into a Lotus-invented doublebyte character set to represent a
\r
152 * lot of stray values. Internally, in this implementation, we track this
\r
153 * as group '0', as a place to tuck this exceptions list.
\r
155 private static final short ULMBCS_GRP_EXCEPT = 0x00;
\r
157 * Finally, as the durability and usefulness of UNICODE became clear,
\r
158 * LOTUS added a new group 0x14 to hold Unicode values not otherwise
\r
159 * represented in LMBCS:
\r
161 private static final short ULMBCS_GRP_UNICODE = 0x14;
\r
163 * The two bytes appearing after a 0x14 are interpreted as UTF-16 BE
\r
164 * (Big Endian) characters. The exception comes when UTF16
\r
165 * representation would have a zero as the second byte. In that case,
\r
166 * 'F6' is used in its place, and the bytes are swapped. (This prevents
\r
167 * LMBCS from encoding any Unicode values of the form U+F6xx, but that's OK:
\r
168 * 0xF6xx is in the middle of the Private Use Area.)
\r
170 private static char ULMBCS_UNICOMPATZERO = 0x00F6;
\r
172 * It is also useful in our code to have a constant for the size of
\r
173 * a LMBCS char that holds a literal Unicode value.
\r
175 private static final short ULMBCS_UNICODE_SIZE = 3;
\r
177 * To squish the LMBCS representation down even further, and to make
\r
178 * translations even faster, sometimes the optimization group byte can be dropped
\r
179 * from a LMBCS character. This is decided on a process-by-process basis. The
\r
180 * group byte that is dropped is called the 'optimization group.'
\r
182 * For Notes, the optimization group is always 0x1.
\r
184 //private static final short ULMBCS_DEFAULTOPTGROUP = 0x01;
\r
185 /* For 1-2-3 files, the optimization group is stored in the header of the 1-2-3
\r
187 * In any case, when using ICU, you either pass in the
\r
188 * optimization group as part of the name of the converter (LMBCS-1, LMBCS-2,
\r
189 * etc.). Using plain 'LMBCS' as the name of the converter will give you
\r
193 /* Implementation strategy */
\r
195 * Because of the extensive use of other character sets, the LMBCS converter
\r
196 * keeps a mapping between optimization groups and IBM character sets, so that
\r
197 * ICU converters can be created and used as needed.
\r
199 * As you can see, even though any byte below 0x20 could be an optimization
\r
200 * byte, only those at 0x13 or below can map to an actual converter. To limit
\r
201 * some loops and searches, we define a value for that last group converter:
\r
203 private static final short ULMBCS_GRP_LAST = 0x13; /* last LMBCS group that has a converter */
\r
205 private static final String[] OptGroupByteToCPName = {
\r
206 /* 0x0000 */ "lmb-excp", /* internal home for the LOTUS exceptions list */
\r
207 /* 0x0001 */ "ibm-850",
\r
208 /* 0x0002 */ "ibm-851",
\r
209 /* 0x0003 */ "windows-1255",
\r
210 /* 0x0004 */ "windows-1256",
\r
211 /* 0x0005 */ "windows-1251",
\r
212 /* 0x0006 */ "ibm-852",
\r
213 /* 0x0007 */ null, /* Unused */
\r
214 /* 0x0008 */ "windows-1254",
\r
215 /* 0x0009 */ null, /* Control char HT */
\r
216 /* 0x000A */ null, /* Control char LF */
\r
217 /* 0x000B */ "windows-874",
\r
218 /* 0x000C */ null, /* Unused */
\r
219 /* 0x000D */ null, /* Control char CR */
\r
220 /* 0x000E */ null, /* Unused */
\r
221 /* 0x000F */ null, /* Control chars: 0x0F20 + C0/C1 character: algorithmic */
\r
222 /* 0x0010 */ "windows-932",
\r
223 /* 0x0011 */ "windows-949",
\r
224 /* 0x0012 */ "windows-950",
\r
225 /* 0x0013 */ "windows-936",
\r
226 /* The rest are null, including the 0x0014 Unicode compatibility region
\r
227 * and 0x0019, the 1-2-3 system range control char */
\r
231 /* That's approximately all the data that's needed for translating
\r
232 * LMBCS to Unicode.
\r
234 * However, to translate Unicode to LMBCS, we need some more support.
\r
236 * That's because there are often more than one possible mappings from a Unicode
\r
237 * code point back into LMBCS. The first thing we do is look up into a table
\r
238 * to figure out if there are more than one possible mapplings. This table,
\r
239 * arranged by Unicode values (including ranges) either lists which group
\r
240 * to use, or says that it could go into one or more of the SBCS sets, or
\r
241 * into one or more of the DBCS sets. (If the character exists in both DBCS &
\r
242 * SBCS, the table will place it in the SBCS sets, to make the LMBCS code point
\r
243 * length as small as possible. Here's the two special markers we use to indicate
\r
244 * ambiguous mappings:
\r
246 private static final short ULMBCS_AMBIGUOUS_SBCS = 0x80; /* could fit in more than one
\r
247 LMBCS sbcs native encoding
\r
248 (example: most accented latin) */
\r
249 private static final short ULMBCS_AMBIGUOUS_MBCS = 0x81; /* could fit in more than one
\r
250 LMBCS mbcs native encoding
\r
251 (example: Unihan) */
\r
253 /* And here's a simple way to see if a group falls in an appropriate range */
\r
254 private boolean ULMBCS_AMBIGUOUS_MATCH(short agroup, short xgroup) {
\r
255 return (((agroup == ULMBCS_AMBIGUOUS_SBCS) &&
\r
256 (xgroup < ULMBCS_DOUBLEOPTGROUP_START)) ||
\r
257 ((agroup == ULMBCS_AMBIGUOUS_MBCS) &&
\r
258 (xgroup >= ULMBCS_DOUBLEOPTGROUP_START)));
\r
261 /* The table & some code to use it: */
\r
262 private static class _UniLMBCSGrpMap {
\r
266 _UniLMBCSGrpMap(int uniStartRange, int uniEndRange, short GrpType) {
\r
267 this.uniStartRange = uniStartRange;
\r
268 this.uniEndRange = uniEndRange;
\r
269 this.GrpType = GrpType;
\r
273 private static final _UniLMBCSGrpMap[] UniLMBCSGrpMap = {
\r
274 new _UniLMBCSGrpMap(0x0001, 0x001F, ULMBCS_GRP_CTRL),
\r
275 new _UniLMBCSGrpMap(0x0080, 0x009F, ULMBCS_GRP_CTRL),
\r
276 new _UniLMBCSGrpMap(0x00A0, 0x01CD, ULMBCS_AMBIGUOUS_SBCS),
\r
277 new _UniLMBCSGrpMap(0x01CE, 0x01CE, ULMBCS_GRP_TW),
\r
278 new _UniLMBCSGrpMap(0x01CF, 0x02B9, ULMBCS_AMBIGUOUS_SBCS),
\r
279 new _UniLMBCSGrpMap(0x02BA, 0x02BA, ULMBCS_GRP_CN),
\r
280 new _UniLMBCSGrpMap(0x02BC, 0x02C8, ULMBCS_AMBIGUOUS_SBCS),
\r
281 new _UniLMBCSGrpMap(0x02C9, 0x02D0, ULMBCS_AMBIGUOUS_MBCS),
\r
282 new _UniLMBCSGrpMap(0x02D8, 0x02DD, ULMBCS_AMBIGUOUS_SBCS),
\r
283 new _UniLMBCSGrpMap(0x0384, 0x03CE, ULMBCS_AMBIGUOUS_SBCS),
\r
284 new _UniLMBCSGrpMap(0x0400, 0x044E, ULMBCS_GRP_RU),
\r
285 new _UniLMBCSGrpMap(0x044F, 0x044F, ULMBCS_AMBIGUOUS_MBCS),
\r
286 new _UniLMBCSGrpMap(0x0450, 0x0491, ULMBCS_GRP_RU),
\r
287 new _UniLMBCSGrpMap(0x05B0, 0x05F2, ULMBCS_GRP_HE),
\r
288 new _UniLMBCSGrpMap(0x060C, 0x06AF, ULMBCS_GRP_AR),
\r
289 new _UniLMBCSGrpMap(0x0E01, 0x0E5B, ULMBCS_GRP_TH),
\r
290 new _UniLMBCSGrpMap(0x200C, 0x200F, ULMBCS_AMBIGUOUS_SBCS),
\r
291 new _UniLMBCSGrpMap(0x2010, 0x2010, ULMBCS_AMBIGUOUS_MBCS),
\r
292 new _UniLMBCSGrpMap(0x2013, 0x2015, ULMBCS_AMBIGUOUS_SBCS),
\r
293 new _UniLMBCSGrpMap(0x2016, 0x2016, ULMBCS_AMBIGUOUS_MBCS),
\r
294 new _UniLMBCSGrpMap(0x2017, 0x2024, ULMBCS_AMBIGUOUS_SBCS),
\r
295 new _UniLMBCSGrpMap(0x2025, 0x2025, ULMBCS_AMBIGUOUS_MBCS),
\r
296 new _UniLMBCSGrpMap(0x2026, 0x2026, ULMBCS_AMBIGUOUS_SBCS),
\r
297 new _UniLMBCSGrpMap(0x2027, 0x2027, ULMBCS_GRP_CN),
\r
298 new _UniLMBCSGrpMap(0x2030, 0x2033, ULMBCS_AMBIGUOUS_SBCS),
\r
299 new _UniLMBCSGrpMap(0x2035, 0x2035, ULMBCS_AMBIGUOUS_MBCS),
\r
300 new _UniLMBCSGrpMap(0x2039, 0x203A, ULMBCS_AMBIGUOUS_SBCS),
\r
301 new _UniLMBCSGrpMap(0x203B, 0x203B, ULMBCS_AMBIGUOUS_MBCS),
\r
302 new _UniLMBCSGrpMap(0x2074, 0x2074, ULMBCS_GRP_KO),
\r
303 new _UniLMBCSGrpMap(0x207F, 0x207F, ULMBCS_GRP_EXCEPT),
\r
304 new _UniLMBCSGrpMap(0x2081, 0x2084, ULMBCS_GRP_KO),
\r
305 new _UniLMBCSGrpMap(0x20A4, 0x20AC, ULMBCS_AMBIGUOUS_SBCS),
\r
306 new _UniLMBCSGrpMap(0x2103, 0x2109, ULMBCS_AMBIGUOUS_MBCS),
\r
307 new _UniLMBCSGrpMap(0x2111, 0x2126, ULMBCS_AMBIGUOUS_SBCS),
\r
308 new _UniLMBCSGrpMap(0x212B, 0x212B, ULMBCS_AMBIGUOUS_MBCS),
\r
309 new _UniLMBCSGrpMap(0x2135, 0x2135, ULMBCS_AMBIGUOUS_SBCS),
\r
310 new _UniLMBCSGrpMap(0x2153, 0x2154, ULMBCS_GRP_KO),
\r
311 new _UniLMBCSGrpMap(0x215B, 0x215E, ULMBCS_GRP_EXCEPT),
\r
312 new _UniLMBCSGrpMap(0x2160, 0x2179, ULMBCS_AMBIGUOUS_MBCS),
\r
313 new _UniLMBCSGrpMap(0x2190, 0x2195, ULMBCS_GRP_EXCEPT),
\r
314 new _UniLMBCSGrpMap(0x2196, 0x2199, ULMBCS_AMBIGUOUS_MBCS),
\r
315 new _UniLMBCSGrpMap(0x21A8, 0x21A8, ULMBCS_GRP_EXCEPT),
\r
316 new _UniLMBCSGrpMap(0x21B8, 0x21B9, ULMBCS_GRP_CN),
\r
317 new _UniLMBCSGrpMap(0x21D0, 0x21D5, ULMBCS_GRP_EXCEPT),
\r
318 new _UniLMBCSGrpMap(0x21E7, 0x21E7, ULMBCS_GRP_CN),
\r
319 new _UniLMBCSGrpMap(0x2200, 0x220B, ULMBCS_GRP_EXCEPT),
\r
320 new _UniLMBCSGrpMap(0x220F, 0x2215, ULMBCS_AMBIGUOUS_MBCS),
\r
321 new _UniLMBCSGrpMap(0x2219, 0x2220, ULMBCS_GRP_EXCEPT),
\r
322 new _UniLMBCSGrpMap(0x2223, 0x2228, ULMBCS_AMBIGUOUS_MBCS),
\r
323 new _UniLMBCSGrpMap(0x2229, 0x222B, ULMBCS_GRP_EXCEPT),
\r
324 new _UniLMBCSGrpMap(0x222C, 0x223D, ULMBCS_AMBIGUOUS_MBCS),
\r
325 new _UniLMBCSGrpMap(0x2245, 0x2248, ULMBCS_GRP_EXCEPT),
\r
326 new _UniLMBCSGrpMap(0x224C, 0x224C, ULMBCS_GRP_TW),
\r
327 new _UniLMBCSGrpMap(0x2252, 0x2252, ULMBCS_AMBIGUOUS_MBCS),
\r
328 new _UniLMBCSGrpMap(0x2260, 0x2265, ULMBCS_GRP_EXCEPT),
\r
329 new _UniLMBCSGrpMap(0x2266, 0x226F, ULMBCS_AMBIGUOUS_MBCS),
\r
330 new _UniLMBCSGrpMap(0x2282, 0x2297, ULMBCS_GRP_EXCEPT),
\r
331 new _UniLMBCSGrpMap(0x2299, 0x22BF, ULMBCS_AMBIGUOUS_MBCS),
\r
332 new _UniLMBCSGrpMap(0x22C0, 0x22C0, ULMBCS_GRP_EXCEPT),
\r
333 new _UniLMBCSGrpMap(0x2310, 0x2310, ULMBCS_GRP_EXCEPT),
\r
334 new _UniLMBCSGrpMap(0x2312, 0x2312, ULMBCS_AMBIGUOUS_MBCS),
\r
335 new _UniLMBCSGrpMap(0x2318, 0x2321, ULMBCS_GRP_EXCEPT),
\r
336 new _UniLMBCSGrpMap(0x2318, 0x2321, ULMBCS_GRP_CN),
\r
337 new _UniLMBCSGrpMap(0x2460, 0x24E9, ULMBCS_AMBIGUOUS_MBCS),
\r
338 new _UniLMBCSGrpMap(0x2500, 0x2500, ULMBCS_AMBIGUOUS_SBCS),
\r
339 new _UniLMBCSGrpMap(0x2501, 0x2501, ULMBCS_AMBIGUOUS_MBCS),
\r
340 new _UniLMBCSGrpMap(0x2502, 0x2502, ULMBCS_AMBIGUOUS_SBCS),
\r
341 new _UniLMBCSGrpMap(0x2503, 0x2503, ULMBCS_AMBIGUOUS_MBCS),
\r
342 new _UniLMBCSGrpMap(0x2504, 0x2505, ULMBCS_GRP_TW),
\r
343 new _UniLMBCSGrpMap(0x2506, 0x2665, ULMBCS_AMBIGUOUS_MBCS),
\r
344 new _UniLMBCSGrpMap(0x2666, 0x2666, ULMBCS_GRP_EXCEPT),
\r
345 new _UniLMBCSGrpMap(0x2666, 0x2666, ULMBCS_GRP_EXCEPT),
\r
346 new _UniLMBCSGrpMap(0x2667, 0x2E7F, ULMBCS_AMBIGUOUS_SBCS),
\r
347 new _UniLMBCSGrpMap(0x2E80, 0xF861, ULMBCS_AMBIGUOUS_MBCS),
\r
348 new _UniLMBCSGrpMap(0xF862, 0xF8FF, ULMBCS_GRP_EXCEPT),
\r
349 new _UniLMBCSGrpMap(0xF900, 0xFA2D, ULMBCS_AMBIGUOUS_MBCS),
\r
350 new _UniLMBCSGrpMap(0xFB00, 0xFEFF, ULMBCS_AMBIGUOUS_SBCS),
\r
351 new _UniLMBCSGrpMap(0xFF01, 0xFFEE, ULMBCS_AMBIGUOUS_MBCS),
\r
352 new _UniLMBCSGrpMap(0xFFFF, 0xFFFF, ULMBCS_GRP_UNICODE)
\r
355 static short FindLMBCSUniRange(char uniChar) {
\r
358 while (uniChar > UniLMBCSGrpMap[index].uniEndRange) {
\r
362 if (uniChar >= UniLMBCSGrpMap[index].uniStartRange) {
\r
363 return UniLMBCSGrpMap[index].GrpType;
\r
365 return ULMBCS_GRP_UNICODE;
\r
369 * We also ask the creator of a converter to send in a preferred locale
\r
370 * that we can use in resolving ambiguous mappings. They send the locale
\r
371 * in as a string, and we map it, if possible, to one of the
\r
372 * LMBCS groups. We use this table, and the associated code, to
\r
375 * This table maps locale ID's to LMBCS opt groups.
\r
376 * The default return is group 0x01. Note that for
\r
377 * performance reasons, the table is sorted in
\r
378 * increasing alphabetic order, with the notable
\r
379 * exception of zhTW. This is to force the check
\r
380 * for Traditional Chinese before dropping back to
\r
382 * Note too that the Latin-1 groups have been
\r
383 * commented out because it's the default, and
\r
384 * this shortens the table, allowing a serial
\r
385 * search to go quickly.
\r
387 private static class _LocaleLMBCSGrpMap {
\r
390 _LocaleLMBCSGrpMap(String LocaleID, short OptGroup) {
\r
391 this.LocaleID = LocaleID;
\r
392 this.OptGroup = OptGroup;
\r
395 private static final _LocaleLMBCSGrpMap[] LocaleLMBCSGrpMap = {
\r
396 new _LocaleLMBCSGrpMap("ar", ULMBCS_GRP_AR),
\r
397 new _LocaleLMBCSGrpMap("be", ULMBCS_GRP_RU),
\r
398 new _LocaleLMBCSGrpMap("bg", ULMBCS_GRP_L2),
\r
399 // new _LocaleLMBCSGrpMap("ca", ULMBCS_GRP_L1),
\r
400 new _LocaleLMBCSGrpMap("cs", ULMBCS_GRP_L2),
\r
401 // new _LocaleLMBCSGrpMap("da", ULMBCS_GRP_L1),
\r
402 // new _LocaleLMBCSGrpMap("de", ULMBCS_GRP_L1),
\r
403 new _LocaleLMBCSGrpMap("el", ULMBCS_GRP_GR),
\r
404 // new _LocaleLMBCSGrpMap("en", ULMBCS_GRP_L1),
\r
405 // new _LocaleLMBCSGrpMap("es", ULMBCS_GRP_L1),
\r
406 // new _LocaleLMBCSGrpMap("et", ULMBCS_GRP_L1),
\r
407 // new _LocaleLMBCSGrpMap("fi", ULMBCS_GRP_L1),
\r
408 // new _LocaleLMBCSGrpMap("fr", ULMBCS_GRP_L1),
\r
409 new _LocaleLMBCSGrpMap("he", ULMBCS_GRP_HE),
\r
410 new _LocaleLMBCSGrpMap("hu", ULMBCS_GRP_L2),
\r
411 // new _LocaleLMBCSGrpMap("is", ULMBCS_GRP_L1),
\r
412 // new _LocaleLMBCSGrpMap("it", ULMBCS_GRP_L1),
\r
413 new _LocaleLMBCSGrpMap("iw", ULMBCS_GRP_HE),
\r
414 new _LocaleLMBCSGrpMap("ja", ULMBCS_GRP_JA),
\r
415 new _LocaleLMBCSGrpMap("ko", ULMBCS_GRP_KO),
\r
416 // new _LocaleLMBCSGrpMap("lt", ULMBCS_GRP_L1),
\r
417 // new _LocaleLMBCSGrpMap("lv", ULMBCS_GRP_L1),
\r
418 new _LocaleLMBCSGrpMap("mk", ULMBCS_GRP_RU),
\r
419 // new _LocaleLMBCSGrpMap("nl", ULMBCS_GRP_L1),
\r
420 // new _LocaleLMBCSGrpMap("no", ULMBCS_GRP_L1),
\r
421 new _LocaleLMBCSGrpMap("pl", ULMBCS_GRP_L2),
\r
422 // new _LocaleLMBCSGrpMap("pt", ULMBCS_GRP_L1),
\r
423 new _LocaleLMBCSGrpMap("ro", ULMBCS_GRP_L2),
\r
424 new _LocaleLMBCSGrpMap("ru", ULMBCS_GRP_RU),
\r
425 new _LocaleLMBCSGrpMap("sh", ULMBCS_GRP_L2),
\r
426 new _LocaleLMBCSGrpMap("sk", ULMBCS_GRP_L2),
\r
427 new _LocaleLMBCSGrpMap("sl", ULMBCS_GRP_L2),
\r
428 new _LocaleLMBCSGrpMap("sq", ULMBCS_GRP_L2),
\r
429 new _LocaleLMBCSGrpMap("sr", ULMBCS_GRP_RU),
\r
430 // new _LocaleLMBCSGrpMap("sv", ULMBCS_GRP_L1),
\r
431 new _LocaleLMBCSGrpMap("th", ULMBCS_GRP_TH),
\r
432 new _LocaleLMBCSGrpMap("tr", ULMBCS_GRP_TR),
\r
433 new _LocaleLMBCSGrpMap("uk", ULMBCS_GRP_RU),
\r
434 // new _LocaleLMBCSGrpMap("vi", ULMBCS_GRP_L1),
\r
435 new _LocaleLMBCSGrpMap("zhTW", ULMBCS_GRP_TW),
\r
436 new _LocaleLMBCSGrpMap("zh", ULMBCS_GRP_CN),
\r
437 new _LocaleLMBCSGrpMap(null, ULMBCS_GRP_L1)
\r
439 static short FindLMBCSLocale(String LocaleID) {
\r
442 if (LocaleID == null) {
\r
446 while (LocaleLMBCSGrpMap[index].LocaleID != null) {
\r
447 if (LocaleLMBCSGrpMap[index].LocaleID == LocaleID) {
\r
448 return LocaleLMBCSGrpMap[index].OptGroup;
\r
449 } else if (LocaleLMBCSGrpMap[index].LocaleID.compareTo(LocaleID) > 0){
\r
454 return ULMBCS_GRP_L1;
\r
458 * Before we get to the main body of code, here's how we hook up the rest
\r
459 * of ICU. ICU converters are required to define a structure that includes
\r
460 * some function pointers, and some common data, in the style of a C++
\r
461 * vtable. There is also room in there for converter-specific data. LMBCS
\r
462 * uses that converter-specific data to keep track of the 12 subconverters
\r
463 * we use, the optimization group, and the group (if any) that matches the
\r
464 * locale. We have one structure instantiated for each of the 12 possible
\r
465 * optimization groups.
\r
467 private class UConverterDataLMBCS {
\r
468 UConverterSharedData[] OptGrpConverter; /* Converter per Opt. grp. */
\r
469 short OptGroup; /* default Opt. grp. for this LMBCS session */
\r
470 short localeConverterIndex; /* reasonable locale match for index */
\r
471 CharsetDecoderMBCS decoder;
\r
472 CharsetEncoderMBCS encoder;
\r
473 CharsetMBCS charset;
\r
474 UConverterDataLMBCS() {
\r
475 OptGrpConverter = new UConverterSharedData[ULMBCS_GRP_LAST + 1];
\r
476 charset = (CharsetMBCS)CharsetICU.forNameICU("ibm-850");
\r
477 encoder = (CharsetEncoderMBCS)charset.newEncoder();
\r
478 decoder = (CharsetDecoderMBCS)charset.newDecoder();
\r
482 private UConverterDataLMBCS extraInfo; /* extraInfo in ICU4C implementation */
\r
484 public CharsetLMBCS(String icuCanonicalName, String javaCanonicalName, String[] aliases) {
\r
485 super(icuCanonicalName, javaCanonicalName, aliases);
\r
486 maxBytesPerChar = ULMBCS_CHARSIZE_MAX;
\r
487 minBytesPerChar = 1;
\r
488 maxCharsPerByte = 1;
\r
490 extraInfo = new UConverterDataLMBCS();
\r
492 for (int i = 0; i <= ULMBCS_GRP_LAST; i++) {
\r
493 if (OptGroupByteToCPName[i] != null) {
\r
494 extraInfo.OptGrpConverter[i] = ((CharsetMBCS)CharsetICU.forNameICU(OptGroupByteToCPName[i])).sharedData;
\r
498 //get the Opt Group number for the LMBCS converter
\r
499 int option = Integer.parseInt(icuCanonicalName.substring(6));
\r
500 extraInfo.OptGroup = (short)option;
\r
501 extraInfo.localeConverterIndex = FindLMBCSLocale(ULocale.getDefault().getBaseName());
\r
504 class CharsetDecoderLMBCS extends CharsetDecoderICU {
\r
505 public CharsetDecoderLMBCS(CharsetICU cs) {
\r
510 protected void implReset() {
\r
514 /* A function to call when we are looking at the Unicode group byte in LMBCS */
\r
515 private char GetUniFromLMBCSUni(ByteBuffer ppLMBCSin) {
\r
516 short HighCh = (short)(ppLMBCSin.get() & UConverterConstants.UNSIGNED_BYTE_MASK);
\r
517 short LowCh = (short)(ppLMBCSin.get() & UConverterConstants.UNSIGNED_BYTE_MASK);
\r
519 if (HighCh == ULMBCS_UNICOMPATZERO) {
\r
521 LowCh = 0; /* zero-byte in LSB special character */
\r
524 return (char)((HighCh << 8) | LowCh);
\r
527 private int LMBCS_SimpleGetNextUChar(UConverterSharedData cnv, ByteBuffer source, int positionOffset, int length) {
\r
529 int oldSourceLimit;
\r
532 extraInfo.charset.sharedData = cnv;
\r
534 oldSourceLimit = source.limit();
\r
535 oldSourcePos = source.position();
\r
537 source.position(oldSourcePos + positionOffset);
\r
538 source.limit(source.position() + length);
\r
540 uniChar = extraInfo.decoder.simpleGetNextUChar(source, false);
\r
542 source.limit(oldSourceLimit);
\r
543 source.position(oldSourcePos);
\r
547 /* Return the Unicode representation for the current LMBCS character. */
\r
549 * Note: Because there is no U_TRUNCATED_CHAR_FOUND error code in ICU4J, we
\r
550 * are going to use BufferOverFlow. The error will be handled correctly
\r
551 * by the calling function.
\r
553 private int LMBCSGetNextUCharWorker(ByteBuffer source, CoderResult[] err) {
\r
554 int uniChar = 0; /* an output Unicode char */
\r
555 short CurByte; /* A byte from the input stream */
\r
558 if (!source.hasRemaining()) {
\r
559 err[0] = CoderResult.malformedForLength(0);
\r
562 /* Grab first byte & save address for error recovery */
\r
563 CurByte = (short)(source.get() & UConverterConstants.UNSIGNED_BYTE_MASK);
\r
566 * at entry of each if clause:
\r
567 * 1. 'CurByte' points at the first byte of a LMBCS character
\r
568 * 2. 'source' points to the next byte of the source stream after 'CurByte'
\r
570 * the job of each if clause is:
\r
571 * 1. set 'source' to the point at the beginning of the next char (not if LMBCS char is only 1 byte)
\r
572 * 2. set 'uniChar' up with the right Unicode value, or set 'err' appropriately
\r
574 /* First lets check the simple fixed values. */
\r
575 if ((CurByte > ULMBCS_C0END && CurByte < ULMBCS_C1START) /* ascii range */ ||
\r
576 CurByte == 0 || CurByte == ULMBCS_HT || CurByte == ULMBCS_CR || CurByte == ULMBCS_LF ||
\r
577 CurByte == ULMBCS_123SYSTEMRANGE) {
\r
582 UConverterSharedData cnv;
\r
584 if (CurByte == ULMBCS_GRP_CTRL) { /* Control character group - no opt group update */
\r
586 /* CHECK_SOURCE_LIMIT(1) */
\r
587 if (source.position() + 1 > source.limit()) {
\r
588 err[0] = CoderResult.OVERFLOW;
\r
589 source.position(source.limit());
\r
592 C0C1byte = (short)(source.get() & UConverterConstants.UNSIGNED_BYTE_MASK);
\r
593 uniChar = (C0C1byte < ULMBCS_C1START) ? C0C1byte - ULMBCS_CTRLOFFSET : C0C1byte;
\r
594 } else if (CurByte == ULMBCS_GRP_UNICODE) { /* Unicode Compatibility group: Big Endian UTF16 */
\r
595 /* CHECK_SOURCE_LIMIT(2) */
\r
596 if (source.position() + 2 > source.limit()) {
\r
597 err[0] = CoderResult.OVERFLOW;
\r
598 source.position(source.limit());
\r
602 /* don't check for error indicators fffe/ffff below */
\r
603 return GetUniFromLMBCSUni(source);
\r
604 } else if (CurByte <= ULMBCS_CTRLOFFSET) {
\r
606 if (group > ULMBCS_GRP_LAST || (cnv = extraInfo.OptGrpConverter[group]) == null) {
\r
607 /* this is not a valid group byte - no converter */
\r
608 err[0] = CoderResult.unmappableForLength(1);
\r
609 } else if (group >= ULMBCS_DOUBLEOPTGROUP_START) {
\r
610 /* CHECK_SOURCE_LIMIT(2) */
\r
611 if (source.position() + 2 > source.limit()) {
\r
612 err[0] = CoderResult.OVERFLOW;
\r
613 source.position(source.limit());
\r
617 /* check for LMBCS doubled-group-byte case */
\r
618 if (source.get(source.position()) == group) {
\r
621 uniChar = LMBCS_SimpleGetNextUChar(cnv, source, 0, 1);
\r
625 uniChar = LMBCS_SimpleGetNextUChar(cnv, source, 0, 2);
\r
629 } else { /* single byte conversion */
\r
630 /* CHECK_SOURCE_LIMIT(1) */
\r
631 if (source.position() + 1 > source.limit()) {
\r
632 err[0] = CoderResult.OVERFLOW;
\r
633 source.position(source.limit());
\r
636 CurByte = (short)(source.get() & UConverterConstants.UNSIGNED_BYTE_MASK);
\r
638 if (CurByte >= ULMBCS_C1START) {
\r
639 uniChar = CharsetMBCS.MBCS_SINGLE_SIMPLE_GET_NEXT_BMP(cnv.mbcs, CurByte);
\r
642 * The non-optimizable oddballs where there is an explicit byte
\r
643 * AND the second byte is not in the upper ascii range
\r
645 byte[] bytes = new byte[2];
\r
647 cnv = extraInfo.OptGrpConverter[ULMBCS_GRP_EXCEPT];
\r
649 /* Lookup value must include opt group */
\r
650 bytes[0] = (byte)group;
\r
651 bytes[1] = (byte)CurByte;
\r
652 uniChar = LMBCS_SimpleGetNextUChar(cnv, ByteBuffer.wrap(bytes), 0, 2);
\r
656 } else if (CurByte >= ULMBCS_C1START) { /* group byte is implicit */
\r
657 group = extraInfo.OptGroup;
\r
658 cnv = extraInfo.OptGrpConverter[group];
\r
659 if (group >= ULMBCS_DOUBLEOPTGROUP_START) { /* double byte conversion */
\r
660 if (CharsetMBCS.MBCS_ENTRY_IS_TRANSITION(cnv.mbcs.stateTable[0][CurByte]) /* isLeadByte */) {
\r
661 /* CHECK_SOURCE_LIMIT(0) */
\r
662 if (source.position() + 0 > source.limit()) {
\r
663 err[0] = CoderResult.OVERFLOW;
\r
664 source.position(source.limit());
\r
668 /* let the MBCS conversion consume CurByte again */
\r
669 uniChar = LMBCS_SimpleGetNextUChar(cnv, source, -1, 1);
\r
671 /* CHECK_SOURCE_LIMIT(1) */
\r
672 if (source.position() + 1 > source.limit()) {
\r
673 err[0] = CoderResult.OVERFLOW;
\r
674 source.position(source.limit());
\r
678 /* let the MBCS conversion consume CurByte again */
\r
679 uniChar = LMBCS_SimpleGetNextUChar(cnv, source, -1, 2);
\r
683 uniChar = CharsetMBCS.MBCS_SINGLE_SIMPLE_GET_NEXT_BMP(cnv.mbcs, CurByte);
\r
691 protected CoderResult decodeLoop(ByteBuffer source, CharBuffer target, IntBuffer offsets, boolean flush) {
\r
692 CoderResult[] err = new CoderResult[1];
\r
693 err[0] = CoderResult.UNDERFLOW;
\r
694 byte[] LMBCS = new byte[ULMBCS_CHARSIZE_MAX * 2]; /* Increase the size for proper handling in subsequent calls to MBCS functions */
\r
695 char uniChar; /* one output Unicode char */
\r
696 int saveSource; /* beginning of current code point */
\r
697 int errSource = 0; /* index to actual input in case an error occurs */
\r
698 byte savebytes = 0;
\r
700 /* Process from source to limit, or until error */
\r
701 while (err[0].isUnderflow() && source.hasRemaining() && target.hasRemaining()) {
\r
702 saveSource = source.position(); /* beginning of current code point */
\r
703 if (toULength > 0) { /* reassemble char from previous call */
\r
704 int size_old = toULength;
\r
705 ByteBuffer tmpSourceBuffer;
\r
707 /* limit from source is either remainder of temp buffer, or user limit on source */
\r
708 int size_new_maybe_1 = ULMBCS_CHARSIZE_MAX - size_old;
\r
709 int size_new_maybe_2 = source.remaining();
\r
710 int size_new = (size_new_maybe_1 < size_new_maybe_2) ? size_new_maybe_1 : size_new_maybe_2;
\r
711 savebytes = (byte)(size_old + size_new);
\r
712 for (int i = 0; i < savebytes; i++) {
\r
713 if (i < size_old) {
\r
714 LMBCS[i] = toUBytesArray[i];
\r
716 LMBCS[i] = source.get();
\r
719 tmpSourceBuffer = ByteBuffer.wrap(LMBCS);
\r
720 tmpSourceBuffer.limit(savebytes);
\r
721 uniChar = (char)LMBCSGetNextUCharWorker(tmpSourceBuffer, err);
\r
722 source.position(saveSource + tmpSourceBuffer.position() - size_old);
\r
723 errSource = saveSource - size_old;
\r
725 if (err[0].isOverflow()) { /* err == U_TRUNCATED_CHAR_FOUND */
\r
726 /* evil special case: source buffers so small a char spans more than 2 buffers */
\r
727 toULength = savebytes;
\r
728 for (int i = 0; i < savebytes; i++) {
\r
729 toUBytesArray[i] = LMBCS[i];
\r
731 source.position(source.limit());
\r
732 err[0] = CoderResult.UNDERFLOW;
\r
735 /* clear the partial-char marker */
\r
739 errSource = saveSource;
\r
740 uniChar = (char)LMBCSGetNextUCharWorker(source, err);
\r
741 savebytes = (byte)(source.position() - saveSource);
\r
744 if (err[0].isUnderflow()) {
\r
745 if (uniChar < 0x0fffe) {
\r
746 target.put(uniChar);
\r
747 if (offsets != null) {
\r
748 offsets.put(saveSource);
\r
750 } else if (uniChar == 0xfffe) {
\r
751 err[0] = CoderResult.unmappableForLength(source.position() - saveSource);
\r
752 } else /* if (uniChar == 0xffff) */ {
\r
753 err[0] = CoderResult.malformedForLength(source.position() - saveSource);
\r
757 /* If target ran out before source, return over flow buffer error. */
\r
758 if (err[0].isUnderflow() && source.hasRemaining() && !target.hasRemaining()) {
\r
759 err[0] = CoderResult.OVERFLOW;
\r
760 } else if (!err[0].isUnderflow()) {
\r
761 /* If character incomplete or unmappable/illegal, store it in toUBytesArray[] */
\r
762 toULength = savebytes;
\r
763 if (savebytes > 0) {
\r
764 for (int i = 0; i < savebytes; i++) {
\r
765 toUBytesArray[i] = source.get(errSource + i);
\r
768 if (err[0].isOverflow()) { /* err == U_TRUNCATED_CHAR_FOUND */
\r
769 err[0] = CoderResult.UNDERFLOW;
\r
776 class CharsetEncoderLMBCS extends CharsetEncoderICU {
\r
777 public CharsetEncoderLMBCS(CharsetICU cs) {
\r
778 super(cs, fromUSubstitution);
\r
782 protected void implReset() {
\r
786 * Here's the basic helper function that we use when converting from
\r
787 * Unicode to LMBCS, and we suspect that a Unicode character will fit into
\r
788 * one of the 12 groups. The return value is the number of bytes written
\r
789 * starting at pStartLMBCS (if any).
\r
791 private int LMBCSConversionWorker(short group, byte[] LMBCS, char pUniChar, short[] lastConverterIndex, boolean[] groups_tried) {
\r
793 UConverterSharedData xcnv = extraInfo.OptGrpConverter[group];
\r
795 int bytesConverted;
\r
796 int[] value = new int[1];
\r
799 extraInfo.charset.sharedData = xcnv;
\r
800 bytesConverted = extraInfo.encoder.fromUChar32(pUniChar, value, false);
\r
802 /* get the first result byte */
\r
803 if (bytesConverted > 0) {
\r
804 firstByte = (short)((value[0] >> ((bytesConverted - 1) * 8)) & UConverterConstants.UNSIGNED_BYTE_MASK);
\r
806 /* most common failure mode is an unassigned character */
\r
807 groups_tried[group] = true;
\r
811 lastConverterIndex[0] = group;
\r
814 * All initial byte values in lower ascii range should have been caught by now,
\r
815 * except with the exception group.
\r
818 /* use converted data: first write 0, 1 or two group bytes */
\r
819 if (group != ULMBCS_GRP_EXCEPT && extraInfo.OptGroup != group) {
\r
820 LMBCS[pLMBCS++] = (byte)group;
\r
821 if (bytesConverted == 1 && group >= ULMBCS_DOUBLEOPTGROUP_START) {
\r
822 LMBCS[pLMBCS++] = (byte)group;
\r
826 /* don't emit control chars */
\r
827 if (bytesConverted == 1 && firstByte < 0x20) {
\r
831 /* then move over the converted data */
\r
832 switch (bytesConverted) {
\r
834 LMBCS[pLMBCS++] = (byte)(value[0] >> 24);
\r
836 LMBCS[pLMBCS++] = (byte)(value[0] >> 16);
\r
838 LMBCS[pLMBCS++] = (byte)(value[0] >> 8);
\r
840 LMBCS[pLMBCS++] = (byte)value[0];
\r
842 /* will never occur */
\r
849 * This is a much simpler version of above, when we
\r
850 * know we are writing LMBCS using the Unicode group.
\r
852 private int LMBCSConvertUni(byte[] LMBCS, char uniChar) {
\r
854 short LowCh = (short)(uniChar & UConverterConstants.UNSIGNED_BYTE_MASK);
\r
855 short HighCh = (short)((uniChar >> 8) & UConverterConstants.UNSIGNED_BYTE_MASK);
\r
857 LMBCS[index++] = (byte)ULMBCS_GRP_UNICODE;
\r
860 LMBCS[index++] = (byte)ULMBCS_UNICOMPATZERO;
\r
861 LMBCS[index++] = (byte)HighCh;
\r
863 LMBCS[index++] = (byte)HighCh;
\r
864 LMBCS[index++] = (byte)LowCh;
\r
866 return ULMBCS_UNICODE_SIZE;
\r
868 /* The main Unicode to LMBCS conversion function */
\r
869 protected CoderResult encodeLoop(CharBuffer source, ByteBuffer target, IntBuffer offsets, boolean flush) {
\r
870 CoderResult err = CoderResult.UNDERFLOW;
\r
871 short[] lastConverterIndex = new short[1];
\r
873 byte[] LMBCS = new byte[ULMBCS_CHARSIZE_MAX];
\r
876 boolean[] groups_tried = new boolean[ULMBCS_GRP_LAST+1];
\r
877 int sourceIndex = 0;
\r
880 * Basic strategy: attempt to fill in local LMBCS 1-char buffer.(LMBCS)
\r
881 * If that succeeds, see if it will all fit into the target & copy it over
\r
884 * We try conversions in the following order:
\r
885 * 1. Single-byte ascii & special fixed control chars (&null)
\r
886 * 2. Look up group in table & try that (could b
\r
889 * C) national encodeing
\r
890 * or ambiguous SBCS or MBCS group (on to step 4...)
\r
891 * 3. If its ambiguous, try this order:
\r
892 * A) The optimization group
\r
893 * B) The locale group
\r
894 * C) The last group that succeeded with this string.
\r
895 * D) every other group that's relevant
\r
896 * E) If its single-byte ambiguous, try the exceptions group
\r
897 * 4. And as a grand fallback: Unicode
\r
899 while (source.hasRemaining() && err.isUnderflow()) {
\r
900 if (!target.hasRemaining()) {
\r
901 err = CoderResult.OVERFLOW;
\r
904 uniChar = source.get(source.position());
\r
908 /* check cases in rough order of how common they are, for speed */
\r
910 /* single-byte matches: strategy 1 */
\r
911 if (((uniChar > ULMBCS_C0END) && (uniChar < ULMBCS_C1START)) ||
\r
912 uniChar == 0 || uniChar == ULMBCS_HT || uniChar == ULMBCS_CR ||
\r
913 uniChar == ULMBCS_LF || uniChar == ULMBCS_123SYSTEMRANGE) {
\r
914 LMBCS[pLMBCS++] = (byte)uniChar;
\r
918 if (bytes_written == 0) {
\r
919 /* Check by Unicode rage (Strategy 2) */
\r
920 short group = FindLMBCSUniRange(uniChar);
\r
921 if (group == ULMBCS_GRP_UNICODE) { /* (Strategy 2A) */
\r
922 bytes_written = LMBCSConvertUni(LMBCS, uniChar);
\r
923 } else if (group == ULMBCS_GRP_CTRL) { /* Strategy 2B) */
\r
924 /* Handle control characters here */
\r
925 if (uniChar <= ULMBCS_C0END) {
\r
926 LMBCS[pLMBCS++] = ULMBCS_GRP_CTRL;
\r
927 LMBCS[pLMBCS++] = (byte)(ULMBCS_CTRLOFFSET + uniChar);
\r
928 } else if (uniChar >= ULMBCS_C1START && uniChar <= (ULMBCS_C1START + ULMBCS_CTRLOFFSET)) {
\r
929 LMBCS[pLMBCS++] = ULMBCS_GRP_CTRL;
\r
930 LMBCS[pLMBCS++] = (byte)uniChar;
\r
932 bytes_written = pLMBCS;
\r
933 } else if (group < ULMBCS_GRP_UNICODE) { /* (Strategy 2C) */
\r
934 /* a specific converter has been identified - use it */
\r
935 bytes_written = LMBCSConversionWorker(group, LMBCS, uniChar, lastConverterIndex, groups_tried);
\r
937 if (bytes_written == 0) { /* the ambiguous group cases (Strategy 3) */
\r
938 groups_tried = new boolean[ULMBCS_GRP_LAST+1];
\r
940 /* check for non-default optimization group (Strategy 3A) */
\r
941 if (extraInfo.OptGroup != 1 && ULMBCS_AMBIGUOUS_MATCH(group, extraInfo.OptGroup)) {
\r
942 bytes_written = LMBCSConversionWorker(extraInfo.OptGroup, LMBCS, uniChar, lastConverterIndex, groups_tried);
\r
944 /* check for locale optimization group (Strategy 3B) */
\r
945 if (bytes_written == 0 && extraInfo.localeConverterIndex > 0 &&
\r
946 ULMBCS_AMBIGUOUS_MATCH(group, extraInfo.localeConverterIndex)) {
\r
948 bytes_written = LMBCSConversionWorker(extraInfo.localeConverterIndex, LMBCS, uniChar, lastConverterIndex, groups_tried);
\r
950 /* check for last optimization group used for this string (Strategy 3C) */
\r
951 if (bytes_written == 0 && lastConverterIndex[0] > 0 &&
\r
952 ULMBCS_AMBIGUOUS_MATCH(group, lastConverterIndex[0])) {
\r
954 bytes_written = LMBCSConversionWorker(lastConverterIndex[0], LMBCS, uniChar, lastConverterIndex, groups_tried);
\r
956 if (bytes_written == 0) {
\r
957 /* just check every possible matching converter (Strategy 3D) */
\r
962 grp_start = (group == ULMBCS_AMBIGUOUS_MBCS) ? ULMBCS_DOUBLEOPTGROUP_START : ULMBCS_GRP_L1;
\r
963 grp_end = (group == ULMBCS_AMBIGUOUS_MBCS) ? ULMBCS_GRP_LAST : ULMBCS_GRP_TH;
\r
964 for (grp_ix = grp_start; grp_ix <= grp_end && bytes_written == 0; grp_ix++) {
\r
965 if (extraInfo.OptGrpConverter[grp_ix] != null && !groups_tried[grp_ix]) {
\r
966 bytes_written = LMBCSConversionWorker(grp_ix, LMBCS, uniChar, lastConverterIndex, groups_tried);
\r
970 * a final conversion fallback to the exceptions group if its likely
\r
971 * to be single byte (Strategy 3E)
\r
973 if (bytes_written == 0 && grp_start == ULMBCS_GRP_L1) {
\r
974 bytes_written = LMBCSConversionWorker(ULMBCS_GRP_EXCEPT, LMBCS, uniChar, lastConverterIndex, groups_tried);
\r
977 /* all of our other strategies failed. Fallback to Unicode. (Strategy 4) */
\r
978 if (bytes_written == 0) {
\r
979 bytes_written = LMBCSConvertUni(LMBCS, uniChar);
\r
983 /* we have a translation. increment source and write as much as possible to target */
\r
986 while (target.hasRemaining() && bytes_written > 0) {
\r
988 target.put(LMBCS[pLMBCS++]);
\r
989 if (offsets != null) {
\r
990 offsets.put(sourceIndex);
\r
994 if (bytes_written > 0) {
\r
996 * write any bytes that didn't fit in target to the error buffer,
\r
997 * common code will move this to target if we get called back with
\r
998 * enough target room
\r
1000 err = CoderResult.OVERFLOW;
\r
1001 errorBufferLength = bytes_written;
\r
1002 for (int i = 0; bytes_written > 0; i++, bytes_written--) {
\r
1003 errorBuffer[i] = LMBCS[pLMBCS++];
\r
1011 public CharsetDecoder newDecoder() {
\r
1012 return new CharsetDecoderLMBCS(this);
\r
1015 public CharsetEncoder newEncoder() {
\r
1016 return new CharsetEncoderLMBCS(this);
\r
1019 void getUnicodeSetImpl(UnicodeSet setFillIn, int which){
\r
1020 getCompleteUnicodeSet(setFillIn);
\r
1022 private byte[] fromUSubstitution = new byte[]{ 0x3F };
\r
projects / Dictionary.git / blob