2 *******************************************************************************
\r
3 * Copyright (C) 2003-2006,
\r
4 * International Business Machines Corporation and others. All Rights Reserved.
\r
5 *******************************************************************************
\r
8 package com.ibm.icu.text;
\r
9 import java.util.List;
\r
10 import java.util.ArrayList;
\r
11 import java.util.Iterator;
\r
12 import java.io.OutputStream;
\r
13 import java.io.IOException;
\r
15 import com.ibm.icu.impl.Assert;
\r
16 import com.ibm.icu.impl.IntTrieBuilder;
\r
19 // RBBISetBuilder Handles processing of Unicode Sets from RBBI rules
\r
20 // (part of the rule building process.)
\r
22 // Starting with the rules parse tree from the scanner,
\r
24 // - Enumerate the set of UnicodeSets that are referenced
\r
25 // by the RBBI rules.
\r
26 // - compute a set of non-overlapping character ranges
\r
27 // with all characters within a range belonging to the same
\r
28 // set of input uniocde sets.
\r
29 // - Derive a set of non-overlapping UnicodeSet (like things)
\r
30 // that will correspond to columns in the state table for
\r
31 // the RBBI execution engine. All characters within one
\r
32 // of these sets belong to the same set of the original
\r
33 // UnicodeSets from the user's rules.
\r
34 // - construct the trie table that maps input characters
\r
35 // to the index of the matching non-overlapping set of set from
\r
36 // the previous step.
\r
38 class RBBISetBuilder {
\r
39 static class RangeDescriptor {
\r
40 int fStartChar; // Start of range, unicode 32 bit value.
\r
41 int fEndChar; // End of range, unicode 32 bit value.
\r
42 int fNum; // runtime-mapped input value for this range.
\r
43 List fIncludesSets; // vector of the the original
\r
44 // Unicode sets that include this range.
\r
45 // (Contains ptrs to uset nodes)
\r
46 RangeDescriptor fNext; // Next RangeDescriptor in the linked list.
\r
49 fIncludesSets = new ArrayList();
\r
52 RangeDescriptor(RangeDescriptor other) {
\r
53 fStartChar = other.fStartChar;
\r
54 fEndChar = other.fEndChar;
\r
56 fIncludesSets = new ArrayList(other.fIncludesSets);
\r
59 //-------------------------------------------------------------------------------------
\r
61 // RangeDesriptor::split()
\r
63 //-------------------------------------------------------------------------------------
\r
64 void split(int where) {
\r
65 Assert.assrt(where>fStartChar && where<=fEndChar);
\r
66 RangeDescriptor nr = new RangeDescriptor(this);
\r
68 // RangeDescriptor copy constructor copies all fields.
\r
69 // Only need to update those that are different after the split.
\r
70 nr.fStartChar = where;
\r
71 this.fEndChar = where-1;
\r
72 nr.fNext = this.fNext;
\r
75 // TODO: fIncludesSets is not updated. Check it out.
\r
76 // Probably because they haven't been populated yet,
\r
77 // but still sloppy.
\r
81 //-------------------------------------------------------------------------------------
\r
83 // RangeDescriptor::setDictionaryFlag
\r
85 // Character Category Numbers that include characters from
\r
86 // the original Unicode Set named "dictionary" have bit 14
\r
87 // set to 1. The RBBI runtime engine uses this to trigger
\r
88 // use of the word dictionary.
\r
90 // This function looks through the Unicode Sets that it
\r
91 // (the range) includes, and sets the bit in fNum when
\r
92 // "dictionary" is among them.
\r
94 // TODO: a faster way would be to find the set node for
\r
95 // "dictionary" just once, rather than looking it
\r
96 // up by name every time.
\r
98 // -------------------------------------------------------------------------------------
\r
99 void setDictionaryFlag() {
\r
102 for (i=0; i<this.fIncludesSets.size(); i++) {
\r
103 RBBINode usetNode = (RBBINode)fIncludesSets.get(i);
\r
104 String setName = "";
\r
105 RBBINode setRef = usetNode.fParent;
\r
106 if (setRef != null) {
\r
107 RBBINode varRef = setRef.fParent;
\r
108 if (varRef != null && varRef.fType == RBBINode.varRef) {
\r
109 setName = varRef.fText;
\r
112 if (setName.equals("dictionary")) {
\r
113 this.fNum |= 0x4000;
\r
122 RBBIRuleBuilder fRB; // The RBBI Rule Compiler that owns us.
\r
123 RangeDescriptor fRangeList; // Head of the linked list of RangeDescriptors
\r
125 IntTrieBuilder fTrie; // The mapping TRIE that is the end result of processing
\r
126 int fTrieSize; // the Unicode Sets.
\r
128 // Groups correspond to character categories -
\r
129 // groups of ranges that are in the same original UnicodeSets.
\r
130 // fGroupCount is the index of the last used group.
\r
131 // fGroupCount+1 is also the number of columns in the RBBI state table being compiled.
\r
132 // State table column 0 is not used. Column 1 is for end-of-input.
\r
133 // column 2 is for group 0. Funny counting.
\r
139 //------------------------------------------------------------------------
\r
141 // RBBISetBuilder Constructor
\r
143 //------------------------------------------------------------------------
\r
144 RBBISetBuilder(RBBIRuleBuilder rb)
\r
150 //------------------------------------------------------------------------
\r
152 // build Build the list of non-overlapping character ranges
\r
153 // from the Unicode Sets.
\r
155 //------------------------------------------------------------------------
\r
158 RangeDescriptor rlRange;
\r
160 if (fRB.fDebugEnv!=null && fRB.fDebugEnv.indexOf("usets")>=0) {printSets();}
\r
162 // Initialize the process by creating a single range encompassing all characters
\r
163 // that is in no sets.
\r
165 fRangeList = new RangeDescriptor();
\r
166 fRangeList.fStartChar = 0;
\r
167 fRangeList.fEndChar = 0x10ffff;
\r
170 // Find the set of non-overlapping ranges of characters
\r
172 Iterator ni = fRB.fUSetNodes.iterator();
\r
173 while (ni.hasNext()) {
\r
174 usetNode = (RBBINode)ni.next();
\r
176 UnicodeSet inputSet = usetNode.fInputSet;
\r
177 int inputSetRangeCount = inputSet.getRangeCount();
\r
178 int inputSetRangeIndex = 0;
\r
179 rlRange = fRangeList;
\r
182 if (inputSetRangeIndex >= inputSetRangeCount) {
\r
185 int inputSetRangeBegin = inputSet.getRangeStart(inputSetRangeIndex);
\r
186 int inputSetRangeEnd = inputSet.getRangeEnd(inputSetRangeIndex);
\r
188 // skip over ranges from the range list that are completely
\r
189 // below the current range from the input unicode set.
\r
190 while (rlRange.fEndChar < inputSetRangeBegin) {
\r
191 rlRange = rlRange.fNext;
\r
194 // If the start of the range from the range list is before with
\r
195 // the start of the range from the unicode set, split the range list range
\r
196 // in two, with one part being before (wholly outside of) the unicode set
\r
197 // and the other containing the rest.
\r
198 // Then continue the loop; the post-split current range will then be skipped
\r
200 if (rlRange.fStartChar < inputSetRangeBegin) {
\r
201 rlRange.split(inputSetRangeBegin);
\r
205 // Same thing at the end of the ranges...
\r
206 // If the end of the range from the range list doesn't coincide with
\r
207 // the end of the range from the unicode set, split the range list
\r
208 // range in two. The first part of the split range will be
\r
209 // wholly inside the Unicode set.
\r
210 if (rlRange.fEndChar > inputSetRangeEnd) {
\r
211 rlRange.split(inputSetRangeEnd+1);
\r
214 // The current rlRange is now entirely within the UnicodeSet range.
\r
215 // Add this unicode set to the list of sets for this rlRange
\r
216 if (rlRange.fIncludesSets.indexOf(usetNode) == -1) {
\r
217 rlRange.fIncludesSets.add(usetNode);
\r
220 // Advance over ranges that we are finished with.
\r
221 if (inputSetRangeEnd == rlRange.fEndChar) {
\r
222 inputSetRangeIndex++;
\r
224 rlRange = rlRange.fNext;
\r
228 if (fRB.fDebugEnv!=null && fRB.fDebugEnv.indexOf("range")>=0) { printRanges();}
\r
231 // Group the above ranges, with each group consisting of one or more
\r
232 // ranges that are in exactly the same set of original UnicodeSets.
\r
233 // The groups are numbered, and these group numbers are the set of
\r
234 // input symbols recognized by the run-time state machine.
\r
236 // Numbering: # 0 (state table column 0) is unused.
\r
237 // # 1 is reserved - table column 1 is for end-of-input
\r
238 // # 2 is reserved - table column 2 is for beginning-in-input
\r
239 // # 3 is the first range list.
\r
241 RangeDescriptor rlSearchRange;
\r
242 for (rlRange = fRangeList; rlRange!=null; rlRange=rlRange.fNext) {
\r
243 for (rlSearchRange=fRangeList; rlSearchRange != rlRange; rlSearchRange=rlSearchRange.fNext) {
\r
244 if (rlRange.fIncludesSets.equals(rlSearchRange.fIncludesSets)) {
\r
245 rlRange.fNum = rlSearchRange.fNum;
\r
249 if (rlRange.fNum == 0) {
\r
251 rlRange.fNum = fGroupCount+2;
\r
252 rlRange.setDictionaryFlag();
\r
253 addValToSets(rlRange.fIncludesSets, fGroupCount+2);
\r
257 // Handle input sets that contain the special string {eof}.
\r
258 // Column 1 of the state table is reserved for EOF on input.
\r
259 // Column 2 is reserved for before-the-start-input.
\r
260 // (This column can be optimized away later if there are no rule
\r
261 // references to {bof}.)
\r
262 // Add this column value (1 or 2) to the equivalent expression
\r
263 // subtree for each UnicodeSet that contains the string {eof}
\r
264 // Because {bof} and {eof} are not a characters in the normal sense,
\r
265 // they doesn't affect the computation of ranges or TRIE.
\r
267 String eofString = "eof";
\r
268 String bofString = "bof";
\r
270 ni = fRB.fUSetNodes.iterator();
\r
271 while (ni.hasNext()) {
\r
272 usetNode = (RBBINode )ni.next();
\r
273 UnicodeSet inputSet = usetNode.fInputSet;
\r
274 if (inputSet.contains(eofString)) {
\r
275 addValToSet(usetNode, 1);
\r
277 if (inputSet.contains(bofString)) {
\r
278 addValToSet(usetNode, 2);
\r
284 if (fRB.fDebugEnv!=null && fRB.fDebugEnv.indexOf("rgroup")>=0) {printRangeGroups();}
\r
285 if (fRB.fDebugEnv!=null && fRB.fDebugEnv.indexOf("esets")>=0) {printSets();}
\r
288 //IntTrieBuilder(int aliasdata[], int maxdatalength,
\r
289 // int initialvalue, int leadunitvalue,
\r
290 // boolean latin1linear)
\r
292 fTrie = new IntTrieBuilder(null, // Data array (utrie will allocate one)
\r
293 100000, // Max Data Length
\r
294 0, // Initial value for all code points
\r
295 0, // Lead Surrogate unit value,
\r
296 true); // Keep Latin 1 in separately.
\r
298 for (rlRange = fRangeList; rlRange!=null; rlRange=rlRange.fNext) {
\r
299 fTrie.setRange(rlRange.fStartChar, rlRange.fEndChar+1, rlRange.fNum, true);
\r
305 //-----------------------------------------------------------------------------------
\r
307 // RBBIDataManipulate A little internal class needed only to wrap of the
\r
308 // getFoldedValue() function needed for Trie table creation.
\r
310 //-----------------------------------------------------------------------------------
\r
311 class RBBIDataManipulate implements IntTrieBuilder.DataManipulate {
\r
312 public int getFoldedValue(int start, int offset) {
\r
315 boolean [] inBlockZero = new boolean[1];
\r
317 limit = start + 0x400;
\r
318 while(start<limit) {
\r
319 value = fTrie.getValue(start, inBlockZero);
\r
320 if (inBlockZero[0]) {
\r
321 start += IntTrieBuilder.DATA_BLOCK_LENGTH;
\r
322 } else if (value != 0) {
\r
323 return offset | 0x08000;
\r
331 RBBIDataManipulate dm = new RBBIDataManipulate();
\r
333 //-----------------------------------------------------------------------------------
\r
335 // getTrieSize() Return the size that will be required to serialize the Trie.
\r
337 //-----------------------------------------------------------------------------------
\r
338 int getTrieSize() {
\r
341 // The trie serialize function returns the size of the data written.
\r
342 // null output stream says give size only, don't actually write anything.
\r
343 size = fTrie.serialize(null, true, dm );
\r
344 } catch (IOException e) {
\r
345 Assert.assrt (false);
\r
351 //-----------------------------------------------------------------------------------
\r
353 // serializeTrie() Write the serialized trie to an output stream
\r
355 //-----------------------------------------------------------------------------------
\r
356 void serializeTrie(OutputStream os) throws IOException {
\r
357 fTrie.serialize(os, true, dm );
\r
360 //------------------------------------------------------------------------
\r
362 // addValToSets Add a runtime-mapped input value to each uset from a
\r
363 // list of uset nodes. (val corresponds to a state table column.)
\r
364 // For each of the original Unicode sets - which correspond
\r
365 // directly to uset nodes - a logically equivalent expression
\r
366 // is constructed in terms of the remapped runtime input
\r
367 // symbol set. This function adds one runtime input symbol to
\r
370 // The "logically equivalent expression" is the tree for an
\r
371 // or-ing together of all of the symbols that go into the set.
\r
373 //------------------------------------------------------------------------
\r
374 void addValToSets(List sets, int val) {
\r
377 for (ix=0; ix<sets.size(); ix++) {
\r
378 RBBINode usetNode = (RBBINode )sets.get(ix);
\r
379 addValToSet(usetNode, val);
\r
383 void addValToSet(RBBINode usetNode, int val) {
\r
384 RBBINode leafNode = new RBBINode(RBBINode.leafChar);
\r
385 leafNode.fVal = val;
\r
386 if (usetNode.fLeftChild == null) {
\r
387 usetNode.fLeftChild = leafNode;
\r
388 leafNode.fParent = usetNode;
\r
390 // There are already input symbols present for this set.
\r
391 // Set up an OR node, with the previous stuff as the left child
\r
392 // and the new value as the right child.
\r
393 RBBINode orNode = new RBBINode(RBBINode.opOr);
\r
394 orNode.fLeftChild = usetNode.fLeftChild;
\r
395 orNode.fRightChild = leafNode;
\r
396 orNode.fLeftChild.fParent = orNode;
\r
397 orNode.fRightChild.fParent = orNode;
\r
398 usetNode.fLeftChild = orNode;
\r
399 orNode.fParent = usetNode;
\r
404 //------------------------------------------------------------------------
\r
406 // getNumCharCategories
\r
408 //------------------------------------------------------------------------
\r
409 int getNumCharCategories() {
\r
410 return fGroupCount + 3;
\r
414 //------------------------------------------------------------------------
\r
418 //------------------------------------------------------------------------
\r
424 //------------------------------------------------------------------------
\r
426 // getFirstChar Given a runtime RBBI character category, find
\r
427 // the first UChar32 that is in the set of chars
\r
428 // in the category.
\r
429 //------------------------------------------------------------------------
\r
430 int getFirstChar(int category) {
\r
431 RangeDescriptor rlRange;
\r
433 for (rlRange = fRangeList; rlRange!=null; rlRange=rlRange.fNext) {
\r
434 if (rlRange.fNum == category) {
\r
435 retVal = rlRange.fStartChar;
\r
444 //------------------------------------------------------------------------
\r
446 // printRanges A debugging function.
\r
447 // dump out all of the range definitions.
\r
449 //------------------------------------------------------------------------
\r
451 void printRanges() {
\r
452 RangeDescriptor rlRange;
\r
455 System.out.print("\n\n Nonoverlapping Ranges ...\n");
\r
456 for (rlRange = fRangeList; rlRange!=null; rlRange=rlRange.fNext) {
\r
457 System.out.print(" " + rlRange.fNum + " " + (int)rlRange.fStartChar + "-" + (int)rlRange.fEndChar);
\r
459 for (i=0; i<rlRange.fIncludesSets.size(); i++) {
\r
460 RBBINode usetNode = (RBBINode )rlRange.fIncludesSets.get(i);
\r
461 String setName = "anon";
\r
462 RBBINode setRef = usetNode.fParent;
\r
463 if (setRef != null) {
\r
464 RBBINode varRef = setRef.fParent;
\r
465 if (varRef != null && varRef.fType == RBBINode.varRef) {
\r
466 setName = varRef.fText;
\r
469 System.out.print(setName); System.out.print(" ");
\r
471 System.out.println("");
\r
477 //------------------------------------------------------------------------
\r
479 // printRangeGroups A debugging function.
\r
480 // dump out all of the range groups.
\r
482 //------------------------------------------------------------------------
\r
484 void printRangeGroups() {
\r
485 RangeDescriptor rlRange;
\r
486 RangeDescriptor tRange;
\r
488 int lastPrintedGroupNum = 0;
\r
490 System.out.print("\nRanges grouped by Unicode Set Membership...\n");
\r
491 for (rlRange = fRangeList; rlRange!=null; rlRange=rlRange.fNext) {
\r
492 int groupNum = rlRange.fNum & 0xbfff;
\r
493 if (groupNum > lastPrintedGroupNum) {
\r
494 lastPrintedGroupNum = groupNum;
\r
495 if (groupNum<10) {System.out.print(" ");}
\r
496 System.out.print(groupNum + " ");
\r
498 if ((rlRange.fNum & 0x4000) != 0) { System.out.print(" <DICT> ");}
\r
500 for (i=0; i<rlRange.fIncludesSets.size(); i++) {
\r
501 RBBINode usetNode = (RBBINode )rlRange.fIncludesSets.get(i);
\r
502 String setName = "anon";
\r
503 RBBINode setRef = usetNode.fParent;
\r
504 if (setRef != null) {
\r
505 RBBINode varRef = setRef.fParent;
\r
506 if (varRef != null && varRef.fType == RBBINode.varRef) {
\r
507 setName = varRef.fText;
\r
510 System.out.print(setName); System.out.print(" ");
\r
514 for (tRange = rlRange; tRange != null; tRange = tRange.fNext) {
\r
515 if (tRange.fNum == rlRange.fNum) {
\r
516 if (i++ % 5 == 0) {
\r
517 System.out.print("\n ");
\r
519 RBBINode.printHex((int)tRange.fStartChar, -1);
\r
520 System.out.print("-");
\r
521 RBBINode.printHex((int)tRange.fEndChar, 0);
\r
524 System.out.print("\n");
\r
527 System.out.print("\n");
\r
532 //------------------------------------------------------------------------
\r
534 // printSets A debugging function.
\r
535 // dump out all of the set definitions.
\r
537 //------------------------------------------------------------------------
\r
541 System.out.print("\n\nUnicode Sets List\n------------------\n");
\r
542 for (i=0; i<fRB.fUSetNodes.size(); i++) {
\r
548 usetNode = (RBBINode )fRB.fUSetNodes.get(i);
\r
550 //System.out.print(" " + i + " ");
\r
551 RBBINode.printInt(2, i);
\r
552 setName = "anonymous";
\r
553 setRef = usetNode.fParent;
\r
554 if (setRef != null) {
\r
555 varRef = setRef.fParent;
\r
556 if (varRef != null && varRef.fType == RBBINode.varRef) {
\r
557 setName = varRef.fText;
\r
560 System.out.print(" " + setName);
\r
561 System.out.print(" ");
\r
562 System.out.print(usetNode.fText);
\r
563 System.out.print("\n");
\r
564 if (usetNode.fLeftChild != null) {
\r
565 usetNode.fLeftChild.printTree(true);
\r
568 System.out.print("\n");
\r