2 *******************************************************************************
\r
3 * Copyright (C) 1996-2009, International Business Machines Corporation and *
\r
4 * others. All Rights Reserved. *
\r
5 *******************************************************************************
\r
7 package com.ibm.icu.text;
\r
9 import java.util.Hashtable;
\r
13 * <code>RuleBasedTransliterator</code> is a transliterator
\r
14 * that reads a set of rules in order to determine how to perform
\r
15 * translations. Rule sets are stored in resource bundles indexed by
\r
16 * name. Rules within a rule set are separated by semicolons (';').
\r
17 * To include a literal semicolon, prefix it with a backslash ('\').
\r
18 * Whitespace, as defined by <code>UCharacterProperty.isRuleWhiteSpace()</code>,
\r
19 * is ignored. If the first non-blank character on a line is '#',
\r
20 * the entire line is ignored as a comment. </p>
\r
22 * <p>Each set of rules consists of two groups, one forward, and one
\r
23 * reverse. This is a convention that is not enforced; rules for one
\r
24 * direction may be omitted, with the result that translations in
\r
25 * that direction will not modify the source text. In addition,
\r
26 * bidirectional forward-reverse rules may be specified for
\r
27 * symmetrical transformations.</p>
\r
29 * <p><b>Rule syntax</b> </p>
\r
31 * <p>Rule statements take one of the following forms: </p>
\r
34 * <dt><code>$alefmadda=\u0622;</code></dt>
\r
35 * <dd><strong>Variable definition.</strong> The name on the
\r
36 * left is assigned the text on the right. In this example,
\r
37 * after this statement, instances of the left hand name,
\r
38 * "<code>$alefmadda</code>", will be replaced by
\r
39 * the Unicode character U+0622. Variable names must begin
\r
40 * with a letter and consist only of letters, digits, and
\r
41 * underscores. Case is significant. Duplicate names cause
\r
42 * an exception to be thrown, that is, variables cannot be
\r
43 * redefined. The right hand side may contain well-formed
\r
44 * text of any length, including no text at all ("<code>$empty=;</code>").
\r
45 * The right hand side may contain embedded <code>UnicodeSet</code>
\r
46 * patterns, for example, "<code>$softvowel=[eiyEIY]</code>".</dd>
\r
48 * <dt><code>ai>$alefmadda;</code></dt>
\r
49 * <dd><strong>Forward translation rule.</strong> This rule
\r
50 * states that the string on the left will be changed to the
\r
51 * string on the right when performing forward
\r
52 * transliteration.</dd>
\r
54 * <dt><code>ai<$alefmadda;</code></dt>
\r
55 * <dd><strong>Reverse translation rule.</strong> This rule
\r
56 * states that the string on the right will be changed to
\r
57 * the string on the left when performing reverse
\r
58 * transliteration.</dd>
\r
62 * <dt><code>ai<>$alefmadda;</code></dt>
\r
63 * <dd><strong>Bidirectional translation rule.</strong> This
\r
64 * rule states that the string on the right will be changed
\r
65 * to the string on the left when performing forward
\r
66 * transliteration, and vice versa when performing reverse
\r
67 * transliteration.</dd>
\r
70 * <p>Translation rules consist of a <em>match pattern</em> and an <em>output
\r
71 * string</em>. The match pattern consists of literal characters,
\r
72 * optionally preceded by context, and optionally followed by
\r
73 * context. Context characters, like literal pattern characters,
\r
74 * must be matched in the text being transliterated. However, unlike
\r
75 * literal pattern characters, they are not replaced by the output
\r
76 * text. For example, the pattern "<code>abc{def}</code>"
\r
77 * indicates the characters "<code>def</code>" must be
\r
78 * preceded by "<code>abc</code>" for a successful match.
\r
79 * If there is a successful match, "<code>def</code>" will
\r
80 * be replaced, but not "<code>abc</code>". The final '<code>}</code>'
\r
81 * is optional, so "<code>abc{def</code>" is equivalent to
\r
82 * "<code>abc{def}</code>". Another example is "<code>{123}456</code>"
\r
83 * (or "<code>123}456</code>") in which the literal
\r
84 * pattern "<code>123</code>" must be followed by "<code>456</code>".
\r
87 * <p>The output string of a forward or reverse rule consists of
\r
88 * characters to replace the literal pattern characters. If the
\r
89 * output string contains the character '<code>|</code>', this is
\r
90 * taken to indicate the location of the <em>cursor</em> after
\r
91 * replacement. The cursor is the point in the text at which the
\r
92 * next replacement, if any, will be applied. The cursor is usually
\r
93 * placed within the replacement text; however, it can actually be
\r
94 * placed into the precending or following context by using the
\r
95 * special character '<code>@</code>'. Examples:</p>
\r
98 * <p><code>a {foo} z > | @ bar; # foo -> bar, move cursor
\r
100 * {foo} xyz > bar @@|; # foo -> bar, cursor between
\r
101 * y and z</code></p>
\r
104 * <p><b>UnicodeSet</b></p>
\r
106 * <p><code>UnicodeSet</code> patterns may appear anywhere that
\r
107 * makes sense. They may appear in variable definitions.
\r
108 * Contrariwise, <code>UnicodeSet</code> patterns may themselves
\r
109 * contain variable references, such as "<code>$a=[a-z];$not_a=[^$a]</code>",
\r
110 * or "<code>$range=a-z;$ll=[$range]</code>".</p>
\r
112 * <p><code>UnicodeSet</code> patterns may also be embedded directly
\r
113 * into rule strings. Thus, the following two rules are equivalent:</p>
\r
116 * <p><code>$vowel=[aeiou]; $vowel>'*'; # One way to do this<br>
\r
118 * #
\r
119 * Another way</code></p>
\r
122 * <p>See {@link UnicodeSet} for more documentation and examples.</p>
\r
124 * <p><b>Segments</b></p>
\r
126 * <p>Segments of the input string can be matched and copied to the
\r
127 * output string. This makes certain sets of rules simpler and more
\r
128 * general, and makes reordering possible. For example:</p>
\r
131 * <p><code>([a-z]) > $1 $1;
\r
132 * #
\r
133 * double lowercase letters<br>
\r
134 * ([:Lu:]) ([:Ll:]) > $2 $1; # reverse order of Lu-Ll pairs</code></p>
\r
137 * <p>The segment of the input string to be copied is delimited by
\r
138 * "<code>(</code>" and "<code>)</code>". Up to
\r
139 * nine segments may be defined. Segments may not overlap. In the
\r
140 * output string, "<code>$1</code>" through "<code>$9</code>"
\r
141 * represent the input string segments, in left-to-right order of
\r
144 * <p><b>Anchors</b></p>
\r
146 * <p>Patterns can be anchored to the beginning or the end of the text. This is done with the
\r
147 * special characters '<code>^</code>' and '<code>$</code>'. For example:</p>
\r
150 * <p><code>^ a > 'BEG_A'; # match 'a' at start of text<br>
\r
151 * a > 'A'; # match other instances
\r
153 * z $ > 'END_Z'; # match 'z' at end of text<br>
\r
154 * z > 'Z'; # match other instances
\r
155 * of 'z'</code></p>
\r
158 * <p>It is also possible to match the beginning or the end of the text using a <code>UnicodeSet</code>.
\r
159 * This is done by including a virtual anchor character '<code>$</code>' at the end of the
\r
160 * set pattern. Although this is usually the match chafacter for the end anchor, the set will
\r
161 * match either the beginning or the end of the text, depending on its placement. For
\r
165 * <p><code>$x = [a-z$]; # match 'a' through 'z' OR anchor<br>
\r
166 * $x 1 > 2; # match '1' after a-z or at the start<br>
\r
167 * 3 $x > 4; # match '3' before a-z or at the end</code></p>
\r
170 * <p><b>Example</b> </p>
\r
172 * <p>The following example rules illustrate many of the features of
\r
173 * the rule language. </p>
\r
175 * <table border="0" cellpadding="4">
\r
177 * <td valign="top">Rule 1.</td>
\r
178 * <td valign="top" nowrap><code>abc{def}>x|y</code></td>
\r
181 * <td valign="top">Rule 2.</td>
\r
182 * <td valign="top" nowrap><code>xyz>r</code></td>
\r
185 * <td valign="top">Rule 3.</td>
\r
186 * <td valign="top" nowrap><code>yz>q</code></td>
\r
190 * <p>Applying these rules to the string "<code>adefabcdefz</code>"
\r
191 * yields the following results: </p>
\r
193 * <table border="0" cellpadding="4">
\r
195 * <td valign="top" nowrap><code>|adefabcdefz</code></td>
\r
196 * <td valign="top">Initial state, no rules match. Advance
\r
200 * <td valign="top" nowrap><code>a|defabcdefz</code></td>
\r
201 * <td valign="top">Still no match. Rule 1 does not match
\r
202 * because the preceding context is not present.</td>
\r
205 * <td valign="top" nowrap><code>ad|efabcdefz</code></td>
\r
206 * <td valign="top">Still no match. Keep advancing until
\r
207 * there is a match...</td>
\r
210 * <td valign="top" nowrap><code>ade|fabcdefz</code></td>
\r
211 * <td valign="top">...</td>
\r
214 * <td valign="top" nowrap><code>adef|abcdefz</code></td>
\r
215 * <td valign="top">...</td>
\r
218 * <td valign="top" nowrap><code>adefa|bcdefz</code></td>
\r
219 * <td valign="top">...</td>
\r
222 * <td valign="top" nowrap><code>adefab|cdefz</code></td>
\r
223 * <td valign="top">...</td>
\r
226 * <td valign="top" nowrap><code>adefabc|defz</code></td>
\r
227 * <td valign="top">Rule 1 matches; replace "<code>def</code>"
\r
228 * with "<code>xy</code>" and back up the cursor
\r
229 * to before the '<code>y</code>'.</td>
\r
232 * <td valign="top" nowrap><code>adefabcx|yz</code></td>
\r
233 * <td valign="top">Although "<code>xyz</code>" is
\r
234 * present, rule 2 does not match because the cursor is
\r
235 * before the '<code>y</code>', not before the '<code>x</code>'.
\r
236 * Rule 3 does match. Replace "<code>yz</code>"
\r
237 * with "<code>q</code>".</td>
\r
240 * <td valign="top" nowrap><code>adefabcxq|</code></td>
\r
241 * <td valign="top">The cursor is at the end;
\r
242 * transliteration is complete.</td>
\r
246 * <p>The order of rules is significant. If multiple rules may match
\r
247 * at some point, the first matching rule is applied. </p>
\r
249 * <p>Forward and reverse rules may have an empty output string.
\r
250 * Otherwise, an empty left or right hand side of any statement is a
\r
251 * syntax error. </p>
\r
253 * <p>Single quotes are used to quote any character other than a
\r
254 * digit or letter. To specify a single quote itself, inside or
\r
255 * outside of quotes, use two single quotes in a row. For example,
\r
256 * the rule "<code>'>'>o''clock</code>" changes the
\r
257 * string "<code>></code>" to the string "<code>o'clock</code>".
\r
260 * <p><b>Notes</b> </p>
\r
262 * <p>While a RuleBasedTransliterator is being built, it checks that
\r
263 * the rules are added in proper order. For example, if the rule
\r
264 * "a>x" is followed by the rule "ab>y",
\r
265 * then the second rule will throw an exception. The reason is that
\r
266 * the second rule can never be triggered, since the first rule
\r
267 * always matches anything it matches. In other words, the first
\r
268 * rule <em>masks</em> the second rule. </p>
\r
270 * <p>Copyright (c) IBM Corporation 1999-2000. All rights reserved.</p>
\r
274 * @deprecated This API is ICU internal only.
\r
276 public class RuleBasedTransliterator extends Transliterator {
\r
281 * Constructs a new transliterator from the given rules.
\r
282 * @param rules rules, separated by ';'
\r
283 * @param direction either FORWARD or REVERSE.
\r
284 * @exception IllegalArgumentException if rules are malformed
\r
285 * or direction is invalid.
\r
287 * @deprecated This API is ICU internal only.
\r
289 /*public RuleBasedTransliterator(String ID, String rules, int direction,
\r
290 UnicodeFilter filter) {
\r
292 if (direction != FORWARD && direction != REVERSE) {
\r
293 throw new IllegalArgumentException("Invalid direction");
\r
296 TransliteratorParser parser = new TransliteratorParser();
\r
297 parser.parse(rules, direction);
\r
298 if (parser.idBlockVector.size() != 0 ||
\r
299 parser.compoundFilter != null) {
\r
300 throw new IllegalArgumentException("::ID blocks illegal in RuleBasedTransliterator constructor");
\r
303 data = (Data)parser.dataVector.get(0);
\r
304 setMaximumContextLength(data.ruleSet.getMaximumContextLength());
\r
308 * Constructs a new transliterator from the given rules in the
\r
309 * <code>FORWARD</code> direction.
\r
310 * @param rules rules, separated by ';'
\r
311 * @exception IllegalArgumentException if rules are malformed
\r
312 * or direction is invalid.
\r
314 * @deprecated This API is ICU internal only.
\r
316 /*public RuleBasedTransliterator(String ID, String rules) {
\r
317 this(ID, rules, FORWARD, null);
\r
320 RuleBasedTransliterator(String ID, Data data, UnicodeFilter filter) {
\r
323 setMaximumContextLength(data.ruleSet.getMaximumContextLength());
\r
327 * Implements {@link Transliterator#handleTransliterate}.
\r
329 * @deprecated This API is ICU internal only.
\r
331 protected synchronized void handleTransliterate(Replaceable text,
\r
332 Position index, boolean incremental) {
\r
333 /* We keep start and limit fixed the entire time,
\r
334 * relative to the text -- limit may move numerically if text is
\r
335 * inserted or removed. The cursor moves from start to limit, with
\r
336 * replacements happening under it.
\r
338 * Example: rules 1. ab>x|y
\r
341 * |eabcd start - no match, advance cursor
\r
342 * e|abcd match rule 1 - change text & adjust cursor
\r
343 * ex|ycd match rule 2 - change text & adjust cursor
\r
344 * exz|d no match, advance cursor
\r
350 * creates an infinite loop. To prevent that, we put an arbitrary
\r
351 * limit on the number of iterations that we take, one that is
\r
352 * high enough that any reasonable rules are ok, but low enough to
\r
353 * prevent a server from hanging. The limit is 16 times the
\r
354 * number of characters n, unless n is so large that 16n exceeds a
\r
358 int loopLimit = (index.limit - index.start) << 4;
\r
359 if (loopLimit < 0) {
\r
360 loopLimit = 0x7FFFFFFF;
\r
363 while (index.start < index.limit &&
\r
364 loopCount <= loopLimit &&
\r
365 data.ruleSet.transliterate(text, index, incremental)) {
\r
371 static class Data {
\r
373 variableNames = new Hashtable<String, char[]>();
\r
374 ruleSet = new TransliterationRuleSet();
\r
378 * Rule table. May be empty.
\r
380 public TransliterationRuleSet ruleSet;
\r
383 * Map variable name (String) to variable (char[]). A variable name
\r
384 * corresponds to zero or more characters, stored in a char[] array in
\r
385 * this hash. One or more of these chars may also correspond to a
\r
386 * UnicodeSet, in which case the character in the char[] in this hash is
\r
387 * a stand-in: it is an index for a secondary lookup in
\r
388 * data.variables. The stand-in also represents the UnicodeSet in
\r
389 * the stored rules.
\r
391 Hashtable<String, char[]> variableNames;
\r
394 * Map category variable (Character) to UnicodeMatcher or UnicodeReplacer.
\r
395 * Variables that correspond to a set of characters are mapped
\r
396 * from variable name to a stand-in character in data.variableNames.
\r
397 * The stand-in then serves as a key in this hash to lookup the
\r
398 * actual UnicodeSet object. In addition, the stand-in is
\r
399 * stored in the rule text to represent the set of characters.
\r
400 * variables[i] represents character (variablesBase + i).
\r
402 Object[] variables;
\r
405 * The character that represents variables[0]. Characters
\r
406 * variablesBase through variablesBase +
\r
407 * variables.length - 1 represent UnicodeSet objects.
\r
409 char variablesBase;
\r
412 * Return the UnicodeMatcher represented by the given character, or
\r
415 public UnicodeMatcher lookupMatcher(int standIn) {
\r
416 int i = standIn - variablesBase;
\r
417 return (i >= 0 && i < variables.length)
\r
418 ? (UnicodeMatcher) variables[i] : null;
\r
422 * Return the UnicodeReplacer represented by the given character, or
\r
425 public UnicodeReplacer lookupReplacer(int standIn) {
\r
426 int i = standIn - variablesBase;
\r
427 return (i >= 0 && i < variables.length)
\r
428 ? (UnicodeReplacer) variables[i] : null;
\r
434 * Return a representation of this transliterator as source rules.
\r
435 * These rules will produce an equivalent transliterator if used
\r
436 * to construct a new transliterator.
\r
437 * @param escapeUnprintable if TRUE then convert unprintable
\r
438 * character to their hex escape representations, \\uxxxx or
\r
439 * \\Uxxxxxxxx. Unprintable characters are those other than
\r
440 * U+000A, U+0020..U+007E.
\r
441 * @return rules string
\r
443 * @deprecated This API is ICU internal only.
\r
445 public String toRules(boolean escapeUnprintable) {
\r
446 return data.ruleSet.toRules(escapeUnprintable);
\r
450 * Return the set of all characters that may be modified by this
\r
451 * Transliterator, ignoring the effect of our filter.
\r
453 * @deprecated This API is ICU internal only.
\r
455 protected UnicodeSet handleGetSourceSet() {
\r
456 return data.ruleSet.getSourceTargetSet(false);
\r
460 * Returns the set of all characters that may be generated as
\r
461 * replacement text by this transliterator.
\r
463 * @deprecated This API is ICU internal only.
\r
465 public UnicodeSet getTargetSet() {
\r
466 return data.ruleSet.getSourceTargetSet(true);
\r
470 * Temporary hack for registry problem. Needs to be replaced by better architecture.
\r
472 * @deprecated This API is ICU internal only.
\r
474 public Transliterator safeClone() {
\r
475 UnicodeFilter filter = getFilter();
\r
476 if (filter != null && filter instanceof UnicodeSet) {
\r
477 filter = new UnicodeSet((UnicodeSet)filter);
\r
479 return new RuleBasedTransliterator(getID(), data, filter);
\r
projects / Dictionary.git / blob