2 *******************************************************************************
\r
3 * Copyright (C) 1996-2004, International Business Machines Corporation and *
\r
4 * others. All Rights Reserved. *
\r
5 *******************************************************************************
\r
7 package com.ibm.icu.dev.test.rbbi;
\r
9 import java.util.ListResourceBundle;
\r
12 * This resource bundle is included for testing and demonstration purposes only.
\r
13 * It applies the dictionary-based algorithm to English text that has had all the
\r
14 * spaces removed. Once we have good test cases for Thai, we will replace this
\r
15 * with good resource data (and a good dictionary file) for Thai
\r
17 public class BreakIteratorRules_en_US_TEST extends ListResourceBundle {
\r
18 private static final String DATA_NAME = "/com/ibm/icu/dev/data/rbbi/english.dict";
\r
20 // calling code will handle case where dictionary does not exist
\r
22 public Object[][] getContents() {
\r
23 return new Object[][] {
\r
24 // names of classes to instantiate for the different kinds of break
\r
25 // iterator. Notice we're now using DictionaryBasedBreakIterator
\r
26 // for word and line breaking.
\r
27 { "BreakIteratorClasses",
\r
29 "RuleBasedBreakIterator",
\r
30 // character-break iterator class
\r
31 "DictionaryBasedBreakIterator",
\r
32 // word-break iterator class
\r
33 "DictionaryBasedBreakIterator",
\r
34 // line-break iterator class
\r
35 "RuleBasedBreakIterator" } // sentence-break iterator class
\r
38 // These are the same word-breaking rules as are specified in the default
\r
39 // resource, except that the Latin letters, apostrophe, and hyphen are
\r
40 // specified as dictionary characters
\r
43 // ignore non-spacing marks, enclosing marks, and format characters,
\r
44 // all of which should not influence the algorithm
\r
45 "$_ignore_=[[:Mn:][:Me:][:Cf:]];"
\r
47 // lower and upper case Roman letters, apostrophy and dash are
\r
48 // in the English dictionary
\r
49 +"$_dictionary_=[a-zA-Z\\'\\-];"
\r
51 // Hindi phrase separator, kanji, katakana, hiragana, CJK diacriticals,
\r
52 // other letters, and digits
\r
53 +"$danda=[\u0964\u0965];"
\r
54 + "$kanji=[\u3005\u4e00-\u9fa5\uf900-\ufa2d];"
\r
55 + "$kata=[\u3099-\u309c\u30a1-\u30fe];"
\r
56 + "$hira=[\u3041-\u309e\u30fc];"
\r
57 + "$let=[[[:L:][:Mc:]]-[$kanji$kata$hira]];"
\r
60 // punctuation that can occur in the middle of a word: currently
\r
61 // dashes, apostrophes, and quotation marks
\r
62 +"$mid_word=[[:Pd:]\u00ad\u2027\\\"\\\'];"
\r
64 // punctuation that can occur in the middle of a number: currently
\r
65 // apostrophes, qoutation marks, periods, commas, and the Arabic
\r
67 +"$mid_num=[\\\"\\\'\\,\u066b\\.];"
\r
69 // punctuation that can occur at the beginning of a number: currently
\r
70 // the period, the number sign, and all currency symbols except the cents sign
\r
71 +"$pre_num=[[[:Sc:]-[\u00a2]]\\#\\.];"
\r
73 // punctuation that can occur at the end of a number: currently
\r
74 // the percent, per-thousand, per-ten-thousand, and Arabic percent
\r
75 // signs, the cents sign, and the ampersand
\r
76 +"$post_num=[\\%\\&\u00a2\u066a\u2030\u2031];"
\r
78 // line separators: currently LF, FF, PS, and LS
\r
79 +"$ls=[\n\u000c\u2028\u2029];"
\r
81 // whitespace: all space separators and the tab character
\r
84 // a word is a sequence of letters that may contain internal
\r
85 // punctuation, as long as it begins and ends with a letter and
\r
86 // never contains two punctuation marks in a row
\r
87 +"$word=($let+($mid_word$let+)*$danda?);"
\r
89 // a number is a sequence of digits that may contain internal
\r
90 // punctuation, as long as it begins and ends with a digit and
\r
91 // never contains two punctuation marks in a row.
\r
92 +"$number=($dgt+($mid_num$dgt+)*);"
\r
94 // break after every character, with the following exceptions
\r
95 // (this will cause punctuation marks that aren't considered
\r
96 // part of words or numbers to be treated as words unto themselves)
\r
99 // keep together any sequence of contiguous words and numbers
\r
100 // (including just one of either), plus an optional trailing
\r
101 // number-suffix character
\r
102 +"$word?($number$word)*($number$post_num?)?;"
\r
104 // keep together and sequence of contiguous words and numbers
\r
105 // that starts with a number-prefix character and a number,
\r
106 // and may end with a number-suffix character
\r
107 +"$pre_num($number$word)*($number$post_num?)?;"
\r
109 // keep together runs of whitespace (optionally with a single trailing
\r
110 // line separator or CRLF sequence)
\r
113 // keep together runs of Katakana
\r
116 // keep together runs of Hiragana
\r
119 // keep together runs of Kanji
\r
122 // These are the same line-breaking rules as are specified in the default
\r
123 // resource, except that the Latin letters, apostrophe, and hyphen are
\r
124 // specified as dictionary characters
\r
125 { "LineBreakRules",
\r
126 // ignore non-spacing marks, enclosing marks, and format characters
\r
127 "$_ignore_=[[:Mn:][:Me:][:Cf:]];"
\r
129 // lower and upper case Roman letters, apostrophy and dash
\r
130 // are in the English dictionary
\r
131 +"$_dictionary_=[a-zA-Z\\'\\-];"
\r
133 // Hindi phrase separators
\r
134 +"$danda=[\u0964\u0965];"
\r
136 // characters that always cause a break: ETX, tab, LF, FF, LS, and PS
\r
137 +"$break=[\u0003\t\n\f\u2028\u2029];"
\r
139 // characters that always prevent a break: the non-breaking space
\r
140 // and similar characters
\r
141 +"$nbsp=[\u00a0\u2007\u2011\ufeff];"
\r
143 // whitespace: space separators and control characters, except for
\r
144 // CR and the other characters mentioned above
\r
145 +"$space=[[[:Zs:][:Cc:]]-[$nbsp$break\r]];"
\r
147 // dashes: dash punctuation and the discretionary hyphen, except for
\r
148 // non-breaking hyphens
\r
149 +"$dash=[[[:Pd:]\u00ad]-[$nbsp]];"
\r
151 // characters that stick to a word if they precede it: currency symbols
\r
152 // (except the cents sign) and starting punctuation
\r
153 +"$pre_word=[[[:Sc:]-[\u00a2]][:Ps:]\\\"\\\'];"
\r
155 // characters that stick to a word if they follow it: ending punctuation,
\r
156 // other punctuation that usually occurs at the end of a sentence,
\r
157 // small Kana characters, some CJK diacritics, etc.
\r
158 +"$post_word=[[:Pe:]\\!\\\"\\\'\\%\\.\\,\\:\\;\\?\u00a2\u00b0\u066a\u2030-\u2034"
\r
159 + "\u2103\u2105\u2109\u3001\u3002\u3005\u3041\u3043\u3045\u3047\u3049\u3063"
\r
160 + "\u3083\u3085\u3087\u308e\u3099-\u309e\u30a1\u30a3\u30a5\u30a7\u30a9"
\r
161 + "\u30c3\u30e3\u30e5\u30e7\u30ee\u30f5\u30f6\u30fc-\u30fe\uff01\uff0c"
\r
164 // Kanji: actually includes both Kanji and Kana, except for small Kana and
\r
166 +"$kanji=[[\u4e00-\u9fa5\uf900-\ufa2d\u3041-\u3094\u30a1-\u30fa]-[$post_word$_ignore_]];"
\r
169 +"$digit=[[:Nd:][:No:]];"
\r
171 // punctuation that can occur in the middle of a number: periods and commas
\r
172 +"$mid_num=[\\.\\,];"
\r
174 // everything not mentioned above, plus the quote marks (which are both
\r
175 // <pre-word>, <post-word>, and <char>)
\r
176 +"$char=[^$break$space$dash$kanji$nbsp$_ignore_$pre_word$post_word$mid_num$danda\r\\\"\\\'];"
\r
178 // a "number" is a run of prefix characters and dashes, followed by one or
\r
179 // more digits with isolated number-punctuation characters interspersed
\r
180 +"$number=([$pre_word$dash]*$digit+($mid_num$digit+)*);"
\r
182 // the basic core of a word can be either a "number" as defined above, a single
\r
183 // "Kanji" character, or a run of any number of not-explicitly-mentioned
\r
184 // characters (this includes Latin letters)
\r
185 +"$word_core=([$pre_word$char]*|$kanji|$number);"
\r
187 // a word may end with an optional suffix that be either a run of one or
\r
188 // more dashes or a run of word-suffix characters, followed by an optional
\r
189 // run of whitespace
\r
190 +"$word_suffix=(($dash+|$post_word*)$space*);"
\r
192 // a word, thus, is an optional run of word-prefix characters, followed by
\r
193 // a word core and a word suffix (the syntax of <word-core> and <word-suffix>
\r
194 // actually allows either of them to match the empty string, putting a break
\r
195 // between things like ")(" or "aaa(aaa"
\r
196 +"$word=($pre_word*$word_core$word_suffix);"
\r
198 // finally, the rule that does the work: Keep together any run of words that
\r
199 // are joined by runs of one of more non-spacing mark. Also keep a trailing
\r
200 // line-break character or CRLF combination with the word. (line separators
\r
201 // "win" over nbsp's)
\r
202 +"$word($nbsp+$word)*\r?$break?;" },
\r
204 // these two resources specify the pathnames of the dictionary files to
\r
205 // use for word breaking and line breaking. Both currently refer to
\r
206 // a file called english.dict placed in com.ibm.icu.impl.data
\r
207 // somewhere in the class path. It's important to note that
\r
208 // english.dict was created for testing purposes only, and doesn't
\r
209 // come anywhere close to being an exhaustive dictionary of English
\r
210 // words (basically, it contains all the words in the Declaration of
\r
211 // Independence, and the Revised Standard Version of the book of Genesis,
\r
212 // plus a few other words thrown in to show more interesting cases).
\r
213 // { "WordBreakDictionary", "com\\ibm\\text\\resources\\english.dict" },
\r
214 // { "LineBreakDictionary", "com\\ibm\\text\\resources\\english.dict" }
\r
215 { "WordBreakDictionary", DATA_NAME },
\r
216 { "LineBreakDictionary", DATA_NAME }
\r