go

[Dictionary.git] / jars / icu4j-4_4_2-src / main / tests / core / src / com / ibm / icu / dev / test / rbbi / BreakIteratorRules_en_US_TEST.java

/*\r
 *******************************************************************************\r
 * Copyright (C) 1996-2004, International Business Machines Corporation and    *\r
 * others. All Rights Reserved.                                                *\r
 *******************************************************************************\r
 */\r
package com.ibm.icu.dev.test.rbbi;\r
\r
import java.util.ListResourceBundle;\r
\r
/**\r
 * This resource bundle is included for testing and demonstration purposes only.\r
 * It applies the dictionary-based algorithm to English text that has had all the\r
 * spaces removed.  Once we have good test cases for Thai, we will replace this\r
 * with good resource data (and a good dictionary file) for Thai\r
 */\r
public class BreakIteratorRules_en_US_TEST extends ListResourceBundle {\r
    private static final String DATA_NAME = "/com/ibm/icu/dev/data/rbbi/english.dict";\r
\r
    // calling code will handle case where dictionary does not exist\r
\r
    public Object[][] getContents() {\r
        return new Object[][] {\r
            // names of classes to instantiate for the different kinds of break\r
            // iterator.  Notice we're now using DictionaryBasedBreakIterator\r
            // for word and line breaking.\r
            {   "BreakIteratorClasses",\r
                new String[] {\r
                    "RuleBasedBreakIterator",\r
                    // character-break iterator class\r
                    "DictionaryBasedBreakIterator",\r
                    // word-break iterator class\r
                    "DictionaryBasedBreakIterator",\r
                    // line-break iterator class\r
                    "RuleBasedBreakIterator" } // sentence-break iterator class\r
            },\r
\r
            // These are the same word-breaking rules as are specified in the default\r
            // resource, except that the Latin letters, apostrophe, and hyphen are\r
            // specified as dictionary characters\r
            {\r
                "WordBreakRules",\r
                // ignore non-spacing marks, enclosing marks, and format characters,\r
                // all of which should not influence the algorithm\r
                "$_ignore_=[[:Mn:][:Me:][:Cf:]];"\r
\r
                // lower and upper case Roman letters, apostrophy and dash are\r
                // in the English dictionary\r
                +"$_dictionary_=[a-zA-Z\\'\\-];"\r
\r
                // Hindi phrase separator, kanji, katakana, hiragana, CJK diacriticals,\r
                // other letters, and digits\r
                +"$danda=[\u0964\u0965];"\r
                    + "$kanji=[\u3005\u4e00-\u9fa5\uf900-\ufa2d];"\r
                    + "$kata=[\u3099-\u309c\u30a1-\u30fe];"\r
                    + "$hira=[\u3041-\u309e\u30fc];"\r
                    + "$let=[[[:L:][:Mc:]]-[$kanji$kata$hira]];"\r
                    + "$dgt=[:N:];"\r
\r
                // punctuation that can occur in the middle of a word: currently\r
                // dashes, apostrophes, and quotation marks\r
                +"$mid_word=[[:Pd:]\u00ad\u2027\\\"\\\'];"\r
\r
                // punctuation that can occur in the middle of a number: currently\r
                // apostrophes, qoutation marks, periods, commas, and the Arabic\r
                // decimal point\r
                +"$mid_num=[\\\"\\\'\\,\u066b\\.];"\r
\r
                // punctuation that can occur at the beginning of a number: currently\r
                // the period, the number sign, and all currency symbols except the cents sign\r
                +"$pre_num=[[[:Sc:]-[\u00a2]]\\#\\.];"\r
\r
                // punctuation that can occur at the end of a number: currently\r
                // the percent, per-thousand, per-ten-thousand, and Arabic percent\r
                // signs, the cents sign, and the ampersand\r
                +"$post_num=[\\%\\&\u00a2\u066a\u2030\u2031];"\r
\r
                // line separators: currently LF, FF, PS, and LS\r
                +"$ls=[\n\u000c\u2028\u2029];"\r
\r
                // whitespace: all space separators and the tab character\r
                +"$ws=[[:Zs:]\t];"\r
\r
                // a word is a sequence of letters that may contain internal\r
                // punctuation, as long as it begins and ends with a letter and\r
                // never contains two punctuation marks in a row\r
                +"$word=($let+($mid_word$let+)*$danda?);"\r
\r
                // a number is a sequence of digits that may contain internal\r
                // punctuation, as long as it begins and ends with a digit and\r
                // never contains two punctuation marks in a row.\r
                +"$number=($dgt+($mid_num$dgt+)*);"\r
\r
                // break after every character, with the following exceptions\r
                // (this will cause punctuation marks that aren't considered\r
                // part of words or numbers to be treated as words unto themselves)\r
                +".;"\r
\r
                // keep together any sequence of contiguous words and numbers\r
                // (including just one of either), plus an optional trailing\r
                // number-suffix character\r
                +"$word?($number$word)*($number$post_num?)?;"\r
\r
                // keep together and sequence of contiguous words and numbers\r
                // that starts with a number-prefix character and a number,\r
                // and may end with a number-suffix character\r
                +"$pre_num($number$word)*($number$post_num?)?;"\r
\r
                // keep together runs of whitespace (optionally with a single trailing\r
                // line separator or CRLF sequence)\r
                +"$ws*\r?$ls?;"\r
\r
                // keep together runs of Katakana\r
                +"$kata*;"\r
\r
                // keep together runs of Hiragana\r
                +"$hira*;"\r
\r
                // keep together runs of Kanji\r
                +"$kanji*;" },\r
\r
            // These are the same line-breaking rules as are specified in the default\r
            // resource, except that the Latin letters, apostrophe, and hyphen are\r
            // specified as dictionary characters\r
            {   "LineBreakRules",\r
                // ignore non-spacing marks, enclosing marks, and format characters\r
                "$_ignore_=[[:Mn:][:Me:][:Cf:]];"\r
\r
                // lower and upper case Roman letters, apostrophy and dash\r
                // are in the English dictionary\r
                +"$_dictionary_=[a-zA-Z\\'\\-];"\r
\r
                // Hindi phrase separators\r
                +"$danda=[\u0964\u0965];"\r
\r
                // characters that always cause a break: ETX, tab, LF, FF, LS, and PS\r
                +"$break=[\u0003\t\n\f\u2028\u2029];"\r
\r
                // characters that always prevent a break: the non-breaking space\r
                // and similar characters\r
                +"$nbsp=[\u00a0\u2007\u2011\ufeff];"\r
\r
                // whitespace: space separators and control characters, except for\r
                // CR and the other characters mentioned above\r
                +"$space=[[[:Zs:][:Cc:]]-[$nbsp$break\r]];"\r
\r
                // dashes: dash punctuation and the discretionary hyphen, except for\r
                // non-breaking hyphens\r
                +"$dash=[[[:Pd:]\u00ad]-[$nbsp]];"\r
\r
                // characters that stick to a word if they precede it: currency symbols\r
                // (except the cents sign) and starting punctuation\r
                +"$pre_word=[[[:Sc:]-[\u00a2]][:Ps:]\\\"\\\'];"\r
\r
                // characters that stick to a word if they follow it: ending punctuation,\r
                // other punctuation that usually occurs at the end of a sentence,\r
                // small Kana characters, some CJK diacritics, etc.\r
                +"$post_word=[[:Pe:]\\!\\\"\\\'\\%\\.\\,\\:\\;\\?\u00a2\u00b0\u066a\u2030-\u2034"\r
                    + "\u2103\u2105\u2109\u3001\u3002\u3005\u3041\u3043\u3045\u3047\u3049\u3063"\r
                    + "\u3083\u3085\u3087\u308e\u3099-\u309e\u30a1\u30a3\u30a5\u30a7\u30a9"\r
                    + "\u30c3\u30e3\u30e5\u30e7\u30ee\u30f5\u30f6\u30fc-\u30fe\uff01\uff0c"\r
                    + "\uff0e\uff1f];"\r
\r
                // Kanji: actually includes both Kanji and Kana, except for small Kana and\r
                // CJK diacritics\r
                +"$kanji=[[\u4e00-\u9fa5\uf900-\ufa2d\u3041-\u3094\u30a1-\u30fa]-[$post_word$_ignore_]];"\r
\r
                // digits\r
                +"$digit=[[:Nd:][:No:]];"\r
\r
                // punctuation that can occur in the middle of a number: periods and commas\r
                +"$mid_num=[\\.\\,];"\r
\r
                // everything not mentioned above, plus the quote marks (which are both\r
                // <pre-word>, <post-word>, and <char>)\r
                +"$char=[^$break$space$dash$kanji$nbsp$_ignore_$pre_word$post_word$mid_num$danda\r\\\"\\\'];"\r
\r
                // a "number" is a run of prefix characters and dashes, followed by one or\r
                // more digits with isolated number-punctuation characters interspersed\r
                +"$number=([$pre_word$dash]*$digit+($mid_num$digit+)*);"\r
\r
                // the basic core of a word can be either a "number" as defined above, a single\r
                // "Kanji" character, or a run of any number of not-explicitly-mentioned\r
                // characters (this includes Latin letters)\r
                +"$word_core=([$pre_word$char]*|$kanji|$number);"\r
\r
                // a word may end with an optional suffix that be either a run of one or\r
                // more dashes or a run of word-suffix characters, followed by an optional\r
                // run of whitespace\r
                +"$word_suffix=(($dash+|$post_word*)$space*);"\r
\r
                // a word, thus, is an optional run of word-prefix characters, followed by\r
                // a word core and a word suffix (the syntax of <word-core> and <word-suffix>\r
                // actually allows either of them to match the empty string, putting a break\r
                // between things like ")(" or "aaa(aaa"\r
                +"$word=($pre_word*$word_core$word_suffix);"\r
\r
                // finally, the rule that does the work: Keep together any run of words that\r
                // are joined by runs of one of more non-spacing mark.  Also keep a trailing\r
                // line-break character or CRLF combination with the word.  (line separators\r
                // "win" over nbsp's)\r
                +"$word($nbsp+$word)*\r?$break?;" },\r
\r
            // these two resources specify the pathnames of the dictionary files to\r
            // use for word breaking and line breaking.  Both currently refer to\r
            // a file called english.dict placed in com.ibm.icu.impl.data\r
            // somewhere in the class path.  It's important to note that\r
            // english.dict was created for testing purposes only, and doesn't\r
            // come anywhere close to being an exhaustive dictionary of English\r
            // words (basically, it contains all the words in the Declaration of\r
            // Independence, and the Revised Standard Version of the book of Genesis,\r
            // plus a few other words thrown in to show more interesting cases).\r
            // { "WordBreakDictionary", "com\\ibm\\text\\resources\\english.dict" },\r
            // { "LineBreakDictionary", "com\\ibm\\text\\resources\\english.dict" }\r
            {   "WordBreakDictionary", DATA_NAME }, \r
            {   "LineBreakDictionary", DATA_NAME }\r
        };\r
    }\r
}\r