go

[Dictionary.git] / jars / icu4j-4_4_2-src / main / tests / core / src / com / ibm / icu / dev / test / rbbi / SimpleBITest.java

/*\r
 *******************************************************************************\r
 * Copyright (C) 1996-2006, International Business Machines Corporation and    *\r
 * others. All Rights Reserved.                                                *\r
 *******************************************************************************\r
 */\r
package com.ibm.icu.dev.test.rbbi;\r
\r
import java.io.IOException;\r
import java.io.InputStream;\r
import java.util.ListResourceBundle;\r
import java.util.MissingResourceException;\r
\r
import com.ibm.icu.dev.test.TestFmwk;\r
import com.ibm.icu.text.BreakIterator;\r
import com.ibm.icu.text.DictionaryBasedBreakIterator;\r
import com.ibm.icu.text.RuleBasedBreakIterator;\r
\r
// TODO: {dlf} this test currently doesn't test anything!\r
// You'll notice that the resource that uses the dictionary isn't even on the resource path,\r
// so the dictionary never gets used.  Good thing, too, because it would throw a security\r
// exception if run with a security manager.  Not that it would matter, the dictionary \r
// resource isn't even in the icu source tree!\r
// In order to fix this:\r
// 1) make sure english.dict matches the current dictionary format required by dbbi\r
// 2) make sure english.dict gets included in icu4jtests.jar\r
// 3) have this test use getResourceAsStream to get a stream on the dictionary, and\r
//    directly instantiate a DictionaryBasedBreakIterator.  It can use the rules from\r
//    the appropriate section of ResourceBundle_en_US_TEST.  I'd suggest just copying\r
//    the rules into this file.\r
// 4) change the test text by inserting '|' at word breaks, and '||' at line breaks.  \r
// 5) process this text to a) create tables of break indices, and b) clean up the test\r
//    for the break iterator to work on\r
// \r
// This would NOT test the ability to load dictionary-based break iterators through our\r
// normal resource mechanism.  One could install such a break iterator and its\r
// resources into the icu4j jar, and it would work, but there's no way to register entire\r
// resources from outside yet.  Even if there were, the access restrictions are a bit\r
// difficult to manage, if one wanted to register a break iterator whose code and data\r
// resides outside the icu4j jar.  Since the code to instantiate would be going through \r
// two protection domains, each domain would have to allow access to the data-- but \r
// icu4j's domain wouldn't know about ours.  So we could instantiate before registering\r
// the break iterator, but this would mean we'd have to fully initialize the dictionary(s)\r
// at instantiation time, rather than let this be deferred until they are actually needed.\r
//\r
// I've done items 2 and 3 above.  Unfortunately, since I haven't done item 1, the\r
// dictionary builder crashes.  So for now I'm disabling this test.  This is not\r
// that important, since we have a thai dictionary that we do test thoroughly.\r
//\r
\r
public class SimpleBITest extends TestFmwk{\r
    public static final String testText =\r
//        "The rain in Spain stays mainly on the plain.  The plains in Spain are mainly pained with rain.";\r
//"one-two now--  Hah!  You owe me exactly $1,345.67...  Pay up, huh?  By the way, why don't I send you my re\u0301sume\u0301?  This is a line\r\nbreak.";\r
//"nowisthetimeforallgoodmen...  tocometothehelpoftheircountry";\r
"When, in the course of human events, it becomes necessary for one people to dissolve the political bonds which have "\r
//"When,inthecourseofhumanevents,itbecomesnecessaryforonepeopletodissolvethepoliticalbondswhichhave"\r
+ "connectedthemwithanother,andtoassumeamongthepowersoftheearth,theseparateandequalstationtowhichthelaws"\r
+ "ofnatureandofnature'sGodentitlethem,adecentrespecttotheopinionsofmankindrequiresthattheyshoulddeclarethe"\r
+ "causeswhichimpelthemtotheseparation\n"\r
+ "Weholdthesetruthstobeself-evident,thatallmenarecreatedequal,thattheyareendowedbytheirCreatorwithcertain"\r
+ "unalienablerights,thatamongthesearelife,libertyandthepursuitofhappiness.Thattosecuretheserights,governmentsare"\r
+ "institutedamongmen,derivingtheirjustpowersfromtheconsentofthegoverned.Thatwheneveranyformofgovernment"\r
+ "becomesdestructivetotheseends,itistherightofthepeopletoalterortoabolishit,andtoinstitutenewgovernment,laying"\r
+ "itsfoundationonsuchprinciplesandorganizingitspowersinsuchform,astothemshallseemmostlikelytoeffecttheirsafety"\r
+ "andhappiness.Prudence,indeed,willdictatethatgovernmentslongestablishedshouldnotbechangedforlightandtransient"\r
+ "causes;andaccordinglyallexperiencehathshownthatmankindaremoredisposedtosuffer,whileevilsaresufferable,than"\r
+ "torightthemselvesbyabolishingtheformstowhichtheyareaccustomed.Butwhenalongtrainofabusesandusurpations,"\r
+ "pursuinginvariablythesameobjectevincesadesigntoreducethemunderabsolutedespotism,itistheirright,itistheirduty,"\r
+ "tothrowoffsuchgovernment,andtoprovidenewguardsfortheirfuturesecurity.--Suchhasbeenthepatientsufferanceof"\r
+ "thesecolonies;andsuchisnowthenecessitywhichconstrainsthemtoaltertheirformersystemsofgovernment.Thehistory"\r
+ "ofthepresentKingofGreatBritainisahistoryofrepeatedinjuriesandusurpations,allhavingindirectobjectthe"\r
+ "establishmentofanabsolutetyrannyoverthesestates.Toprovethis,letfactsbesubmittedtoacandidworld.\n"\r
+ "Hehasrefusedhisassenttolaws,themostwholesomeandnecessaryforthepublicgood.\n"\r
+ "Hehasforbiddenhisgovernorstopasslawsofimmediateandpressingimportance,unlesssuspendedintheiroperationtill"\r
+ "hisassentshouldbeobtained;andwhensosuspended,hehasutterlyneglectedtoattendtothem.\n"\r
+ "Hehasrefusedtopassotherlawsfortheaccommodationoflargedistrictsofpeople,unlessthosepeoplewouldrelinquish"\r
+ "therightofrepresentationinthelegislature,arightinestimabletothemandformidabletotyrantsonly.\n"\r
+ "Hehascalledtogetherlegislativebodiesatplacesunusual,uncomfortable,anddistantfromthedepositoryoftheirpublic"\r
+ "records,forthesolepurposeoffatiguingthemintocompliancewithhismeasures.\n"\r
+ "Hehasdissolvedrepresentativehousesrepeatedly,foropposingwithmanlyfirmnesshisinvasionsontherightsofthepeople.\n"\r
+ "Hehasrefusedforalongtime,aftersuchdissolutions,tocauseotherstobeelected;wherebythelegislativepowers,"\r
+ "incapableofannihilation,havereturnedtothepeopleatlargefortheirexercise;thestateremaininginthemeantimeexposed"\r
+ "toallthedangersofinvasionfromwithout,andconvulsionswithin.\n"\r
+ "Hehasendeavoredtopreventthepopulationofthesestates;forthatpurposeobstructingthelawsfornaturalizationof"\r
+ "foreigners;refusingtopassotherstoencouragetheirmigrationhither,andraisingtheconditionsofnewappropriationsof"\r
+ "lands.\n"\r
+ "Hehasobstructedtheadministrationofjustice,byrefusinghisassenttolawsforestablishingjudiciarypowers.\n"\r
+ "Hehasmadejudgesdependentonhiswillalone,forthetenureoftheiroffices,andtheamountandpaymentoftheirsalaries.\n"\r
+ "Hehaserectedamultitudeofnewoffices,andsenthitherswarmsofofficerstoharassourpeople,andeatouttheir"\r
+ "substance.\n"\r
+ "Hehaskeptamongus,intimesofpeace,standingarmieswithouttheconsentofourlegislature.\n"\r
+ "Hehasaffectedtorenderthemilitaryindependentofandsuperiortocivilpower.\n"\r
+ "Hehascombinedwithotherstosubjectustoajurisdictionforeigntoourconstitution,andunacknowledgedbyourlaws;"\r
+ "givinghisassenttotheiractsofpretendedlegislation:\n"\r
+ "Forquarteringlargebodiesofarmedtroopsamongus:\n"\r
+ "Forprotectingthem,bymocktrial,frompunishmentforanymurderswhichtheyshouldcommitontheinhabitantsofthese"\r
+ "states:\n"\r
+ "Forcuttingoffourtradewithallpartsoftheworld:\n"\r
+ "Forimposingtaxesonuswithoutourconsent:\n"\r
+ "Fordeprivingusinmanycases,ofthebenefitsoftrialbyjury:\n"\r
+ "Fortransportingusbeyondseastobetriedforpretendedoffenses:\n"\r
+ "ForabolishingthefreesystemofEnglishlawsinaneighboringprovince,establishingthereinanarbitrarygovernment,and"\r
+ "enlargingitsboundariessoastorenderitatonceanexampleandfitinstrumentforintroducingthesameabsoluteruleinthese"\r
+ "colonies:\n"\r
+ "Fortakingawayourcharters,abolishingourmostvaluablelaws,andalteringfundamentallytheformsofourgovernments:\n"\r
+ "Forsuspendingourownlegislatures,anddeclaringthemselvesinvestedwithpowertolegislateforusinallcaseswhatsoever.\n"\r
+ "Hehasabdicatedgovernmenthere,bydeclaringusoutofhisprotectionandwagingwaragainstus.\n"\r
+ "Hehasplunderedourseas,ravagedourcoasts,burnedourtowns,anddestroyedthelivesofourpeople.\n"\r
+ "Heisatthistimetransportinglargearmiesofforeignmercenariestocompletetheworksofdeath,desolationandtyranny,"\r
+ "alreadybegunwithcircumstancesofcrueltyandperfidyscarcelyparalleledinthemostbarbarousages,andtotalyunworth"\r
+ "theheadofacivilizednation.\n"\r
+ "Hehasconstrainedourfellowcitizenstakencaptiveonthehighseastobeararmsagainsttheircountry,tobecomethe"\r
+ "executionersoftheirfriendsandbrethren,ortofallthemselvesbytheirhands.\n"\r
+ "Hehasexciteddomesticinsurrectionsamongstus,andhasendeavoredtobringontheinhabitantsofourfrontiers,the"\r
+ "mercilessIndiansavages,whoseknownruleofwarfare,isundistinguisheddestructionofallages,sexesandconditions.\n"\r
+ "Ineverystageoftheseoppressionswehavepetitionedforredressinthemosthumbleterms:ourrepeatedpetitionshave"\r
+ "beenansweredonlybyrepeatedinjury.Aprince,whosecharacteristhusmarkedbyeveryactwhichmaydefineatyrant,is"\r
+ "unfittobetherulerofafreepeople.\n"\r
+ "NorhavewebeenwantinginattentiontoourBritishbrethren.Wehavewarnedthemfromtimetotimeofattemptsbytheir"\r
+ "legislaturetoextendanunwarrantablejurisdictionoverus.Wehaveremindedthemofthecircumstancesofouremigration"\r
+ "andsettlementhere.Wehaveappealedtotheirnativejusticeandmagnanimity,andwehaveconjuredthembythetiesofour"\r
+ "commonkindredtodisavowtheseusurpations,which,wouldinevitablyinterruptourconnectionsandcorrespondence.We"\r
+ "must,therefore,acquiesceinthenecessity,whichdenouncesourseparation,andholdthem,asweholdtherestofmankind,"\r
+ "enemiesinwar,inpeacefriends.\n"\r
+ "We,therefore,therepresentativesoftheUnitedStatesofAmerica,inGeneralCongress,assembled,appealingtothe"\r
+ "SupremeJudgeoftheworldfortherectitudeofourintentions,do,inthename,andbytheauthorityofthegoodpeopleof"\r
+ "thesecolonies,solemnlypublishanddeclare,thattheseunitedcoloniesare,andofrightoughttobefreeandindependent"\r
+ "states;thattheyareabsolvedfromallallegiancetotheBritishCrown,andthatallpoliticalconnectionbetweenthemandthe"\r
+ "stateofGreatBritain,isandoughttobetotallydissolved;andthatasfreeandindependentstates,theyhavefullpowerto"\r
+ "leveywar,concludepeace,contractalliances,establishcommerce,andtodoallotheractsandthingswhichindependent"\r
+ "statesmayofrightdo.Andforthesupportofthisdeclaration,withafirmrelianceontheprotectionofDivineProvidence,we"\r
+ "mutuallypledgetoeachotherourlives,ourfortunesandoursacredhonor.\n";\r
\r
    public static void main(String[] args) throws Exception {\r
        new SimpleBITest().run(args);\r
    }\r
    \r
    protected boolean validate() {\r
        // TODO: remove when english.dict gets fixed\r
        return false;\r
    }\r
\r
    private BreakIterator createTestIterator(int kind) {\r
        final String bname = "com.ibm.icu.dev.test.rbbi.BreakIteratorRules_en_US_TEST";\r
\r
        BreakIterator iter = null;\r
\r
        ListResourceBundle bundle = null;\r
        try {\r
            Class cls = Class.forName(bname);\r
            bundle = (ListResourceBundle)cls.newInstance();\r
        }\r
        catch (Exception e) {\r
            errln("could not create bundle: " + bname + "exception: " + e.getMessage());\r
            return null;\r
        }\r
        \r
        final String[] kindNames = {\r
            "Character", "Word", "Line", "Sentence"\r
        };\r
        String rulesName = kindNames[kind] + "BreakRules";\r
        String dictionaryName = kindNames[kind] + "BreakDictionary";\r
        \r
        String[] classNames = bundle.getStringArray("BreakIteratorClasses");\r
        String rules = bundle.getString(rulesName);\r
        if (classNames[kind].equals("RuleBasedBreakIterator")) {\r
            iter = new RuleBasedBreakIterator(rules);\r
        }\r
        else if (classNames[kind].equals("DictionaryBasedBreakIterator")) {\r
            try {\r
                String dictionaryPath = bundle.getString(dictionaryName);\r
                InputStream dictionary = bundle.getClass().getResourceAsStream(dictionaryPath);\r
                System.out.println("looking for " + dictionaryPath + " from " + bundle.getClass() + " returned " + dictionary);\r
                iter = new DictionaryBasedBreakIterator(rules, dictionary);\r
            }\r
            catch(IOException e) {\r
                e.printStackTrace();\r
                errln(e.getMessage());\r
                System.out.println(e); // debug\r
            }\r
            catch(MissingResourceException e) {\r
                errln(e.getMessage());\r
                System.out.println(e); // debug\r
            }\r
        }\r
        if (iter == null) {\r
            errln("could not create iterator");\r
        }\r
        \r
        return iter;\r
    }\r
    \r
    public void testWordBreak() throws Exception {\r
        BreakIterator wordBreak = createTestIterator(BreakIterator.KIND_WORD);\r
        int breaks = doTest(wordBreak);\r
        logln(String.valueOf(breaks));\r
    }\r
\r
    public void testLineBreak() throws Exception {\r
        BreakIterator lineBreak = createTestIterator(BreakIterator.KIND_LINE);\r
        int breaks = doTest(lineBreak);\r
        logln(String.valueOf(breaks));\r
    }\r
\r
    public void testSentenceBreak() throws Exception {\r
        BreakIterator sentenceBreak = createTestIterator(BreakIterator.KIND_SENTENCE);\r
        int breaks = doTest(sentenceBreak);\r
        logln(String.valueOf(breaks));\r
    }\r
\r
    private int doTest(BreakIterator bi) {\r
        // forward\r
        bi.setText(testText);\r
        int p = bi.first();\r
        int lastP = p;\r
        String fragment;\r
        int breaks = 0;\r
        logln("Forward...");\r
        while (p != BreakIterator.DONE) {\r
            p = bi.next();\r
            if (p != BreakIterator.DONE) {\r
                fragment = testText.substring(lastP, p);\r
            } else {\r
                fragment = testText.substring(lastP);\r
            }\r
            debugPrintln(": >" + fragment + "<");\r
            ++breaks;\r
            lastP = p;\r
        }\r
        return breaks;\r
    }\r
\r
    private void debugPrintln(String s) {\r
        final String zeros = "0000";\r
        String temp;\r
        StringBuffer out = new StringBuffer();\r
        for (int i = 0; i < s.length(); i++) {\r
            char c = s.charAt(i);\r
            if (c >= ' ' && c < '\u007f')\r
                out.append(c);\r
            else {\r
                out.append("\\u");\r
                temp = Integer.toHexString((int)c);\r
                out.append(zeros.substring(0, 4 - temp.length()));\r
                out.append(temp);\r
            }\r
        }\r
        logln(out.toString());\r
    }\r
\r
/*    private void debugPrintln2(String s) {\r
        StringBuffer out = new StringBuffer();\r
        for (int i = 0; i < s.length(); i++) {\r
            char c = s.charAt(i);\r
            if (c >= '\u0100')\r
                out.append("<" + ((int)c - 0x100) + ">");\r
            else\r
                out.append(c);\r
        }\r
        logln(out.toString());\r
    }*/\r
}\r
\r