/* ******************************************************************************* * Copyright (C) 2005-2010 International Business Machines Corporation and * * others. All Rights Reserved. * ******************************************************************************* */ package com.ibm.icu.text; import java.io.ByteArrayInputStream; import java.io.ByteArrayOutputStream; import java.io.IOException; import java.io.InputStream; import java.io.OutputStream; import java.text.CharacterIterator; import com.ibm.icu.impl.Assert; import com.ibm.icu.impl.ICUDebug; /** * Rule Based Break Iterator * This is a port of the C++ class RuleBasedBreakIterator from ICU4C. * * @stable ICU 2.0 */ public class RuleBasedBreakIterator extends BreakIterator { //======================================================================= // Constructors & Factories //======================================================================= /** * @internal * @deprecated This API is ICU internal only. */ public RuleBasedBreakIterator() { } /** * Create a break iterator from a precompiled set of rules. * @internal * @deprecated This API is ICU internal only. */ public static RuleBasedBreakIterator getInstanceFromCompiledRules(InputStream is) throws IOException { RuleBasedBreakIterator This = new RuleBasedBreakIterator(); This.fRData = RBBIDataWrapper.get(is); return This; } /*private RuleBasedBreakIterator(RuleBasedBreakIterator other) { // TODO: check types. fRData = other.fRData; if (fText != null) { fText = (CharacterIterator)(other.fText.clone()); } }*/ /** * Construct a RuleBasedBreakIterator from a set of rules supplied as a string. * @param rules The break rules to be used. * @stable ICU 2.2 */ public RuleBasedBreakIterator(String rules) { init(); try { ByteArrayOutputStream ruleOS = new ByteArrayOutputStream(); compileRules(rules, ruleOS); byte [] ruleBA = ruleOS.toByteArray(); InputStream ruleIS = new ByteArrayInputStream(ruleBA); fRData = RBBIDataWrapper.get(ruleIS); } catch (IOException e) { ///CLOVER:OFF // An IO exception can only arrive here if there is a bug in the RBBI Rule compiler, // causing bogus compiled rules to be produced, but with no compile error raised. RuntimeException rte = new RuntimeException("RuleBasedBreakIterator rule compilation internal error: " + e.getMessage()); throw rte; ///CLOVER:ON } } //======================================================================= // Boilerplate //======================================================================= /** * Clones this iterator. * @return A newly-constructed RuleBasedBreakIterator with the same * behavior as this one. * @stable ICU 2.0 */ public Object clone() { RuleBasedBreakIterator result = (RuleBasedBreakIterator)super.clone(); if (fText != null) { result.fText = (CharacterIterator)(fText.clone()); } return result; } /** * Returns true if both BreakIterators are of the same class, have the same * rules, and iterate over the same text. * @stable ICU 2.0 */ public boolean equals(Object that) { try { RuleBasedBreakIterator other = (RuleBasedBreakIterator) that; if (fRData != other.fRData && (fRData == null || other.fRData == null)) {System.out.println("GOT HERE"); return false; } if (fRData != null && other.fRData != null && (!fRData.fRuleSource.equals(other.fRData.fRuleSource))) { return false; } if (fText == null && other.fText == null) { return true; } if (fText == null || other.fText == null) { return false; } return fText.equals(other.fText); } catch(ClassCastException e) { return false; } } /** * Returns the description (rules) used to create this iterator. * (In ICU4C, the same function is RuleBasedBreakIterator::getRules()) * @stable ICU 2.0 */ public String toString() { String retStr = null; if (fRData != null) { retStr = fRData.fRuleSource; } return retStr; } /** * Compute a hashcode for this BreakIterator * @return A hash code * @stable ICU 2.0 */ public int hashCode() { return fRData.fRuleSource.hashCode(); } /** * Tag value for "words" that do not fit into any of other categories. * Includes spaces and most punctuation. * @draft ICU 3.0 * @provisional This is a draft API and might change in a future release of ICU. */ public static final int WORD_NONE = 0; /** * Upper bound for tags for uncategorized words. * @draft ICU 3.0 * @provisional This is a draft API and might change in a future release of ICU. */ public static final int WORD_NONE_LIMIT = 100; /** * Tag value for words that appear to be numbers, lower limit. * @draft ICU 3.0 * @provisional This is a draft API and might change in a future release of ICU. */ public static final int WORD_NUMBER = 100; /** * Tag value for words that appear to be numbers, upper limit. * @draft ICU 3.0 * @provisional This is a draft API and might change in a future release of ICU. */ public static final int WORD_NUMBER_LIMIT = 200; /** * Tag value for words that contain letters, excluding * hiragana, katakana or ideographic characters, lower limit. * @draft ICU 3.0 * @provisional This is a draft API and might change in a future release of ICU. */ public static final int WORD_LETTER = 200; /** * Tag value for words containing letters, upper limit * @draft ICU 3.0 * @provisional This is a draft API and might change in a future release of ICU. */ public static final int WORD_LETTER_LIMIT = 300; /** * Tag value for words containing kana characters, lower limit * @draft ICU 3.0 * @provisional This is a draft API and might change in a future release of ICU. */ public static final int WORD_KANA = 300; /** * Tag value for words containing kana characters, upper limit * @draft ICU 3.0 * @provisional This is a draft API and might change in a future release of ICU. */ public static final int WORD_KANA_LIMIT = 400; /** * Tag value for words containing ideographic characters, lower limit * @draft ICU 3.0 * @provisional This is a draft API and might change in a future release of ICU. */ public static final int WORD_IDEO = 400; /** * Tag value for words containing ideographic characters, upper limit * @draft ICU 3.0 * @provisional This is a draft API and might change in a future release of ICU. */ public static final int WORD_IDEO_LIMIT = 500; private static final int START_STATE = 1; // The state number of the starting state private static final int STOP_STATE = 0; // The state-transition value indicating "stop" // RBBIRunMode - the state machine runs an extra iteration at the beginning and end // of user text. A variable with this enum type keeps track of where we // are. The state machine only fetches user text input while in RUN mode. private static final int RBBI_START = 0; private static final int RBBI_RUN = 1; private static final int RBBI_END = 2; /* * The character iterator through which this BreakIterator accesses the text. */ private CharacterIterator fText = new java.text.StringCharacterIterator(""); /** * The rule data for this BreakIterator instance * @internal * @deprecated This API is ICU internal only. */ protected RBBIDataWrapper fRData; /* * Index of the Rule {tag} values for the most recent match. */ private int fLastRuleStatusIndex; /* * Rule tag value valid flag. * Some iterator operations don't intrinsically set the correct tag value. * This flag lets us lazily compute the value if we are ever asked for it. */ private boolean fLastStatusIndexValid; /** * Counter for the number of characters encountered with the "dictionary" * flag set. Normal RBBI iterators don't use it, although the code * for updating it is live. Dictionary Based break iterators (a subclass * of us) access this field directly. * @internal * @deprecated This API is ICU internal only. */ protected int fDictionaryCharCount; /** * Debugging flag. Trace operation of state machine when true. * @internal * @deprecated This API is ICU internal only. */ public static boolean fTrace; /* * ICU debug argument name for RBBI */ private static final String RBBI_DEBUG_ARG = "rbbi"; /** * Dump the contents of the state table and character classes for this break iterator. * For debugging only. * @internal * @deprecated This API is ICU internal only. */ public void dump() { this.fRData.dump(); } private static boolean debugInitDone = false; private void init() { fLastStatusIndexValid = true; fDictionaryCharCount = 0; if (debugInitDone == false) { fTrace = ICUDebug.enabled(RBBI_DEBUG_ARG) && ICUDebug.value(RBBI_DEBUG_ARG).indexOf("trace") >= 0; debugInitDone = true; } } private static void compileRules(String rules, OutputStream ruleBinary) throws IOException { RBBIRuleBuilder.compileRules(rules, ruleBinary); } //======================================================================= // BreakIterator overrides //======================================================================= /** * Sets the current iteration position to the beginning of the text. * (i.e., the CharacterIterator's starting offset). * @return The offset of the beginning of the text. * @stable ICU 2.0 */ public int first() { fLastRuleStatusIndex = 0; fLastStatusIndexValid = true; if (fText == null) { return BreakIterator.DONE; } fText.first(); return fText.getIndex(); } /** * Sets the current iteration position to the end of the text. * (i.e., the CharacterIterator's ending offset). * @return The text's past-the-end offset. * @stable ICU 2.0 */ public int last() { if (fText == null) { fLastRuleStatusIndex = 0; fLastStatusIndexValid = true; return BreakIterator.DONE; } // I'm not sure why, but t.last() returns the offset of the last character, // rather than the past-the-end offset // // (It's so a loop like for(p=it.last(); p!=DONE; p=it.previous()) ... // will work correctly.) fLastStatusIndexValid = false; int pos = fText.getEndIndex(); fText.setIndex(pos); return pos; } /** * Advances the iterator either forward or backward the specified number of steps. * Negative values move backward, and positive values move forward. This is * equivalent to repeatedly calling next() or previous(). * @param n The number of steps to move. The sign indicates the direction * (negative is backwards, and positive is forwards). * @return The character offset of the boundary position n boundaries away from * the current one. * @stable ICU 2.0 */ public int next(int n) { int result = current(); while (n > 0) { result = handleNext(); --n; } while (n < 0) { result = previous(); ++n; } return result; } /** * Advances the iterator to the next boundary position. * @return The position of the first boundary after this one. * @stable ICU 2.0 */ public int next() { return handleNext(); } /** * Moves the iterator backwards, to the last boundary preceding this one. * @return The position of the last boundary position preceding this one. * @stable ICU 2.0 */ public int previous() { // if we're already sitting at the beginning of the text, return DONE if (fText == null || current() == fText.getBeginIndex()) { fLastRuleStatusIndex = 0; fLastStatusIndexValid = true; return BreakIterator.DONE; } if (fRData.fSRTable != null || fRData.fSFTable != null) { return handlePrevious(fRData.fRTable); } // old rule syntax // set things up. handlePrevious() will back us up to some valid // break position before the current position (we back our internal // iterator up one step to prevent handlePrevious() from returning // the current position), but not necessarily the last one before // where we started int start = current(); CIPrevious32(fText); int lastResult = handlePrevious(fRData.fRTable); if (lastResult == BreakIterator.DONE) { lastResult = fText.getBeginIndex(); fText.setIndex(lastResult); } int result = lastResult; int lastTag = 0; boolean breakTagValid = false; // iterate forward from the known break position until we pass our // starting point. The last break position before the starting // point is our return value for (;;) { result = handleNext(); if (result == BreakIterator.DONE || result >= start) { break; } lastResult = result; lastTag = fLastRuleStatusIndex; breakTagValid = true; } // fLastBreakTag wants to have the value for section of text preceding // the result position that we are to return (in lastResult.) If // the backwards rules overshot and the above loop had to do two or more // handleNext()s to move up to the desired return position, we will have a valid // tag value. But, if handlePrevious() took us to exactly the correct result positon, // we wont have a tag value for that position, which is only set by handleNext(). // set the current iteration position to be the last break position // before where we started, and then return that value fText.setIndex(lastResult); fLastRuleStatusIndex = lastTag; // for use by getRuleStatus() fLastStatusIndexValid = breakTagValid; return lastResult; } /** * Sets the iterator to refer to the first boundary position following * the specified position. * @param offset The position from which to begin searching for a break position. * @return The position of the first break after the current position. * @stable ICU 2.0 */ public int following(int offset) { // if the offset passed in is already past the end of the text, // just return DONE; if it's before the beginning, return the // text's starting offset fLastRuleStatusIndex = 0; fLastStatusIndexValid = true; if (fText == null || offset >= fText.getEndIndex()) { last(); return next(); } else if (offset < fText.getBeginIndex()) { return first(); } // otherwise, set our internal iteration position (temporarily) // to the position passed in. If this is the _beginning_ position, // then we can just use next() to get our return value int result = 0; if (fRData.fSRTable != null) { // Safe Point Reverse rules exist. // This allows us to use the optimum algorithm. fText.setIndex(offset); // move forward one codepoint to prepare for moving back to a // safe point. // this handles offset being between a supplementary character CINext32(fText); // handlePrevious will move most of the time to < 1 boundary away handlePrevious(fRData.fSRTable); result = next(); while (result <= offset) { result = next(); } return result; } if (fRData.fSFTable != null) { // No Safe point reverse table, but there is a safe pt forward table. // fText.setIndex(offset); CIPrevious32(fText); // handle next will give result >= offset handleNext(fRData.fSFTable); // previous will give result 0 or 1 boundary away from offset, // most of the time // we have to int oldresult = previous(); while (oldresult > offset) { result = previous(); if (result <= offset) { return oldresult; } oldresult = result; } result = next(); if (result <= offset) { return next(); } return result; } // otherwise, we have to sync up first. Use handlePrevious() to back // us up to a known break position before the specified position (if // we can determine that the specified position is a break position, // we don't back up at all). This may or may not be the last break // position at or before our starting position. Advance forward // from here until we've passed the starting position. The position // we stop on will be the first break position after the specified one. // old rule syntax fText.setIndex(offset); if (offset == fText.getBeginIndex()) { return handleNext(); } result = previous(); while (result != BreakIterator.DONE && result <= offset) { result = next(); } return result; } /** * Sets the iterator to refer to the last boundary position before the * specified position. * @param offset The position to begin searching for a break from. * @return The position of the last boundary before the starting position. * @stable ICU 2.0 */ public int preceding(int offset) { // if the offset passed in is already past the end of the text, // just return DONE; if it's before the beginning, return the // text's starting offset if (fText == null || offset > fText.getEndIndex()) { // return BreakIterator::DONE; return last(); } else if (offset < fText.getBeginIndex()) { return first(); } // if we start by updating the current iteration position to the // position specified by the caller, we can just use previous() // to carry out this operation int result; if (fRData.fSFTable != null) { /// todo synwee // new rule syntax fText.setIndex(offset); // move backwards one codepoint to prepare for moving forwards to a // safe point. // this handles offset being between a supplementary character CIPrevious32(fText); handleNext(fRData.fSFTable); result = previous(); while (result >= offset) { result = previous(); } return result; } if (fRData.fSRTable != null) { // backup plan if forward safe table is not available fText.setIndex(offset); CINext32(fText); // handle previous will give result <= offset handlePrevious(fRData.fSRTable); // next will give result 0 or 1 boundary away from offset, // most of the time // we have to int oldresult = next(); while (oldresult < offset) { result = next(); if (result >= offset) { return oldresult; } oldresult = result; } result = previous(); if (result >= offset) { return previous(); } return result; } // old rule syntax fText.setIndex(offset); return previous(); } /** * Throw IllegalArgumentException unless begin <= offset < end. * @stable ICU 2.0 */ protected static final void checkOffset(int offset, CharacterIterator text) { if (offset < text.getBeginIndex() || offset > text.getEndIndex()) { throw new IllegalArgumentException("offset out of bounds"); } } /** * Returns true if the specfied position is a boundary position. As a side * effect, leaves the iterator pointing to the first boundary position at * or after "offset". * @param offset the offset to check. * @return True if "offset" is a boundary position. * @stable ICU 2.0 */ public boolean isBoundary(int offset) { checkOffset(offset, fText); // the beginning index of the iterator is always a boundary position by definition if (offset == fText.getBeginIndex()) { first(); // For side effects on current position, tag values. return true; } if (offset == fText.getEndIndex()) { last(); // For side effects on current position, tag values. return true; } // otherwise, we can use following() on the position before the specified // one and return true if the position we get back is the one the user // specified // return following(offset - 1) == offset; // TODO: check whether it is safe to revert to the simpler offset-1 code // The safe rules may take care of unpaired surrogates ok. fText.setIndex(offset); CIPrevious32(fText); int pos = fText.getIndex(); boolean result = following(pos) == offset; return result; } /** * Returns the current iteration position. * @return The current iteration position. * @stable ICU 2.0 */ public int current() { return (fText != null) ? fText.getIndex() : BreakIterator.DONE; } private void makeRuleStatusValid() { if (fLastStatusIndexValid == false) { // No cached status is available. if (fText == null || current() == fText.getBeginIndex()) { // At start of text, or there is no text. Status is always zero. fLastRuleStatusIndex = 0; fLastStatusIndexValid = true; } else { // Not at start of text. Find status the tedious way. int pa = current(); previous(); int pb = next(); Assert.assrt (pa == pb); } Assert.assrt(fLastStatusIndexValid == true); Assert.assrt(fLastRuleStatusIndex >= 0 && fLastRuleStatusIndex < fRData.fStatusTable.length); } } /** * Return the status tag from the break rule that determined the most recently * returned break position. The values appear in the rule source * within brackets, {123}, for example. For rules that do not specify a * status, a default value of 0 is returned. If more than one rule applies, * the numerically largest of the possible status values is returned. *
* Of the standard types of ICU break iterators, only the word break
* iterator provides status values. The values are defined in
* class RuleBasedBreakIterator, and allow distinguishing between words
* that contain alphabetic letters, "words" that appear to be numbers,
* punctuation and spaces, words containing ideographic characters, and
* more. Call
* @return the status from the break rule that determined the most recently
* returned break position.
*
* @draft ICU 3.0
* @provisional This is a draft API and might change in a future release of ICU.
*/
public int getRuleStatus() {
makeRuleStatusValid();
// Status records have this form:
// Count N <-- fLastRuleStatusIndex points here.
// Status val 0
// Status val 1
// ...
// Status val N-1 <-- the value we need to return
// The status values are sorted in ascending order.
// This function returns the last (largest) of the array of status values.
int idx = fLastRuleStatusIndex + fRData.fStatusTable[fLastRuleStatusIndex];
int tagVal = fRData.fStatusTable[idx];
return tagVal;
}
/**
* Get the status (tag) values from the break rule(s) that determined the most
* recently returned break position. The values appear in the rule source
* within brackets, {123}, for example. The default status value for rules
* that do not explicitly provide one is zero.
*
* The status values used by the standard ICU break rules are defined
* as public constants in class RuleBasedBreakIterator.
*
* If the size of the output array is insufficient to hold the data,
* the output will be truncated to the available length. No exception
* will be thrown.
*
* @param fillInArray an array to be filled in with the status values.
* @return The number of rule status values from rules that determined
* the most recent boundary returned by the break iterator.
* In the event that the array is too small, the return value
* is the total number of status values that were available,
* not the reduced number that were actually returned.
* @draft ICU 3.0
* @provisional This is a draft API and might change in a future release of ICU.
*/
public int getRuleStatusVec(int[] fillInArray) {
makeRuleStatusValid();
int numStatusVals = fRData.fStatusTable[fLastRuleStatusIndex];
if (fillInArray != null) {
int numToCopy = Math.min(numStatusVals, fillInArray.length);
for (int i=0; igetRuleStatus
after obtaining a boundary
* position from next()
,
previous()
, or
* any other break iterator functions that returns a boundary position.
*