go

[Dictionary.git] / jars / icu4j-4_4_2-src / tools / misc / src / com / ibm / icu / dev / tool / translit / UnicodeSetCloseOver.java

/*\r
**********************************************************************\r
* Copyright (c) 2003-2010, International Business Machines\r
* Corporation and others.  All Rights Reserved.\r
**********************************************************************\r
* Author: Alan Liu\r
* Created: February 11 2003\r
* Since: ICU 2.6\r
**********************************************************************\r
*/\r
package com.ibm.icu.dev.tool.translit;\r
\r
import java.io.FileOutputStream;\r
import java.io.IOException;\r
import java.io.PrintStream;\r
import java.util.Collections;\r
import java.util.Comparator;\r
import java.util.HashMap;\r
import java.util.Iterator;\r
import java.util.Locale;\r
import java.util.Map;\r
import java.util.Set;\r
import java.util.TreeSet;\r
import java.util.Vector;\r
\r
import com.ibm.icu.impl.Utility;\r
import com.ibm.icu.lang.UCharacter;\r
import com.ibm.icu.text.BreakIterator;\r
import com.ibm.icu.text.UTF16;\r
import com.ibm.icu.text.UnicodeSet;\r
\r
/**\r
 * This class produces the data tables used by the closeOver() method\r
 * of UnicodeSet.\r
 *\r
 * Whenever the Unicode database changes, this tool must be re-run\r
 * (AFTER the data file(s) underlying ICU4J are udpated).\r
 *\r
 * The output of this tool should then be pasted into the appropriate\r
 * files:\r
 *\r
 * ICU4J: com.ibm.icu.text.UnicodeSet.java\r
 * ICU4C: /icu/source/common/uniset.cpp\r
 */\r
class UnicodeSetCloseOver {\r
\r
    // Our output files\r
    static final String JAVA_OUT          = "to_UnicodeSet.java";\r
    static final String JAVA_CHARPROP_OUT = "to_UCharacterProperty.java";\r
    static final String C_SET_OUT         = "to_uniset.cpp";\r
    static final String C_UCHAR_OUT       = "to_uchar.c";\r
\r
    // Source code "do not edit" warning\r
    static final String WARNING = "MACHINE-GENERATED; Unicode version " +\r
        UCharacter.getUnicodeVersion() +\r
        "; DO NOT EDIT; See " +\r
        UnicodeSetCloseOver.class.getName();\r
\r
    // Case folding options flag.  This must correspond to the options\r
    // used in UnicodeSet.closeOver() in Java and C++.\r
    static final boolean DEFAULT_CASE_MAP = true; // false for Turkish\r
\r
    public static void main(String[] args) throws IOException {\r
        System.out.println("This tool will generate several output files.  Each is named according");\r
        System.out.println("the target file.  For example, the contents of to_UnicodeSet.java should");\r
        System.out.println("be pasted into UnicodeSet.java.");\r
        System.out.println();\r
\r
        generateCaseData();\r
    }\r
\r
    /**\r
     * Create a map of String => Set.  The String in this case is a\r
     * folded string for which\r
     * UCharacter.foldCase(folded. DEFAULT_CASE_MAP).equals(folded).\r
     * The Set contains all single-character strings x for which\r
     * UCharacter.foldCase(x, DEFAULT_CASE_MAP).equals(folded), as\r
     * well as folded itself.\r
     */\r
    static Map createCaseFoldEquivalencyClasses() {\r
        Map equivClasses = new HashMap();\r
        for (int i = 0; i <= 0x10FFFF; ++i) {\r
            int cat = UCharacter.getType(i);\r
            if (cat == Character.UNASSIGNED || cat == Character.PRIVATE_USE)\r
                continue;\r
\r
            String cp = UTF16.valueOf(i);\r
            String folded = UCharacter.foldCase(cp, DEFAULT_CASE_MAP);\r
            if (folded.equals(cp)) continue;\r
\r
            // At this point, have different case folding.  Add\r
            // the code point and its folded equivalent into the\r
            // equivalency class.\r
            TreeSet s = (TreeSet) equivClasses.get(folded);\r
            if (s == null) {\r
                s = new TreeSet();\r
                s.add(folded); // add the case fold result itself\r
                equivClasses.put(folded, s);\r
            }\r
            s.add(cp);\r
        }\r
        return equivClasses;\r
    }\r
\r
    /**\r
     * Analyze the case fold equivalency classes.  Break them into two\r
     * groups: 'pairs', and 'nonpairs'.  Create a tally of the length\r
     * configurations of the nonpairs.\r
     *\r
     * Length configurations of equivalency classes, as of Unicode\r
     * 3.2.  Most of the classes (83%) have two single codepoints.\r
     * Here "112:28" means there are 28 equivalency classes with 2\r
     * single codepoints and one string of length 2.\r
     *\r
     * 11:656\r
     * 111:16\r
     * 1111:3\r
     * 112:28\r
     * 113:2\r
     * 12:31\r
     * 13:12\r
     * 22:38\r
     *\r
     * Note: This method does not count the frequencies of the\r
     * different length configurations (as shown above after ':'); it\r
     * merely records which configurations occur.\r
     *\r
     * @param pairs Accumulate equivalency classes that consist of\r
     * exactly two codepoints here.  This is 83+% of the classes.\r
     * E.g., {"a", "A"}.\r
     * @param nonpairs Accumulate other equivalency classes here, as\r
     * lists of strings.  E,g, {"st", "\uFB05", "\uFB06"}.\r
     * @param lengths Accumulate a list of unique length structures,\r
     * not including pairs.  Each length structure is represented by a\r
     * string of digits.  The digit string "12" means the equivalency\r
     * class contains a single code point and a string of length 2.\r
     * Typical contents of 'lengths': { "111", "1111", "112",\r
     * "113", "12", "13", "22" }.  Note the absence of "11".\r
     */\r
    static void analyzeCaseData(Map equivClasses,\r
                                StringBuffer pairs,\r
                                Vector nonpairs,\r
                                Vector lengths) {\r
        Iterator i = new TreeSet(equivClasses.keySet()).iterator();\r
        StringBuffer buf = new StringBuffer();\r
        while (i.hasNext()) {\r
            Object key = i.next();\r
            Vector v = new Vector((Set) equivClasses.get(key));\r
            if (v.size() == 2) {\r
                String a = (String) v.elementAt(0);\r
                String b = (String) v.elementAt(1);\r
                if (a.length() == 1 && b.length() == 1) {\r
                    pairs.append(a).append(b);\r
                    continue;\r
                    // Note that pairs are included in 'lengths'\r
                }\r
            }\r
            String[] a = new String[v.size()];\r
            v.toArray(a);\r
            nonpairs.add(a);\r
            //int singleCount = 0;\r
            //int stringCount = 0;\r
            // Make a string of the lengths, e.g., "111" means 3\r
            // single code points; "13" means a single code point\r
            // and a string of length 3.\r
            v.clear();\r
            for (int j=0; j<a.length; ++j) {\r
                v.add(new Integer(a[j].length()));\r
            }\r
            Collections.sort(v);\r
            buf.setLength(0);\r
            for (int j=0; j<v.size(); ++j) {\r
                buf.append(String.valueOf(v.elementAt(j)));\r
            }\r
            if (!lengths.contains(buf.toString())) {\r
                lengths.add(buf.toString());\r
            }\r
        }\r
    }\r
\r
    static void generateCaseData() throws IOException {\r
\r
        Map equivClasses = createCaseFoldEquivalencyClasses();\r
\r
        // Accumulate equivalency classes that consist of exactly\r
        // two codepoints here.  This is 83+% of the classes.\r
        // E.g., {"a", "A"}.\r
        StringBuffer pairs = new StringBuffer();\r
\r
        // Accumulate other equivalency classes here, as lists\r
        // of strings.  E,g, {"st", "\uFB05", "\uFB06"}.\r
        Vector nonpairs = new Vector(); // contains String[]\r
        Vector lengths = new Vector(); // "111", "12", "22", etc.\r
\r
        analyzeCaseData(equivClasses, pairs, nonpairs, lengths);\r
\r
        //-------------------------------------------------------------\r
        // Emit Java source\r
        PrintStream out = new PrintStream(new FileOutputStream(JAVA_OUT));\r
        System.out.println("Writing " + JAVA_OUT);\r
\r
        out.println("    // " + WARNING);\r
        out.println("    private static final String CASE_PAIRS =");\r
        out.println(Utility.formatForSource(pairs.toString()) + ";");\r
        out.println();\r
        out.println("    // " + WARNING);\r
        out.println("    private static final String[][] CASE_NONPAIRS = {");\r
        for (int j=0; j<nonpairs.size(); ++j) {\r
            String[] a = (String[]) nonpairs.elementAt(j);\r
            out.print("        {");\r
            for (int k=0; k<a.length; ++k) {\r
                if (k != 0) out.print(", ");\r
                out.print(Utility.format1ForSource(a[k]));\r
            }\r
            out.println("},");\r
        }\r
        out.println("    };");\r
\r
        //-------------------------------------------------------------\r
        // Emit C++ source\r
\r
        // In C++, the pairs are again emitted in an array, but this\r
        // array is the final representation form -- it will not be\r
        // reprocessed into a hash.  It will be binary searched by\r
        // looking at the even elements [0], [2], [4], etc., and\r
        // ignoring the odd elements.  The even elements must contain\r
        // the folded members of the pairs.  That is, in the pair\r
        // {'A', 'a'}, the even element must be 'a', not 'A'.  Then a\r
        // code point to be located is first folded ('Y' => 'y') then\r
        // it binary searched against [0]='A', [2]='B', etc.  When a\r
        // match is found at k, the pair is [k], [k+1].\r
\r
        out = new PrintStream(new FileOutputStream(C_SET_OUT));\r
        System.out.println("Writing " + C_SET_OUT);\r
\r
        // Sort the pairs.  They must be ordered by the folded element.\r
        // Store these as two-character strings, with charAt(0) being\r
        // the folded member of the pair.\r
        TreeSet sortPairs = new TreeSet(new Comparator() {\r
            public int compare(Object a, Object b) {\r
                return ((int) ((String) a).charAt(0)) -\r
                       ((int) ((String) b).charAt(0));\r
            }\r
            public boolean equals(Object obj) {\r
                return false;\r
            }\r
        });\r
        for (int i=0; i<pairs.length(); i+=2) {\r
            String a = String.valueOf(pairs.charAt(i));\r
            String b = String.valueOf(pairs.charAt(i+1));\r
            String folded = UCharacter.foldCase(a, DEFAULT_CASE_MAP);\r
            if (a.equals(folded)) {\r
                sortPairs.add(a + b);\r
            } else {\r
                sortPairs.add(b + a);\r
            }\r
        }\r
\r
        // Emit the pairs\r
        out.println("// " + WARNING);\r
        out.println("static const UChar CASE_PAIRS[] = {");\r
        Iterator it = sortPairs.iterator();\r
        while (it.hasNext()) {\r
            out.print("    ");\r
            int n = 0;\r
            while (n++ < 5 && it.hasNext()) {\r
                String s = (String) it.next();\r
                //out.print((int) s.charAt(0) + "," +\r
                //                 (int) s.charAt(1) + ",");\r
                out.print("0x" + Utility.hex(s.charAt(0)) + ",0x" +\r
                                 Utility.hex(s.charAt(1)) + ",");\r
            }\r
            out.println();\r
        }\r
        out.println("};");\r
        out.println();\r
\r
        // The non-pairs are encoded in the following way.  All the\r
        // single codepoints in each class are grouped together\r
        // followed by a zero.  Then each multi-character string is\r
        // added, followed by a zero.  Finally, another zero is added.\r
        // Some examples:\r
        //  {"iQ", "R"}           =>  [ 'R', 0, 'i', 'Q', 0, 0 ]\r
        //  {"S", "D", "F", "G"}  =>  [ 'S', 'D', 'F', 'G', 0, 0 ]\r
        //  {"jW", "jY"}          =>  [ 0, 'j', 'W', 0, 'j', 'Y', 0, 0 ]\r
        // The end-result is a short, flat array of UChar values that\r
        // can be used to initialize a UChar[] array in C.\r
        \r
        int maxLen = 0; // Maximum encoded length of any class, including zeros\r
        out.println("// " + WARNING);\r
        out.println("static const CaseEquivClass CASE_NONPAIRS[] = {");\r
        for (int j=0; j<nonpairs.size(); ++j) {\r
            int len = 0;\r
            String[] a = (String[]) nonpairs.elementAt(j);\r
            out.print("    {");\r
            // Emit single code points\r
            for (int k=0; k<a.length; ++k) {\r
                if (a[k].length() != 1) continue;\r
                //out.print((int) a[k].charAt(0) + ",");\r
                out.print("0x"+Utility.hex(a[k].charAt(0)) + ",");\r
                ++len;\r
            }\r
            out.print("0,  "); // End of single code points\r
            ++len;\r
            // Emit multi-character strings\r
            for (int k=0; k<a.length; ++k) {\r
                if (a[k].length() == 1) continue;\r
                for (int m=0; m<a[k].length(); ++m) {\r
                    //out.print((int) a[k].charAt(m) + ",");\r
                    out.print("0x"+Utility.hex(a[k].charAt(m)) + ",");\r
                    ++len;\r
                }\r
                out.print("0, "); // End of string\r
                ++len;\r
            }\r
            out.println("0},"); // End of equivalency class\r
            ++len;\r
            if (len > maxLen) maxLen = len;\r
        }\r
        out.println("};");\r
\r
        // Make sure the CaseEquivClass data can fit.\r
        if (maxLen > 8) {\r
            throw new RuntimeException("Must adjust CaseEquivClass to accomodate " + maxLen + " UChars");\r
        }\r
\r
        // Also make sure that we can map into this array using a\r
        // CompactByteArray.  We could do this check above, but we\r
        // keep it here, adjacent to the maxLen check.  We use one\r
        // value (-1 == 255) to indicate "no value."\r
        if (nonpairs.size() > 255) {\r
            throw new RuntimeException("Too many CASE_NONPAIRS array elements to be indexed by a CompactByteArray");\r
        }\r
\r
        //-------------------------------------------------------------\r
        // Case-unique set:  All characters c for which closeOver(c)==c.\r
\r
        // UPDATE: Instead of using this, we're using the related\r
        // notion of Case_Sensitive.  See below.  Note that\r
        // Case_Sensitive != ^Case_Unique.\r
\r
        if (false) {\r
            UnicodeSet caseUnique = new UnicodeSet();\r
            for (int i = 0; i <= 0x10FFFF; ++i) {\r
                String cp = UTF16.valueOf(i);\r
                if (equivClasses.get(UCharacter.foldCase(cp, DEFAULT_CASE_MAP)) == null) {\r
                    caseUnique.add(i);\r
                }\r
            }\r
            // out.println("caseUnique = " + caseUnique.toPattern(true));\r
        }\r
\r
        UnicodeSet caseSensitive = getCaseSensitive();\r
        //System.out.println("caseSensitive = " + caseSensitive.toPattern(true));\r
\r
        // Now for C, emit an array of ranges\r
        out = new PrintStream(new FileOutputStream(C_UCHAR_OUT));\r
        System.out.println("Writing " + C_UCHAR_OUT);\r
\r
        out.println("/* " + WARNING + " */");\r
        emitUCharRangesArray(out, caseSensitive, "CASE_SENSITIVE_RANGES");\r
\r
        // For Java, emit a string with the ranges (each pair of chars\r
        // in the string is a range).\r
        out = new PrintStream(new FileOutputStream(JAVA_CHARPROP_OUT));\r
        System.out.println("Writing " + JAVA_CHARPROP_OUT);\r
        out.println("    // " + WARNING);\r
        emitRangesString(out, caseSensitive, "CASE_SENSITIVE_RANGES");\r
    }\r
\r
    /**\r
     * Create the set of case-sensitive characters.  These are characters\r
     * that participate in any case mapping operation as a source or\r
     * as a member of a target string.\r
     */\r
    static UnicodeSet getCaseSensitive() {\r
        UnicodeSet caseSensitive = new UnicodeSet();\r
        Locale loc = Locale.US;\r
        BreakIterator bi = BreakIterator.getTitleInstance(loc);\r
        for (int c = 0; c <= 0x10FFFF; ++c) {\r
            String cp = UTF16.valueOf(c);\r
            for (int j=0; j<4; ++j) {\r
                String s = null;\r
                switch (j) {\r
                case 0: s = UCharacter.toUpperCase(loc, cp); break;\r
                case 1: s = UCharacter.toLowerCase(loc, cp); break;\r
                case 2: s = UCharacter.toTitleCase(loc, cp, bi); break;\r
                case 3: s = UCharacter.foldCase(cp, DEFAULT_CASE_MAP); break;\r
                }\r
                if (!s.equals(cp)) {\r
                    int cc;\r
                    for (int k=0; k<s.length(); k+=UTF16.getCharCount(cc)) {\r
                        cc = UTF16.charAt(s, k);\r
                        caseSensitive.add(cc);\r
                    }\r
                    for (int k=0; k<cp.length(); k+=UTF16.getCharCount(cc)) {\r
                        cc = UTF16.charAt(cp, k);\r
                        caseSensitive.add(cc);\r
                    }\r
                }\r
            }\r
            // Also do the single-codepoint API.  This shouldn't add any\r
            // code points, but we do it for completeness.\r
            for (int j=0; j<4; ++j) {\r
                int d = 0;\r
                switch (j) {\r
                case 0: d = UCharacter.toUpperCase(c); break;\r
                case 1: d = UCharacter.toLowerCase(c); break;\r
                case 2: d = UCharacter.toTitleCase(c); break;\r
                case 3: d = UCharacter.foldCase(c, DEFAULT_CASE_MAP); break;\r
                }\r
                if (d != c) {\r
                    if (!caseSensitive.contains(c) ||\r
                        !caseSensitive.contains(d)) {\r
                        System.out.println("Warning: code point " + c +\r
                                           " => " + d + " created NEW MAPPING"+\r
                                           " for Case_Sensitive");\r
                    }\r
                    caseSensitive.add(c);\r
                    caseSensitive.add(d);\r
                }\r
            }\r
        }\r
        return caseSensitive;\r
    }\r
\r
    /**\r
     * Given a UnicodeSet, emit it as an array of UChar pairs.  Each\r
     * pair will be the start/end of a range.  Code points >= U+10000\r
     * will be represented as surrogate pairs.\r
     */\r
    static void emitUCharRangesArray(PrintStream out, UnicodeSet set, String id) {\r
        // Store the pairs in a StringBuffer.  This handles surrogate\r
        // representation.\r
        StringBuffer buf = new StringBuffer();\r
        for (int i=0; i<set.getRangeCount(); ++i) {\r
            UTF16.append(buf, set.getRangeStart(i));\r
            UTF16.append(buf, set.getRangeEnd(i));\r
        }\r
        // Emit the pairs\r
        out.println("static const UChar " + id + "[] = {");\r
        for (int i=0; i<buf.length(); ) {\r
            out.print("    ");\r
            for (int n=0; n++<10 && i<buf.length(); ++i) {\r
                out.print("0x" + Utility.hex(buf.charAt(i), 4) + ',');\r
            }\r
            out.println();\r
        }\r
        out.println("};");\r
        out.println("#define " + id + "_LENGTH (sizeof(" + id +\r
                    ")/sizeof(" + id + "[0]))");\r
    }\r
\r
    /**\r
     * Given a UnicodeSet, emit it as a Java string.  The most economical\r
     * format is not the pattern, but instead a pairs list, with each\r
     * range pair represented as two adjacent characters.\r
     */\r
    static void emitRangesString(PrintStream out, UnicodeSet set, String id) {\r
        // Store the pairs in a StringBuffer.  This handles surrogate\r
        // representation.\r
        StringBuffer buf = new StringBuffer();\r
        for (int i=0; i<set.getRangeCount(); ++i) {\r
            UTF16.append(buf, set.getRangeStart(i));\r
            UTF16.append(buf, set.getRangeEnd(i));\r
        }\r
        // Emit the pairs\r
        out.println("    private static final String " + id + " =");\r
        out.println(Utility.formatForSource(buf.toString()) + ";");\r
    }\r
}\r