2 //#if defined(FOUNDATION10) || defined(J2SE13)
\r
5 *******************************************************************************
\r
6 * Copyright (C) 2002-2009, International Business Machines Corporation and *
\r
7 * others. All Rights Reserved. *
\r
8 *******************************************************************************
\r
10 package com.ibm.icu.dev.test.util;
\r
12 import java.util.Random;
\r
14 //TODO integrate this into the test framework
\r
16 import com.ibm.icu.text.UnicodeSet;
\r
18 public class TestBNF {
\r
20 static final String[] testRules = {
\r
23 "$root = [ab]{3,};",
\r
25 "$root = [ab]{3,5};",
\r
34 "$root = [0-9$us];",
\r
36 "$root = a $foo b? 25% end 30% | $foo 50%;\r\n" +
\r
37 "$foo = c{1,5} 20%;",
\r
39 "$root = [a-z]{1,5}~;",
\r
41 "$root = [a-z]{5}~;",
\r
43 "$root = '\\' (u | U0010 | U000 $hex) $hex{4} ;\r\n" +
\r
44 "$hex = [0-9A-Fa-f];",
\r
47 static String unicodeSetBNF = "" +
\r
48 "$root = $leaf | '[' $s $root2 $s ']' ;\r\n" +
\r
49 "$root2 = $leaf | '[' $s $root3 $s ']' | ($root3 $s ($op $root3 $s){0,3}) ;\r\n" +
\r
50 "$root3 = $leaf | '[' $s $root4 $s ']' | ($root4 $s ($op $root4 $s){0,3}) ;\r\n" +
\r
51 "$root4 = $leaf | ($leaf $s ($op $leaf $s){0,3}) ;\r\n" +
\r
52 "$op = (('&' | '-') $s)? 70%;" +
\r
53 "$leaf = '[' $s $list $s ']' | $prop;\r\n" +
\r
54 "$list = ($char $s ('-' $s $char $s)? 30%){1,5} ;\r\n" +
\r
55 "$prop = '\\' (p | P) '{' $s $propName $s '}' | '[:' '^'? $s $propName $s ':]';\r\n" +
\r
56 "$needsQuote = [\\-\\][:whitespace:][:control:]] ;\r\n" +
\r
57 "$char = [[\\u0000-\\U00010FFFF]-$needsQuote] | $quoted ;\r\n" +
\r
58 "$quoted = '\\' ('u' | 'U0010' | 'U000' $hex) $hex{4} ;\r\n" +
\r
59 "$hex = [0-9A-Fa-f];\r\n" +
\r
60 "$s = ' '? 20%;\r\n" +
\r
61 "$propName = (whitespace | ws) | (uppercase | uc) | (lowercase | lc) | $category;\r\n" +
\r
62 "$category = ((general | gc) $s '=' $s)? $catvalue;\r\n" +
\r
63 "$catvalue = (C | Other | Cc | Control | Cf | Format | Cn | Unassigned | L | Letter);\r\n";
\r
65 public static void main (String[] args) {
\r
67 for (int i = 0; i < testRules.length; ++i) {
\r
68 testBNF(testRules[i], null, 20);
\r
71 testBNF(unicodeSetBNF, null, 20);
\r
75 static void testBNF(String rules, UnicodeSet chars, int count) {
\r
76 BNF bnf = new BNF(new Random(0), new Quoter.RuleQuoter())
\r
77 .addSet("$chars", chars)
\r
81 System.out.println("====================================");
\r
82 System.out.println("BNF");
\r
83 System.out.println(rules);
\r
84 System.out.println(bnf.getInternal());
\r
85 for (int i = 0; i < count; ++i) {
\r
86 System.out.println(i + ": " + bnf.next());
\r
91 public static testManual() {
\r
92 Pick p = Pick.maybe(75,Pick.unquoted("a"));
\r
94 p = Pick.or(new String[]{"", "a", "bb", "ccc"});
\r
96 p = Pick.repeat(3, 5, new int[]{20, 30, 20}, "a");
\r
98 p = Pick.codePoint("[a-ce]");
\r
100 p = Pick.codePoint("[a-ce]");
\r
101 testCodePoints(p);
\r
102 p = Pick.string(2, 8, p);
\r
105 p = Pick.or(new String[]{"", "a", "bb", "ccc"});
\r
106 p = Pick.and(p).and2(p).and2("&");
\r
107 testMatch(p, "abb&");
\r
108 testMatch(p, "bba");
\r
114 static void testMatch(Pick p, String source) {
\r
115 Pick.Position pp = new Pick.Position();
\r
116 boolean value = p.match(source, pp);
\r
117 System.out.println("Match: " + value + ", " + pp);
\r
120 static void testParser() {
\r
122 Pick.Target target = new Pick.Target();
\r
123 for (int i = 0; i < rules.length; ++i) {
\r
124 target.addRule(rules[i]);
\r
126 } catch (ParseException e) {
\r
127 // TODO Auto-generated catch block
\r
128 e.printStackTrace();
\r
133 static class Counts {
\r
136 counts = new int[max+1];
\r
138 void inc(int index) {
\r
142 System.out.println("Printing Counts");
\r
143 for (int i = 0; i < counts.length; ++i) {
\r
144 if (counts[i] == 0) continue;
\r
145 System.out.println(i + ": " + counts[i]);
\r
147 System.out.println();
\r
151 /* static final String[] rules = {
\r
153 "$noun = dog | house | government | wall | street | zebra;",
\r
154 "$adjective = red | glorious | simple | nasty | heavy | clean;",
\r
155 "$article = quickly | oddly | silently | boldly;",
\r
156 "$adjectivePhrase = ($adverb $s)? 50% $adjective* 0% 30% 20% 10%;",
\r
157 "$nounPhrase = $articles $s ($adjectivePhrase $s)? 30% $noun;",
\r
158 "$verb = goes | fishes | walks | sleeps;",
\r
159 "$tverb = carries | lifts | overturns | hits | jumps on;",
\r
160 "$copula = is 30% | seems 10%;",
\r
161 "$sentence1 = $nounPhrase $s $verb $s ($s $adverb)? 30%;",
\r
162 "$sentence2 = $nounPhrase $s $tverb $s $nounPhrase ($s $adverb)? 30%;",
\r
163 "$sentence3 = $nounPhrase $s $copula $s $adjectivePhrase;",
\r
164 "$conj = but | and | or;",
\r
165 "$sentence4 = $sentence1 | $sentence2 | $sentence3 20% | $sentence4 $conj $sentence4 20%;",
\r
166 "$sentence = $sentence4 '.';"};
\r
169 private static void testEnglish() {
\r
170 Pick s = Pick.unquoted(" ");
\r
171 Pick verbs = Pick.or(new String[]{"goes", "fishes", "walks", "sleeps"});
\r
172 Pick transitive = Pick.or(new String[]{"carries", "lifts", "overturns", "hits", "jumps on"});
\r
173 Pick nouns = Pick.or(new String[]{"dog", "house", "government", "wall", "street", "zebra"});
\r
174 Pick adjectives = Pick.or(new String[]{"red", "glorious", "simple", "nasty", "heavy", "clean"});
\r
175 Pick articles = Pick.or(new String[]{"the", "a"});
\r
176 Pick adverbs = Pick.or(new String[]{"quickly", "oddly", "silently", "boldly"});
\r
177 Pick adjectivePhrase = Pick.and(0.5, Pick.and(adverbs).and2(s)).and2(adjectives);
\r
178 Pick nounPhrase = Pick.and(articles).and2(s)
\r
179 .and2(0.3, Pick.and(adjectivePhrase).and2(s))
\r
181 Pick copula = Pick.or(new String[]{"is", "seems"});
\r
182 Pick sentence1 = Pick.and(nounPhrase).and2(s).and2(verbs)
\r
183 .and2(0.3, Pick.and(s).and2(adverbs)).name("s1");
\r
184 Pick sentence2 = Pick.and(nounPhrase).and2(s).and2(transitive).and2(s).and2(nounPhrase)
\r
185 .and2(0.3, Pick.and(s).and2(adverbs)).name("s2");
\r
186 Pick sentence3 = Pick.and(nounPhrase).and2(s).and2(copula).and2(s).and2(adjectivePhrase).name("s3");
\r
187 Pick conj = Pick.or(new String[]{", but", ", and", ", or"});
\r
188 Pick forward = Pick.unquoted("forward");
\r
189 Pick pair = Pick.and(forward).and2(conj).and2(s).and2(forward).name("part");
\r
190 Pick sentenceBase = Pick.or(sentence1).or2(sentence2).or2(sentence3).or2(0.6666, pair).name("sentence");
\r
191 sentenceBase.replace(forward, sentenceBase);
\r
192 Pick sentence = Pick.and(sentenceBase).and2(Pick.unquoted("."));
\r
193 Pick.Target target = Pick.Target.make(sentence);
\r
194 for (int i = 0; i < 50; ++i) {
\r
195 System.out.println(i + ": " + target.next());
\r
198 private static void testOr(Pick p, int count) {
\r
199 Pick.Target target = Pick.Target.make(p);
\r
200 Counts counts = new Counts(count + 10);
\r
201 for (int i = 0; i < 1000; ++i) {
\r
202 String s = target.next();
\r
203 counts.inc(s.length());
\r
207 private static void testCodePoints(Pick p) {
\r
208 Pick.Target target = Pick.Target.make(p);
\r
209 Counts counts = new Counts(128);
\r
210 for (int i = 0; i < 10000; ++i) {
\r
211 String s = target.next();
\r
212 counts.inc(s.charAt(0));
\r
217 public static void printRandoms() {
\r
218 BNF bnf = new BNF(new Random(0), new Quoter.RuleQuoter())
\r
219 .addRules("[a-z]{2,5}").complete();
\r
220 System.out.println("Start");
\r
221 for (int i = 0; i < 100; ++i) {
\r
222 String temp = bnf.next();
\r
223 System.out.println(i + ")\t" + temp);
\r
227 public static void testTokenizer() {
\r
228 Tokenizer t = new Tokenizer();
\r
230 String[] samples = {"a'b'c d #abc\r e", "'a '123 321",
\r
231 "\\\\", "a'b", "a'", "abc def%?ghi", "%", "a", "\\ a", "a''''b"};
\r
232 for (int i = 0; i < samples.length; ++i) {
\r
233 t.setSource(samples[i]);
\r
234 System.out.println();
\r
235 System.out.println("Input: " + t.getSource());
\r
237 while (type != Tokenizer.DONE) {
\r
239 System.out.println(t.toString(type, false));
\r