2 *******************************************************************************
\r
3 * Copyright (C) 2004-2010, International Business Machines Corporation and *
\r
4 * others. All Rights Reserved. *
\r
5 *******************************************************************************
\r
8 package com.ibm.icu.dev.tool.docs;
\r
10 import java.io.BufferedReader;
\r
11 import java.io.File;
\r
12 import java.io.FileInputStream;
\r
13 import java.io.FileOutputStream;
\r
14 import java.io.FilenameFilter;
\r
15 import java.io.IOException;
\r
16 import java.io.InputStream;
\r
17 import java.io.InputStreamReader;
\r
18 import java.io.PrintStream;
\r
19 import java.util.ArrayList;
\r
20 import java.util.HashMap;
\r
21 import java.util.Iterator;
\r
22 import java.util.Map;
\r
23 import java.util.TreeMap;
\r
27 * A simple facility for adding C-like preprocessing to .java files.
\r
28 * This only understands a subset of the C preprocessing syntax.
\r
29 * Its used to manage files that with only small differences can be
\r
30 * compiled for different JVMs. This changes files in place,
\r
31 * commenting out lines based on the current flag settings.
\r
33 public class CodeMangler {
\r
34 private File indir; // root of input
\r
35 private File outdir; // root of output
\r
36 private String suffix; // suffix to process, default '.jpp'
\r
37 private boolean recurse; // true if recurse on directories
\r
38 private boolean force; // true if force reprocess of files
\r
39 private boolean clean; // true if output is to be cleaned
\r
40 private boolean timestamp; // true if we read/write timestamp
\r
41 private boolean nonames; // true if no names in header
\r
42 private HashMap map; // defines
\r
43 private ArrayList names; // files/directories to process
\r
44 private String header; // sorted list of defines passed in
\r
46 private boolean verbose; // true if we emit debug output
\r
48 private static final String IGNORE_PREFIX = "//##";
\r
49 private static final String HEADER_PREFIX = "//##header";
\r
51 public static void main(String[] args) {
\r
52 new CodeMangler(args).run();
\r
55 private static final String usage = "Usage:\n" +
\r
56 " CodeMangler [flags] file... dir... @argfile... \n" +
\r
57 "-in[dir] path - root directory of input files, otherwise use current directory\n" +
\r
58 "-out[dir] path - root directory of output files, otherwise use input directory\n" +
\r
59 "-s[uffix] string - suffix of inputfiles to process, otherwise use '.java' (directories only)\n" +
\r
60 "-c[lean] - remove all control flags from code on output (does not proceed if overwriting)\n" +
\r
61 "-r[ecurse] - if present, recursively process subdirectories\n" +
\r
62 "-f[orce] - force reprocessing of files even if timestamp and headers match\n" +
\r
63 "-t[imestamp] - expect/write timestamp in header\n" +
\r
64 "-dNAME[=VALUE] - define NAME with optional value VALUE\n" +
\r
65 " (or -d NAME[=VALUE])\n" +
\r
66 "-n - do not put NAME/VALUE in header\n" +
\r
67 "-help - print this usage message and exit.\n" +
\r
69 "For file arguments, output '.java' files using the same path/name under the output directory.\n" +
\r
70 "For directory arguments, process all files with the defined suffix in the directory.\n" +
\r
71 " (if recursing, do the same for all files recursively under each directory)\n" +
\r
72 "For @argfile arguments, read the specified text file (strip the '@'), and process each line of that file as \n" +
\r
75 "Directives are one of the following:\n" +
\r
76 " #ifdef, #ifndef, #else, #endif, #if, #elif, #define, #undef\n" +
\r
77 "These may optionally be preceeded by whitespace or //.\n" +
\r
78 "#if, #elif args are of the form 'key == value' or 'key != value'.\n" +
\r
79 "Only exact character match key with value is performed.\n" +
\r
80 "#define args are 'key [==] value', the '==' is optional.\n";
\r
82 CodeMangler(String[] args) {
\r
83 map = new HashMap();
\r
84 names = new ArrayList();
\r
89 String inname = null;
\r
90 String outname = null;
\r
91 boolean processArgs = true;
\r
94 for (int i = 0; i < args.length; ++i) {
\r
96 if ("--".equals(arg)) {
\r
97 processArgs = false;
\r
98 } else if (processArgs && arg.charAt(0) == '-') {
\r
99 if (arg.startsWith("-in")) {
\r
100 inname = args[++i];
\r
101 } else if (arg.startsWith("-out")) {
\r
102 outname = args[++i];
\r
103 } else if (arg.startsWith("-d")) {
\r
104 String id = arg.substring(2);
\r
105 if (id.length() == 0) {
\r
109 int ix = id.indexOf('=');
\r
111 val = id.substring(ix+1);
\r
112 id = id.substring(0,ix);
\r
115 } else if (arg.startsWith("-s")) {
\r
116 suffix = args[++i];
\r
117 } else if (arg.startsWith("-r")) {
\r
119 } else if (arg.startsWith("-f")) {
\r
121 } else if (arg.startsWith("-c")) {
\r
123 } else if (arg.startsWith("-t")) {
\r
125 } else if (arg.startsWith("-h")) {
\r
126 System.out.print(usage);
\r
127 break; // stop before processing arguments, so we will do nothing
\r
128 } else if (arg.startsWith("-v")) {
\r
130 } else if (arg.startsWith("-n")) {
\r
133 System.err.println("Error: unrecognized argument '" + arg + "'");
\r
134 System.err.println(usage);
\r
135 throw new IllegalArgumentException(arg);
\r
138 if (arg.charAt(0) == '@') {
\r
139 File argfile = new File(arg.substring(1));
\r
140 if (argfile.exists() && !argfile.isDirectory()) {
\r
142 BufferedReader br = new BufferedReader(new InputStreamReader(new FileInputStream(argfile)));
\r
143 ArrayList list = new ArrayList();
\r
144 for (int x = 0; x < args.length; ++x) {
\r
148 while (null != (line = br.readLine())) {
\r
149 line = line.trim();
\r
150 if (line.length() > 0 && line.charAt(0) != '#') {
\r
151 if (verbose) System.out.println("adding argument: " + line);
\r
155 args = (String[])list.toArray(new String[list.size()]);
\r
157 catch (IOException e) {
\r
158 System.err.println("error reading arg file: " + e);
\r
166 } catch (IndexOutOfBoundsException e) {
\r
167 String msg = "Error: argument '" + arg + "' missing value";
\r
168 System.err.println(msg);
\r
169 System.err.println(usage);
\r
170 throw new IllegalArgumentException(msg);
\r
173 String username = System.getProperty("user.dir");
\r
174 if (inname == null) {
\r
176 } else if (!(inname.startsWith("\\") || inname.startsWith("/"))) {
\r
177 inname = username + File.separator + inname;
\r
179 indir = new File(inname);
\r
181 indir = indir.getCanonicalFile();
\r
183 catch (IOException e) {
\r
184 // continue, but most likely we'll fail later
\r
186 if (!indir.exists()) {
\r
187 throw new IllegalArgumentException("Input directory '" + indir.getAbsolutePath() + "' does not exist.");
\r
188 } else if (!indir.isDirectory()) {
\r
189 throw new IllegalArgumentException("Input path '" + indir.getAbsolutePath() + "' is not a directory.");
\r
191 if (verbose) System.out.println("indir: " + indir.getAbsolutePath());
\r
193 if (outname == null) {
\r
195 } else if (!(outname.startsWith("\\") || outname.startsWith("/"))) {
\r
196 outname = username + File.separator + outname;
\r
198 outdir = new File(outname);
\r
200 outdir = outdir.getCanonicalFile();
\r
202 catch (IOException e) {
\r
203 // continue, but most likely we'll fail later
\r
205 if (!outdir.exists()) {
\r
206 throw new IllegalArgumentException("Output directory '" + outdir.getAbsolutePath() + "' does not exist.");
\r
207 } else if (!outdir.isDirectory()) {
\r
208 throw new IllegalArgumentException("Output path '" + outdir.getAbsolutePath() + "' is not a directory.");
\r
210 if (verbose) System.out.println("outdir: " + outdir.getAbsolutePath());
\r
212 if (clean && suffix.equals(".java")) {
\r
214 if (outdir.getCanonicalPath().equals(indir.getCanonicalPath())) {
\r
215 throw new IllegalArgumentException("Cannot use 'clean' to overwrite .java files in same directory tree");
\r
218 catch (IOException e) {
\r
219 System.err.println("possible overwrite, error: " + e.getMessage());
\r
220 throw new IllegalArgumentException("Cannot use 'clean' to overrwrite .java files");
\r
224 if (names.isEmpty()) {
\r
228 TreeMap sort = new TreeMap(String.CASE_INSENSITIVE_ORDER);
\r
230 Iterator iter = sort.entrySet().iterator();
\r
231 StringBuffer buf = new StringBuffer();
\r
233 while (iter.hasNext()) {
\r
234 Map.Entry e = (Map.Entry)iter.next();
\r
235 if (buf.length() > 0) {
\r
238 buf.append(e.getKey());
\r
239 String v = (String)e.getValue();
\r
240 if (v != null && v.length() > 0) {
\r
246 header = buf.toString();
\r
250 return process("", (String[])names.toArray(new String[names.size()]));
\r
253 public int process(String path, String[] filenames) {
\r
254 if (verbose) System.out.println("path: '" + path + "'");
\r
256 for (int i = 0; i < filenames.length; ++i) {
\r
257 if (verbose) System.out.println("name " + i + " of " + filenames.length + ": '" + filenames[i] + "'");
\r
258 String name = path + filenames[i];
\r
259 File fin = new File(indir, name);
\r
261 fin = fin.getCanonicalFile();
\r
263 catch (IOException e) {
\r
265 if (!fin.exists()) {
\r
266 System.err.println("File " + fin.getAbsolutePath() + " does not exist.");
\r
269 if (fin.isFile()) {
\r
270 if (verbose) System.out.println("processing file: '" + fin.getAbsolutePath() + "'");
\r
272 int ix = name.lastIndexOf(".");
\r
274 oname = name.substring(0, ix);
\r
279 File fout = new File(outdir, oname);
\r
280 if (processFile(fin, fout)) {
\r
283 } else if (fin.isDirectory()) {
\r
284 if (verbose) System.out.println("recursing on directory '" + fin.getAbsolutePath() + "'");
\r
285 String npath = ".".equals(name) ? path : path + fin.getName() + File.separator;
\r
286 count += process(npath, fin.list(filter)); // recursive call
\r
293 private final FilenameFilter filter = new FilenameFilter() {
\r
294 public boolean accept(File dir, String name) {
\r
295 File f = new File(dir, name);
\r
296 return (f.isFile() && name.endsWith(suffix)) || (f.isDirectory() && recurse);
\r
300 public boolean processFile(File infile, File outfile) {
\r
301 File backup = null;
\r
306 boolean emit = true;
\r
308 private State next;
\r
310 public String toString() {
\r
311 return "line " + lc
\r
313 + "' (emit: " + emit
\r
314 + " tripped: " + tripped
\r
318 void trip(boolean trip) {
\r
319 if (!tripped & trip) {
\r
321 emit = next != null ? next.emit : true;
\r
327 State push(int lc, String line, boolean trip) {
\r
330 State ret = new State();
\r
332 ret.emit = this.emit & trip;
\r
333 ret.tripped = trip;
\r
342 HashMap oldMap = null;
\r
344 long outModTime = 0;
\r
347 PrintStream outstream = null;
\r
348 InputStream instream = new FileInputStream(infile);
\r
350 BufferedReader reader = new BufferedReader(new InputStreamReader(instream));
\r
352 State state = new State();
\r
354 while ((line = reader.readLine()) != null) {
\r
355 if (lc == 0) { // check and write header for output file if needed
\r
356 boolean hasHeader = line.startsWith(HEADER_PREFIX);
\r
357 if (hasHeader && !force) {
\r
358 long expectLastModified = ((infile.lastModified() + 999)/1000)*1000;
\r
359 String headerline = HEADER_PREFIX;
\r
360 if (header.length() > 0) {
\r
362 headerline += header;
\r
366 headerline += String.valueOf(expectLastModified);
\r
368 if (line.equals(headerline)) {
\r
369 if (verbose) System.out.println("no changes necessary to " + infile.getCanonicalPath());
\r
371 return false; // nothing to do
\r
374 System.out.println(" old header: " + line);
\r
375 System.out.println(" != expected: " + headerline);
\r
379 // create output file directory structure
\r
380 String outpname = outfile.getParent();
\r
381 if (outpname != null) {
\r
382 File outp = new File(outpname);
\r
383 if (!(outp.exists() || outp.mkdirs())) {
\r
384 System.err.println("could not create directory: '" + outpname + "'");
\r
389 // if we're overwriting, use a temporary file
\r
390 if (suffix.equals(".java")) {
\r
393 outfile = File.createTempFile(outfile.getName(), null, outfile.getParentFile());
\r
395 catch (IOException ex) {
\r
396 System.err.println(ex.getMessage());
\r
401 outModTime = ((outfile.lastModified()+999)/1000)*1000; // round up
\r
402 outstream = new PrintStream(new FileOutputStream(outfile));
\r
403 String headerline = HEADER_PREFIX;
\r
404 if (header.length() > 0) {
\r
406 headerline += header;
\r
410 headerline += String.valueOf(outModTime);
\r
412 outstream.println(headerline);
\r
413 if (verbose) System.out.println("header: " + headerline);
\r
415 // discard the old header if we had one, otherwise match this line like any other
\r
417 ++lc; // mark as having read a line so we never reexecute this block
\r
422 String[] res = new String[3];
\r
423 if (patMatch(line, res)) {
\r
424 String lead = res[0];
\r
425 String key = res[1];
\r
426 String val = res[2];
\r
428 if (verbose) System.out.println("directive: " + line
\r
430 + "' val: '" + val
\r
432 if (key.equals("ifdef")) {
\r
433 state = state.push(lc, line, map.get(val) != null);
\r
434 } else if (key.equals("ifndef")) {
\r
435 state = state.push(lc, line, map.get(val) == null);
\r
436 } else if (key.equals("else")) {
\r
438 } else if (key.equals("endif")) {
\r
439 state = state.pop();
\r
440 } else if (key.equals("undef")) {
\r
442 if (oldMap == null) {
\r
443 oldMap = (HashMap)map.clone();
\r
447 } else if (key.equals("define")) {
\r
448 if (pat2Match(val, res)) {
\r
449 String key2 = res[0];
\r
450 String val2 = res[2];
\r
452 if (verbose) System.out.println("val2: '" + val2
\r
453 + "' key2: '" + key2
\r
456 if (oldMap == null) {
\r
457 oldMap = (HashMap)map.clone();
\r
459 map.put(key2, val2);
\r
462 } else { // #if, #elif
\r
463 // only top level OR (||) operator is supported for now
\r
466 while ((index = val.indexOf("||", index)) > 0) {
\r
470 String[] expressions = new String[count];
\r
472 expressions[0] = val;
\r
478 index = val.indexOf("||", start);
\r
480 expressions[count++] = val.substring(start, index);
\r
483 expressions[count++] = val.substring(start);
\r
488 boolean eval = false;
\r
489 for (count = 0; count < expressions.length && !eval; count++) {
\r
490 if (pat2Match(expressions[count], res)) {
\r
491 String key2 = res[0];
\r
492 String val2 = res[2];
\r
494 if (key2.equals("defined")) {
\r
496 if (verbose) System.out.println(
\r
498 + "' val2: '" + val2
\r
499 + "' key2: '" + key2
\r
501 eval = map.containsKey(val2);
\r
503 boolean neq = false;
\r
504 if (res[1].equals("!=")) {
\r
506 } else if (!res[1].equals("==")) {
\r
507 System.err.println("Invalid expression: '" + val);
\r
509 if (verbose) System.out.println(
\r
511 + "' val2: '" + val2
\r
512 + "' neq: '" + neq
\r
513 + "' key2: '" + key2
\r
515 eval = (val2.equals(map.get(key2)) != neq);
\r
519 if (key.equals("if")) {
\r
520 state = state.push(lc, line, eval);
\r
521 } else if (key.equals("elif")) {
\r
527 if (!lead.equals("//")) {
\r
528 outstream.print("//");
\r
529 line = line.substring(lead.length());
\r
531 outstream.println(line);
\r
537 String found = pat3Match(line);
\r
538 boolean hasIgnore = found != null;
\r
539 if (state.emit == hasIgnore) {
\r
541 line = line.substring(found.length());
\r
543 line = IGNORE_PREFIX + line;
\r
545 } else if (hasIgnore && !found.equals(IGNORE_PREFIX)) {
\r
546 line = IGNORE_PREFIX + line.substring(found.length());
\r
548 if (!clean || state.emit) {
\r
549 outstream.println(line);
\r
553 state = state.pop();
\r
554 if (state != null) {
\r
555 System.err.println("Error: unclosed directive(s):");
\r
557 System.err.println(state);
\r
558 } while ((state = state.pop()) != null);
\r
559 System.err.println(" in file: " + outfile.getCanonicalPath());
\r
560 if (oldMap != null) {
\r
570 if (backup != null) {
\r
571 if (backup.exists()) {
\r
574 outfile.renameTo(backup);
\r
578 outfile.setLastModified(outModTime); // synch with timestamp
\r
581 if (oldMap != null) {
\r
585 catch (IOException e) {
\r
586 System.err.println(e);
\r
594 * Perform same operation as matching on pat. on exit
\r
595 * leadKeyValue contains the three strings lead, key, and value.
\r
596 * 'lead' is the portion before the #ifdef directive. 'key' is
\r
597 * the directive. 'value' is the portion after the directive. if
\r
598 * there is a match, return true, else return false.
\r
600 static boolean patMatch(String line, String[] leadKeyValue) {
\r
601 if (line.length() == 0) {
\r
604 if (!line.endsWith("\n")) {
\r
605 line = line + '\n';
\r
609 loop: for (int i = 0; i < line.length(); ++i) {
\r
610 char c = line.charAt(i);
\r
612 case 0: // at start of line, haven't seen anything but whitespace yet
\r
613 if (c == ' ' || c == '\t' || c == '\r') continue;
\r
614 if (c == '/') { state = 1; continue; }
\r
615 if (c == '#') { state = 4; continue; }
\r
617 case 1: // have seen a single slash after start of line
\r
618 if (c == '/') { state = 2; continue; }
\r
620 case 2: // have seen two or more slashes
\r
621 if (c == '/') continue;
\r
622 if (c == ' ' || c == '\t' || c == '\r') { state = 3; continue; }
\r
623 if (c == '#') { state = 4; continue; }
\r
625 case 3: // have seen a space after two or more slashes
\r
626 if (c == ' ' || c == '\t' || c == '\r') continue;
\r
627 if (c == '#') { state = 4; continue; }
\r
629 case 4: // have seen a '#'
\r
630 leadKeyValue[0] = line.substring(mark, i-1);
\r
631 if ((c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z')) { mark = i; state = 5; continue; }
\r
633 case 5: // an ascii char followed the '#'
\r
634 if ((c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z')) continue;
\r
635 if (c == ' ' || c == '\t' || c == '\n') {
\r
636 String key = line.substring(mark, i).toLowerCase();
\r
637 if (key.equals("ifdef") ||
\r
638 key.equals("ifndef") ||
\r
639 key.equals("else") ||
\r
640 key.equals("endif") ||
\r
641 key.equals("undef") ||
\r
642 key.equals("define") ||
\r
643 key.equals("if") ||
\r
644 key.equals("elif")) {
\r
645 leadKeyValue[1] = key;
\r
653 throw new IllegalStateException();
\r
657 leadKeyValue[2] = line.substring(mark, line.length()).trim();
\r
660 return false; // never reached, does the compiler know this?
\r
664 * Perform same operation as matching on pat2. on exit
\r
665 * keyRelValue contains the three strings key, rel, and value.
\r
666 * 'key' is the portion before the relation (or final word). 'rel' is
\r
667 * the relation, if present, either == or !=. 'value' is the final
\r
668 * word. if there is a match, return true, else return false.
\r
670 static boolean pat2Match(String line, String[] keyRelVal) {
\r
672 if (line.length() == 0) {
\r
675 keyRelVal[0] = keyRelVal[1] = keyRelVal[2] = "";
\r
678 String command = null;
\r
679 loop: for (int i = 0; i < line.length(); ++i) {
\r
680 char c = line.charAt(i);
\r
682 case 0: // saw beginning or space, no rel yet
\r
683 if (c == ' ' || c == '\t' || c == '\n') {
\r
686 if ((c == '!' || c == '=')) {
\r
691 case 1: // saw start of a word
\r
692 if (c == ' ' || c == '\t') {
\r
695 else if (c == '(') {
\r
696 command = line.substring(0, i).trim();
\r
697 if (!command.equals("defined")) {
\r
700 keyRelVal[0] = command;
\r
703 else if (c == '!' || c == '=') {
\r
707 case 2: // saw end of word, and space
\r
708 if (c == ' ' || c == '\t') {
\r
711 else if (command == null && c == '(') {
\r
714 else if (c == '!' || c == '=') {
\r
718 keyRelVal[0] = line.substring(0, i-1).trim();
\r
722 case 3: // saw end of word, and '!' or '='
\r
724 keyRelVal[0] = line.substring(0, i-1).trim();
\r
725 keyRelVal[1] = line.substring(i-1, i+1);
\r
737 return false; // found nothing
\r
740 keyRelVal[0] = line.trim(); break; // found only a word
\r
742 return false; // found a word and '!' or '=" then end of line, incomplete
\r
744 keyRelVal[2] = line.substring(mark).trim(); // found a word, possible rel, and who knows what
\r
745 if (command != null) {
\r
746 int len = keyRelVal[2].length();
\r
747 if (keyRelVal[2].charAt(len - 1) != ')') {
\r
748 // closing parenthesis is missing
\r
751 keyRelVal[2] = keyRelVal[2].substring(0, len - 1).trim();
\r
755 throw new IllegalStateException();
\r
760 static String pat3Match(String line) {
\r
762 loop: for (int i = 0; i < line.length(); ++i) {
\r
763 char c = line.charAt(i);
\r
765 case 0: if (c == ' ' || c == '\t') continue;
\r
766 if (c == '/') { state = 1; continue; }
\r
769 if (c == '/') { state = 2; continue; }
\r
772 if (c == '#') { state = 3; continue; }
\r
775 if (c == '#') return line.substring(0, i+1);
\r