2 *******************************************************************************
3 * Copyright (C) 2004-2012, International Business Machines Corporation and *
4 * others. All Rights Reserved. *
5 *******************************************************************************
8 package com.ibm.icu.dev.tool.docs;
10 import java.io.BufferedReader;
12 import java.io.FileInputStream;
13 import java.io.FileOutputStream;
14 import java.io.FilenameFilter;
15 import java.io.IOException;
16 import java.io.InputStream;
17 import java.io.InputStreamReader;
18 import java.io.PrintStream;
19 import java.util.ArrayList;
20 import java.util.HashMap;
21 import java.util.Iterator;
23 import java.util.TreeMap;
27 * A simple facility for adding C-like preprocessing to .java files.
28 * This only understands a subset of the C preprocessing syntax.
29 * Its used to manage files that with only small differences can be
30 * compiled for different JVMs. This changes files in place,
31 * commenting out lines based on the current flag settings.
33 public class CodeMangler {
34 private File indir; // root of input
35 private File outdir; // root of output
36 private String suffix; // suffix to process, default '.jpp'
37 private boolean recurse; // true if recurse on directories
38 private boolean force; // true if force reprocess of files
39 private boolean clean; // true if output is to be cleaned
40 private boolean timestamp; // true if we read/write timestamp
41 private boolean nonames; // true if no names in header
42 private HashMap map; // defines
43 private ArrayList names; // files/directories to process
44 private String header; // sorted list of defines passed in
46 private boolean verbose; // true if we emit debug output
48 private static final String IGNORE_PREFIX = "//##";
49 private static final String HEADER_PREFIX = "//##header";
51 public static void main(String[] args) {
52 new CodeMangler(args).run();
55 private static final String usage = "Usage:\n" +
56 " CodeMangler [flags] file... dir... @argfile... \n" +
57 "-in[dir] path - root directory of input files, otherwise use current directory\n" +
58 "-out[dir] path - root directory of output files, otherwise use input directory\n" +
59 "-s[uffix] string - suffix of inputfiles to process, otherwise use '.java' (directories only)\n" +
60 "-c[lean] - remove all control flags from code on output (does not proceed if overwriting)\n" +
61 "-r[ecurse] - if present, recursively process subdirectories\n" +
62 "-f[orce] - force reprocessing of files even if timestamp and headers match\n" +
63 "-t[imestamp] - expect/write timestamp in header\n" +
64 "-dNAME[=VALUE] - define NAME with optional value VALUE\n" +
65 " (or -d NAME[=VALUE])\n" +
66 "-n - do not put NAME/VALUE in header\n" +
67 "-help - print this usage message and exit.\n" +
69 "For file arguments, output '.java' files using the same path/name under the output directory.\n" +
70 "For directory arguments, process all files with the defined suffix in the directory.\n" +
71 " (if recursing, do the same for all files recursively under each directory)\n" +
72 "For @argfile arguments, read the specified text file (strip the '@'), and process each line of that file as \n" +
75 "Directives are one of the following:\n" +
76 " #ifdef, #ifndef, #else, #endif, #if, #elif, #define, #undef\n" +
77 "These may optionally be preceeded by whitespace or //.\n" +
78 "#if, #elif args are of the form 'key == value' or 'key != value'.\n" +
79 "Only exact character match key with value is performed.\n" +
80 "#define args are 'key [==] value', the '==' is optional.\n";
82 CodeMangler(String[] args) {
84 names = new ArrayList();
90 String outname = null;
91 boolean processArgs = true;
94 for (int i = 0; i < args.length; ++i) {
96 if ("--".equals(arg)) {
98 } else if (processArgs && arg.charAt(0) == '-') {
99 if (arg.startsWith("-in")) {
101 } else if (arg.startsWith("-out")) {
103 } else if (arg.startsWith("-d")) {
104 String id = arg.substring(2);
105 if (id.length() == 0) {
109 int ix = id.indexOf('=');
111 val = id.substring(ix+1);
112 id = id.substring(0,ix);
115 } else if (arg.startsWith("-s")) {
117 } else if (arg.startsWith("-r")) {
119 } else if (arg.startsWith("-f")) {
121 } else if (arg.startsWith("-c")) {
123 } else if (arg.startsWith("-t")) {
125 } else if (arg.startsWith("-h")) {
126 System.out.print(usage);
127 break; // stop before processing arguments, so we will do nothing
128 } else if (arg.startsWith("-v")) {
130 } else if (arg.startsWith("-n")) {
133 System.err.println("Error: unrecognized argument '" + arg + "'");
134 System.err.println(usage);
135 throw new IllegalArgumentException(arg);
138 if (arg.charAt(0) == '@') {
139 File argfile = new File(arg.substring(1));
140 if (argfile.exists() && !argfile.isDirectory()) {
141 BufferedReader br = null;
143 br = new BufferedReader(new InputStreamReader(new FileInputStream(argfile)));
144 ArrayList list = new ArrayList();
145 for (int x = 0; x < args.length; ++x) {
149 while (null != (line = br.readLine())) {
151 if (line.length() > 0 && line.charAt(0) != '#') {
152 if (verbose) System.out.println("adding argument: " + line);
156 args = (String[])list.toArray(new String[list.size()]);
158 catch (IOException e) {
159 System.err.println("error reading arg file: " + e);
165 } catch (Exception e){
176 } catch (IndexOutOfBoundsException e) {
177 String msg = "Error: argument '" + arg + "' missing value";
178 System.err.println(msg);
179 System.err.println(usage);
180 throw new IllegalArgumentException(msg);
183 String username = System.getProperty("user.dir");
184 if (inname == null) {
186 } else if (!(inname.startsWith("\\") || inname.startsWith("/"))) {
187 inname = username + File.separator + inname;
189 indir = new File(inname);
191 indir = indir.getCanonicalFile();
193 catch (IOException e) {
194 // continue, but most likely we'll fail later
196 if (!indir.exists()) {
197 throw new IllegalArgumentException("Input directory '" + indir.getAbsolutePath() + "' does not exist.");
198 } else if (!indir.isDirectory()) {
199 throw new IllegalArgumentException("Input path '" + indir.getAbsolutePath() + "' is not a directory.");
201 if (verbose) System.out.println("indir: " + indir.getAbsolutePath());
203 if (outname == null) {
205 } else if (!(outname.startsWith("\\") || outname.startsWith("/"))) {
206 outname = username + File.separator + outname;
208 outdir = new File(outname);
210 outdir = outdir.getCanonicalFile();
212 catch (IOException e) {
213 // continue, but most likely we'll fail later
215 if (!outdir.exists()) {
216 throw new IllegalArgumentException("Output directory '" + outdir.getAbsolutePath() + "' does not exist.");
217 } else if (!outdir.isDirectory()) {
218 throw new IllegalArgumentException("Output path '" + outdir.getAbsolutePath() + "' is not a directory.");
220 if (verbose) System.out.println("outdir: " + outdir.getAbsolutePath());
222 if (clean && suffix.equals(".java")) {
224 if (outdir.getCanonicalPath().equals(indir.getCanonicalPath())) {
225 throw new IllegalArgumentException("Cannot use 'clean' to overwrite .java files in same directory tree");
228 catch (IOException e) {
229 System.err.println("possible overwrite, error: " + e.getMessage());
230 throw new IllegalArgumentException("Cannot use 'clean' to overrwrite .java files");
234 if (names.isEmpty()) {
238 TreeMap sort = new TreeMap(String.CASE_INSENSITIVE_ORDER);
240 Iterator iter = sort.entrySet().iterator();
241 StringBuffer buf = new StringBuffer();
243 while (iter.hasNext()) {
244 Map.Entry e = (Map.Entry)iter.next();
245 if (buf.length() > 0) {
248 buf.append(e.getKey());
249 String v = (String)e.getValue();
250 if (v != null && v.length() > 0) {
256 header = buf.toString();
260 return process("", (String[])names.toArray(new String[names.size()]));
263 public int process(String path, String[] filenames) {
264 if (verbose) System.out.println("path: '" + path + "'");
266 for (int i = 0; i < filenames.length; ++i) {
267 if (verbose) System.out.println("name " + i + " of " + filenames.length + ": '" + filenames[i] + "'");
268 String name = path + filenames[i];
269 File fin = new File(indir, name);
271 fin = fin.getCanonicalFile();
273 catch (IOException e) {
276 System.err.println("File " + fin.getAbsolutePath() + " does not exist.");
280 if (verbose) System.out.println("processing file: '" + fin.getAbsolutePath() + "'");
282 int ix = name.lastIndexOf(".");
284 oname = name.substring(0, ix);
289 File fout = new File(outdir, oname);
290 if (processFile(fin, fout)) {
293 } else if (fin.isDirectory()) {
294 if (verbose) System.out.println("recursing on directory '" + fin.getAbsolutePath() + "'");
295 String npath = ".".equals(name) ? path : path + fin.getName() + File.separator;
296 count += process(npath, fin.list(filter)); // recursive call
303 private final FilenameFilter filter = new FilenameFilter() {
304 public boolean accept(File dir, String name) {
305 File f = new File(dir, name);
306 return (f.isFile() && name.endsWith(suffix)) || (f.isDirectory() && recurse);
310 public boolean processFile(File infile, File outfile) {
320 public String toString() {
324 + " tripped: " + tripped
328 void trip(boolean trip) {
329 if (!tripped & trip) {
331 emit = next != null ? next.emit : true;
337 State push(int lc, String line, boolean trip) {
340 State ret = new State();
342 ret.emit = this.emit & trip;
352 HashMap oldMap = null;
357 PrintStream outstream = null;
358 InputStream instream = new FileInputStream(infile);
360 BufferedReader reader = new BufferedReader(new InputStreamReader(instream));
362 State state = new State();
364 while ((line = reader.readLine()) != null) {
365 if (lc == 0) { // check and write header for output file if needed
366 boolean hasHeader = line.startsWith(HEADER_PREFIX);
367 if (hasHeader && !force) {
368 long expectLastModified = ((infile.lastModified() + 999)/1000)*1000;
369 String headerline = HEADER_PREFIX;
370 if (header.length() > 0) {
372 headerline += header;
376 headerline += String.valueOf(expectLastModified);
378 if (line.equals(headerline)) {
379 if (verbose) System.out.println("no changes necessary to " + infile.getCanonicalPath());
381 return false; // nothing to do
384 System.out.println(" old header: " + line);
385 System.out.println(" != expected: " + headerline);
389 // create output file directory structure
390 String outpname = outfile.getParent();
391 if (outpname != null) {
392 File outp = new File(outpname);
393 if (!(outp.exists() || outp.mkdirs())) {
394 System.err.println("could not create directory: '" + outpname + "'");
400 // if we're overwriting, use a temporary file
401 if (suffix.equals(".java")) {
404 outfile = File.createTempFile(outfile.getName(), null, outfile.getParentFile());
406 catch (IOException ex) {
407 System.err.println(ex.getMessage());
413 outModTime = ((outfile.lastModified()+999)/1000)*1000; // round up
414 outstream = new PrintStream(new FileOutputStream(outfile));
415 String headerline = HEADER_PREFIX;
416 if (header.length() > 0) {
418 headerline += header;
422 headerline += String.valueOf(outModTime);
424 outstream.println(headerline);
425 if (verbose) System.out.println("header: " + headerline);
427 // discard the old header if we had one, otherwise match this line like any other
429 ++lc; // mark as having read a line so we never reexecute this block
434 String[] res = new String[3];
435 if (patMatch(line, res)) {
436 String lead = res[0];
440 if (verbose) System.out.println("directive: " + line
444 if (key.equals("ifdef")) {
445 state = state.push(lc, line, map.get(val) != null);
446 } else if (key.equals("ifndef")) {
447 state = state.push(lc, line, map.get(val) == null);
448 } else if (key.equals("else")) {
450 } else if (key.equals("endif")) {
452 } else if (key.equals("undef")) {
454 if (oldMap == null) {
455 oldMap = (HashMap)map.clone();
459 } else if (key.equals("define")) {
460 if (pat2Match(val, res)) {
461 String key2 = res[0];
462 String val2 = res[2];
464 if (verbose) System.out.println("val2: '" + val2
468 if (oldMap == null) {
469 oldMap = (HashMap)map.clone();
474 } else { // #if, #elif
475 // only top level OR (||) operator is supported for now
478 while ((index = val.indexOf("||", index)) > 0) {
482 String[] expressions = new String[count];
484 expressions[0] = val;
490 index = val.indexOf("||", start);
492 expressions[count++] = val.substring(start, index);
495 expressions[count++] = val.substring(start);
500 boolean eval = false;
501 for (count = 0; count < expressions.length && !eval; count++) {
502 if (pat2Match(expressions[count], res)) {
503 String key2 = res[0];
504 String val2 = res[2];
506 if (key2.equals("defined")) {
508 if (verbose) System.out.println(
513 eval = map.containsKey(val2);
516 if (res[1].equals("!=")) {
518 } else if (!res[1].equals("==")) {
519 System.err.println("Invalid expression: '" + val);
521 if (verbose) System.out.println(
527 eval = (val2.equals(map.get(key2)) != neq);
531 if (key.equals("if")) {
532 state = state.push(lc, line, eval);
533 } else if (key.equals("elif")) {
539 if (!lead.equals("//")) {
540 outstream.print("//");
541 line = line.substring(lead.length());
543 outstream.println(line);
549 String found = pat3Match(line);
550 boolean hasIgnore = found != null;
551 if (state.emit == hasIgnore) {
553 line = line.substring(found.length());
555 line = IGNORE_PREFIX + line;
557 } else if (hasIgnore && !found.equals(IGNORE_PREFIX)) {
558 line = IGNORE_PREFIX + line.substring(found.length());
560 if (!clean || state.emit) {
561 outstream.println(line);
567 System.err.println("Error: unclosed directive(s):");
569 System.err.println(state);
570 } while ((state = state.pop()) != null);
571 System.err.println(" in file: " + outfile.getCanonicalPath());
572 if (oldMap != null) {
583 if (backup != null) {
584 if (backup.exists()) {
587 outfile.renameTo(backup);
591 outfile.setLastModified(outModTime); // synch with timestamp
594 if (oldMap != null) {
598 catch (IOException e) {
599 System.err.println(e);
607 * Perform same operation as matching on pat. on exit
608 * leadKeyValue contains the three strings lead, key, and value.
609 * 'lead' is the portion before the #ifdef directive. 'key' is
610 * the directive. 'value' is the portion after the directive. if
611 * there is a match, return true, else return false.
613 static boolean patMatch(String line, String[] leadKeyValue) {
614 if (line.length() == 0) {
617 if (!line.endsWith("\n")) {
622 loop: for (int i = 0; i < line.length(); ++i) {
623 char c = line.charAt(i);
625 case 0: // at start of line, haven't seen anything but whitespace yet
626 if (c == ' ' || c == '\t' || c == '\r') continue;
627 if (c == '/') { state = 1; continue; }
628 if (c == '#') { state = 4; continue; }
630 case 1: // have seen a single slash after start of line
631 if (c == '/') { state = 2; continue; }
633 case 2: // have seen two or more slashes
634 if (c == '/') continue;
635 if (c == ' ' || c == '\t' || c == '\r') { state = 3; continue; }
636 if (c == '#') { state = 4; continue; }
638 case 3: // have seen a space after two or more slashes
639 if (c == ' ' || c == '\t' || c == '\r') continue;
640 if (c == '#') { state = 4; continue; }
642 case 4: // have seen a '#'
643 leadKeyValue[0] = line.substring(mark, i-1);
644 if ((c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z')) { mark = i; state = 5; continue; }
646 case 5: // an ascii char followed the '#'
647 if ((c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z')) continue;
648 if (c == ' ' || c == '\t' || c == '\n') {
649 String key = line.substring(mark, i).toLowerCase();
650 if (key.equals("ifdef") ||
651 key.equals("ifndef") ||
652 key.equals("else") ||
653 key.equals("endif") ||
654 key.equals("undef") ||
655 key.equals("define") ||
657 key.equals("elif")) {
658 leadKeyValue[1] = key;
666 throw new IllegalStateException();
670 leadKeyValue[2] = line.substring(mark, line.length()).trim();
673 return false; // never reached, does the compiler know this?
677 * Perform same operation as matching on pat2. on exit
678 * keyRelValue contains the three strings key, rel, and value.
679 * 'key' is the portion before the relation (or final word). 'rel' is
680 * the relation, if present, either == or !=. 'value' is the final
681 * word. if there is a match, return true, else return false.
683 static boolean pat2Match(String line, String[] keyRelVal) {
685 if (line.length() == 0) {
688 keyRelVal[0] = keyRelVal[1] = keyRelVal[2] = "";
691 String command = null;
692 loop: for (int i = 0; i < line.length(); ++i) {
693 char c = line.charAt(i);
695 case 0: // saw beginning or space, no rel yet
696 if (c == ' ' || c == '\t' || c == '\n') {
699 if ((c == '!' || c == '=')) {
704 case 1: // saw start of a word
705 if (c == ' ' || c == '\t') {
709 command = line.substring(0, i).trim();
710 if (!command.equals("defined")) {
713 keyRelVal[0] = command;
716 else if (c == '!' || c == '=') {
720 case 2: // saw end of word, and space
721 if (c == ' ' || c == '\t') {
724 else if (command == null && c == '(') {
727 else if (c == '!' || c == '=') {
731 keyRelVal[0] = line.substring(0, i-1).trim();
735 case 3: // saw end of word, and '!' or '='
737 keyRelVal[0] = line.substring(0, i-1).trim();
738 keyRelVal[1] = line.substring(i-1, i+1);
750 return false; // found nothing
753 keyRelVal[0] = line.trim(); break; // found only a word
755 return false; // found a word and '!' or '=" then end of line, incomplete
757 keyRelVal[2] = line.substring(mark).trim(); // found a word, possible rel, and who knows what
758 if (command != null) {
759 int len = keyRelVal[2].length();
760 if (keyRelVal[2].charAt(len - 1) != ')') {
761 // closing parenthesis is missing
764 keyRelVal[2] = keyRelVal[2].substring(0, len - 1).trim();
768 throw new IllegalStateException();
773 static String pat3Match(String line) {
775 loop: for (int i = 0; i < line.length(); ++i) {
776 char c = line.charAt(i);
778 case 0: if (c == ' ' || c == '\t') continue;
779 if (c == '/') { state = 1; continue; }
782 if (c == '/') { state = 2; continue; }
785 if (c == '#') { state = 3; continue; }
788 if (c == '#') return line.substring(0, i+1);