Implement generator rules

Introduce a rule attribute "generator" which, if present, specifies
that this rule is used to re-invoke the generator program.  Files built
using generator rules are treated specially in two ways: firstly,
they will not be rebuilt if the command line changes; and secondly,
they are not cleaned by default.

A command line flag "-g" is introduced for the clean tool, which
causes it to remove generator files.

Fixes issue #102.

configure.py

@@ -234,7 +234,8 @@ n.newline()
 
 n.comment('Regenerate build files if build script changes.')
 n.rule('configure',
-       command='./configure.py $configure_args')
+       command='./configure.py $configure_args',
+       generator=True)
 n.build('build.ninja', 'configure',
         implicit=['configure.py', 'misc/ninja_syntax.py'])
 n.newline()

doc/manual.asciidoc

@@ -382,16 +382,16 @@ than the _depth_ mode.  It returns non-zero if an error occurs.
 one.  It can be used to know which rule name to pass to
 +ninja -t targets rule _name_+.
 
-`clean`:: remove built files.  If used like this +ninja -t clean+ it
-removes all the built files.  If used like this
-+ninja -t clean _targets..._+ or like this
-+ninja -t clean target _targets..._+ it removes the given targets and
-recursively all files built for it.  If used like this
-+ninja -t clean rule _rules_+ it removes all files built using the given
-rules. The depfiles are not removed. Files created but not referenced in
-the graph are not removed. This tool takes in account the +-v+ and the
-+-n+ options (note that +-n+ implies +-v+).  It returns non-zero if an
-error occurs.
+`clean`:: remove built files.  If used like this +ninja -t clean+ it removes
+all the built files, except for those created by the generator.  If used like
+this +ninja -t clean -g+ it also removes built files created by the generator.
+If used like this +ninja -t clean _targets..._+ or like this +ninja -t clean
+target _targets..._+ it removes the given targets and recursively all files
+built for it.  If used like this +ninja -t clean rule _rules_+ it removes
+all files built using the given rules. The depfiles are not removed. Files
+created but not referenced in the graph are not removed. This tool takes
+in account the +-v+ and the +-n+ options (note that +-n+ implies +-v+).
+It returns non-zero if an error occurs.
 
 Ninja file reference
 --------------------
@@ -464,6 +464,12 @@ aborting due to a missing input.
   the full command or its description; if a command fails, the full command
   line will always be printed before the command's output.
 
+`generator`:: if present, specifies that this rule is used to re-invoke
+  the generator program.  Files built using `generator` rules are
+  treated specially in two ways: firstly, they will not be rebuilt
+  if the command line changes; and secondly, they are not cleaned
+  by default.
+
 Additionally, the special `$in` and `$out` variables expand to the
 space-separated list of files provided to the `build` line referencing
 this `rule`.

misc/ninja_syntax.py

@@ -28,13 +28,16 @@ class Writer(object):
             value = ' '.join(value)
         self._line('%s = %s' % (key, value), indent)
 
-    def rule(self, name, command, description=None, depfile=None):
+    def rule(self, name, command, description=None, depfile=None,
+             generator=False):
         self._line('rule %s' % name)
         self.variable('command', command, indent=1)
         if description:
             self.variable('description', description, indent=1)
         if depfile:
             self.variable('depfile', depfile, indent=1)
+        if generator:
+            self.variable('generator', '1', indent=1)
 
     def build(self, outputs, rule, inputs=None, implicit=None, order_only=None,
               variables=None):

src/clean.cc

@@ -96,7 +96,7 @@ void Cleaner::PrintFooter() {
   printf("%d files.\n", cleaned_files_count_);
 }
 
-int Cleaner::CleanAll() {
+int Cleaner::CleanAll(bool generator) {
   Reset();
   PrintHeader();
   for (vector<Edge*>::iterator e = state_->edges_.begin();
@@ -104,6 +104,9 @@ int Cleaner::CleanAll() {
     // Do not try to remove phony targets
     if ((*e)->rule_ == &State::kPhonyRule)
       continue;
+    // Do not remove generator's files unless generator specified.
+    if (!generator && (*e)->rule_->generator_)
+      continue;
     for (vector<Node*>::iterator out_node = (*e)->outputs_.begin();
          out_node != (*e)->outputs_.end(); ++out_node) {
       Remove((*out_node)->file_->path_);

src/clean.h

@@ -49,9 +49,10 @@ class Cleaner {
   /// @return non-zero if an error occurs.
   int CleanTargets(int target_count, char* targets[]);
 
-  /// Clean all built files.
+  /// Clean all built files, except for files created by generator rules.
+  /// @param generator If set, also clean files created by generator rules.
   /// @return non-zero if an error occurs.
-  int CleanAll();
+  int CleanAll(bool generator = false);
 
   /// Clean all the file built with the given rule @a rule.
   /// @return non-zero if an error occurs.

src/clean_test.cc

@@ -212,6 +212,28 @@ TEST_F(CleanTest, CleanRuleDryRun) {
   EXPECT_EQ(0u, fs_.files_removed_.size());
 }
 
+TEST_F(CleanTest, CleanRuleGenerator) {
+  ASSERT_NO_FATAL_FAILURE(AssertParse(&state_,
+"rule regen\n"
+"  command = cat $in > $out\n"
+"  generator = 1\n"
+"build out1: cat in1\n"
+"build out2: regen in2\n"));
+  fs_.Create("out1", 1, "");
+  fs_.Create("out2", 1, "");
+
+  Cleaner cleaner(&state_, config_, &fs_);
+  EXPECT_EQ(0, cleaner.CleanAll());
+  EXPECT_EQ(1, cleaner.cleaned_files_count());
+  EXPECT_EQ(1u, fs_.files_removed_.size());
+
+  fs_.Create("out1", 1, "");
+
+  EXPECT_EQ(0, cleaner.CleanAll(/*generator=*/true));
+  EXPECT_EQ(2, cleaner.cleaned_files_count());
+  EXPECT_EQ(2u, fs_.files_removed_.size());
+}
+
 TEST_F(CleanTest, CleanFailure) {
   ASSERT_NO_FATAL_FAILURE(AssertParse(&state_,
                                       "build dir: cat src1\n"));

src/graph.cc

@@ -93,8 +93,10 @@ bool Edge::RecomputeDirty(State* state, DiskInterface* disk_interface,
       (*i)->dirty_ = true;
     } else {
       // May also be dirty due to the command changing since the last build.
+      // But if this is a generator rule, the command changing does not make us
+      // dirty.
       BuildLog::LogEntry* entry;
-      if (state->build_log_ &&
+      if (!rule_->generator_ && state->build_log_ &&
           (entry = state->build_log_->LookupByOutput((*i)->file_->path_))) {
         if (command != entry->command)
           (*i)->dirty_ = true;

src/graph.h

@@ -74,7 +74,7 @@ struct Node {
 
 /// An invokable build command and associated metadata (description, etc.).
 struct Rule {
-  Rule(const string& name) : name_(name) { }
+  Rule(const string& name) : name_(name), generator_(false) { }
 
   bool ParseCommand(const string& command, string* err) {
     return command_.Parse(command, err);
@@ -83,6 +83,7 @@ struct Rule {
   EvalString command_;
   EvalString description_;
   EvalString depfile_;
+  bool generator_;
 };
 
 struct State;

src/ninja.cc

@@ -358,6 +358,23 @@ int CmdRules(State* state, int argc, char* argv[]) {
 }
 
 int CmdClean(State* state, int argc, char* argv[], const BuildConfig& config) {
+  bool generator = false;
+
+  optind = 1;
+  int opt;
+  while ((opt = getopt(argc, argv, "g")) != -1) {
+    switch (opt) {
+      case 'g':
+        generator = true;
+        break;
+      default:
+        Usage(config);
+        return 1;
+    }
+  }
+  argv += optind;
+  argc -= optind;
+
   Cleaner cleaner(state, config);
   if (argc >= 1)
   {
@@ -381,7 +398,7 @@ int CmdClean(State* state, int argc, char* argv[], const BuildConfig& config) {
     }
   }
   else {
-    return cleaner.CleanAll();
+    return cleaner.CleanAll(generator);
   }
 }
 
@@ -479,8 +496,10 @@ reload:
       return CmdTargets(&state, argc, argv);
     if (tool == "rules")
       return CmdRules(&state, argc, argv);
+    // The clean tool uses getopt, and expects argv[0] to contain the name of
+    // the tool, i.e. "clean".
     if (tool == "clean")
-      return CmdClean(&state, argc, argv, config);
+      return CmdClean(&state, argc+1, argv-1, config);
     Error("unknown tool '%s'", tool.c_str());
   }
 

src/parsers.cc

@@ -383,6 +383,12 @@ bool ManifestParser::ParseRule(string* err) {
         eval_target = &rule->depfile_;
       } else if (key == "description") {
         eval_target = &rule->description_;
+      } else if (key == "generator") {
+        rule->generator_ = true;
+        string dummy;
+        if (!tokenizer_.ReadToNewline(&dummy, err))
+          return false;
+        continue;
       } else {
         // Die on other keyvals for now; revisit if we want to add a
         // scope here.