rustbuild: Rewrite user-facing interface

This commit is a rewrite of the user-facing interface to the rustbuild build
system. The intention here is to make it much easier to compile/test the project
without having to remember weird rule names and such. An overall view of the new
interface is:

    # build everything
    ./x.py build

    # document everyting
    ./x.py doc

    # test everything
    ./x.py test

    # test libstd
    ./x.py test src/libstd

    # build libcore stage0
    ./x.py build src/libcore --stage 0

    # run stage1 run-pass tests
    ./x.py test src/test/run-pass --stage 1

The `src/bootstrap/bootstrap.py` script is now aliased as a top-level `x.py`
script. This `x` was chosen to be both short and easily tab-completable (no
collisions in that namespace!). The build system now accepts a "subcommand" of
what to do next, the main ones being build/doc/test.

Each subcommand then receives an optional list of arguments. These arguments are
paths in the source repo of what to work with. That is, if you want to test a
directory, you just pass that directory as an argument.

The purpose of this rewrite is to do away with all of the arcane renames like
"rpass" is the "run-pass" suite, "cfail" is the "compile-fail" suite, etc. By
simply working with directories and files it's much more intuitive of how to run
a test (just pass it as an argument).

The rustbuild step/dependency management was also rewritten along the way to
make this easy to work with and define, but that's largely just a refactoring of
what was there before.

The *intention* is that this support is extended for arbitrary files (e.g.
`src/test/run-pass/my-test-case.rs`), but that isn't quite implemented just yet.
Instead directories work for now but we can follow up with stricter path
filtering logic to plumb through all the arguments.

Author: alexcrichton

README.md

@@ -127,7 +127,7 @@ ones from MSYS if you have it installed). You'll also need Visual Studio 2013 or
 newer with the C++ tools. Then all you need to do is to kick off rustbuild.
 
 ```
-python .\src\bootstrap\bootstrap.py
+python x.py build
 ```
 
 Currently rustbuild only works with some known versions of Visual Studio. If you
@@ -137,7 +137,7 @@ by manually calling the appropriate vcvars file before running the bootstrap.
 
 ```
 CALL "C:\Program Files (x86)\Microsoft Visual Studio 14.0\VC\bin\amd64\vcvars64.bat"
-python .\src\bootstrap\bootstrap.py
+python x.py build
 ```
 
 ## Building Documentation

src/Cargo.lock

@@ -27,7 +27,7 @@ num_cpus = "0.2"
 toml = "0.1"
 getopts = "0.2"
 rustc-serialize = "0.3"
-gcc = { git = "https://github.com/alexcrichton/gcc-rs" }
+gcc = "0.3.36"
 libc = "0.2"
 md5 = "0.1"
 

src/bootstrap/Cargo.toml

@@ -10,24 +10,72 @@ system.
 
 ## Using rustbuild
 
-When configuring Rust via `./configure`, pass the following to enable building
-via this build system:
+The rustbuild build system has a primary entry point, a top level `x.py` script:
 
 ```
-./configure --enable-rustbuild
-make
+python ./x.py build
 ```
 
-Afterwards the `Makefile` which is generated will have a few commands like
-`make check`, `make tidy`, etc. For finer-grained control, the
-`bootstrap.py` entry point can be used:
+Note that if you're on Unix you should be able to execute the script directly:
 
 ```
-python src/bootstrap/bootstrap.py
+./x.py build
 ```
 
-This accepts a number of options like `--stage` and `--step` which can configure
-what's actually being done.
+The script accepts commands, flags, and filters to determine what to do:
+
+* `build` - a general purpose command for compiling code. Alone `build` will
+  bootstrap the entire compiler, and otherwise arguments passed indicate what to
+  build. For example:
+
+  ```
+  # build the whole compiler
+  ./x.py build
+
+  # build the stage1 compier
+  ./x.py build --stage 1
+
+  # build stage0 libstd
+  ./x.py build --stage 0 src/libstd
+
+  # build a particular crate in stage0
+  ./x.py build --stage 0 src/libtest
+  ```
+
+* `test` - a command for executing unit tests. Like the `build` command this
+  will execute the entire test suite by default, and otherwise it can be used to
+  select which test suite is run:
+
+  ```
+  # run all unit tests
+  ./x.py test
+
+  # execute the run-pass test suite
+  ./x.py test src/test/run-pass
+
+  # execute only some tests in the run-pass test suite
+  ./x.py test src/test/run-pass --filter my-filter
+
+  # execute tests in the standard library in stage0
+  ./x.py test --stage 0 src/libstd
+
+  # execute all doc tests
+  ./x.py test src/doc
+  ```
+
+* `doc` - a command for building documentation. Like above can take arguments
+  for what to document.
+
+If you're more used to `./configure` and `make`, however, then you can also
+configure the build system to use rustbuild instead of the old makefiles:
+
+```
+./configure --enable-rustbuild
+make
+```
+
+Afterwards the `Makefile` which is generated will have a few commands like
+`make check`, `make tidy`, etc.
 
 ## Configuring rustbuild
 
@@ -47,7 +95,7 @@ being invoked manually (via the python script).
 The rustbuild build system goes through a few phases to actually build the
 compiler. What actually happens when you invoke rustbuild is:
 
-1. The entry point script, `src/bootstrap/bootstrap.py` is run. This script is
+1. The entry point script, `x.py` is run. This script is
    responsible for downloading the stage0 compiler/Cargo binaries, and it then
    compiles the build system itself (this folder). Finally, it then invokes the
    actual `bootstrap` binary build system.

src/bootstrap/README.md

@@ -399,12 +399,10 @@ def main():
 
     # Run the bootstrap
     args = [os.path.join(rb.build_dir, "bootstrap/debug/bootstrap")]
-    args.append('--src')
-    args.append(rb.rust_root)
-    args.append('--build')
-    args.append(rb.build)
     args.extend(sys.argv[1:])
     env = os.environ.copy()
+    env["BUILD"] = rb.build
+    env["SRC"] = rb.rust_root
     env["BOOTSTRAP_PARENT_ID"] = str(os.getpid())
     rb.run(args, env)
 

src/bootstrap/bootstrap.py

@@ -13,44 +13,19 @@
 //! This file implements the various regression test suites that we execute on
 //! our CI.
 
-use std::collections::{HashMap, HashSet};
+use std::collections::HashSet;
 use std::env;
 use std::fs;
 use std::path::{PathBuf, Path};
 use std::process::Command;
 
 use build_helper::output;
-use rustc_serialize::json;
 
 use {Build, Compiler, Mode};
 use util::{self, dylib_path, dylib_path_var};
 
 const ADB_TEST_DIR: &'static str = "/data/tmp";
 
-#[derive(RustcDecodable)]
-struct Output {
-    packages: Vec<Package>,
-    resolve: Resolve,
-}
-
-#[derive(RustcDecodable)]
-struct Package {
-    id: String,
-    name: String,
-    source: Option<String>,
-}
-
-#[derive(RustcDecodable)]
-struct Resolve {
-    nodes: Vec<ResolveNode>,
-}
-
-#[derive(RustcDecodable)]
-struct ResolveNode {
-    id: String,
-    dependencies: Vec<String>,
-}
-
 /// Runs the `linkchecker` tool as compiled in `stage` by the `host` compiler.
 ///
 /// This tool in `src/tools` will verify the validity of all our links in the
@@ -181,7 +156,7 @@ pub fn compiletest(build: &Build,
     let llvm_version = output(Command::new(&llvm_config).arg("--version"));
     cmd.arg("--llvm-version").arg(llvm_version);
 
-    cmd.args(&build.flags.args);
+    cmd.args(&build.flags.cmd.test_args());
 
     if build.config.verbose || build.flags.verbose {
         cmd.arg("--verbose");
@@ -282,7 +257,7 @@ fn markdown_test(build: &Build, compiler: &Compiler, markdown: &Path) {
     cmd.arg("--test");
     cmd.arg(markdown);
 
-    let mut test_args = build.flags.args.join(" ");
+    let mut test_args = build.flags.cmd.test_args().join(" ");
     if build.config.quiet_tests {
         test_args.push_str(" --quiet");
     }
@@ -302,7 +277,8 @@ fn markdown_test(build: &Build, compiler: &Compiler, markdown: &Path) {
 pub fn krate(build: &Build,
              compiler: &Compiler,
              target: &str,
-             mode: Mode) {
+             mode: Mode,
+             krate: Option<&str>) {
     let (name, path, features, root) = match mode {
         Mode::Libstd => {
             ("libstd", "src/rustc/std_shim", build.std_features(), "std_shim")
@@ -318,24 +294,6 @@ pub fn krate(build: &Build,
     println!("Testing {} stage{} ({} -> {})", name, compiler.stage,
              compiler.host, target);
 
-    // Run `cargo metadata` to figure out what crates we're testing.
-    //
-    // Down below we're going to call `cargo test`, but to test the right set
-    // of packages we're going to have to know what `-p` arguments to pass it
-    // to know what crates to test. Here we run `cargo metadata` to learn about
-    // the dependency graph and what `-p` arguments there are.
-    let mut cargo = Command::new(&build.cargo);
-    cargo.arg("metadata")
-         .arg("--manifest-path").arg(build.src.join(path).join("Cargo.toml"));
-    let output = output(&mut cargo);
-    let output: Output = json::decode(&output).unwrap();
-    let id2pkg = output.packages.iter()
-                        .map(|pkg| (&pkg.id, pkg))
-                        .collect::<HashMap<_, _>>();
-    let id2deps = output.resolve.nodes.iter()
-                        .map(|node| (&node.id, &node.dependencies))
-                        .collect::<HashMap<_, _>>();
-
     // Build up the base `cargo test` command.
     //
     // Pass in some standard flags then iterate over the graph we've discovered
@@ -346,24 +304,25 @@ pub fn krate(build: &Build,
          .arg(build.src.join(path).join("Cargo.toml"))
          .arg("--features").arg(features);
 
-    let mut visited = HashSet::new();
-    let root_pkg = output.packages.iter().find(|p| p.name == root).unwrap();
-    let mut next = vec![&root_pkg.id];
-    while let Some(id) = next.pop() {
-        // Skip any packages with sources listed, as these come from crates.io
-        // and we shouldn't be testing them.
-        if id2pkg[id].source.is_some() {
-            continue
-        }
-        // Right now jemalloc is our only target-specific crate in the sense
-        // that it's not present on all platforms. Custom skip it here for now,
-        // but if we add more this probably wants to get more generalized.
-        if !id.contains("jemalloc") {
-            cargo.arg("-p").arg(&id2pkg[id].name);
+    match krate {
+        Some(krate) => {
+            cargo.arg("-p").arg(krate);
         }
-        for dep in id2deps[id] {
-            if visited.insert(dep) {
-                next.push(dep);
+        None => {
+            let mut visited = HashSet::new();
+            let mut next = vec![root];
+            while let Some(name) = next.pop() {
+                // Right now jemalloc is our only target-specific crate in the sense
+                // that it's not present on all platforms. Custom skip it here for now,
+                // but if we add more this probably wants to get more generalized.
+                if !name.contains("jemalloc") {
+                    cargo.arg("-p").arg(name);
+                }
+                for dep in build.crates[name].deps.iter() {
+                    if visited.insert(dep) {
+                        next.push(dep);
+                    }
+                }
             }
         }
     }
@@ -389,7 +348,7 @@ pub fn krate(build: &Build,
         build.run(cargo.arg("--no-run"));
         krate_emscripten(build, compiler, target, mode);
     } else {
-        cargo.args(&build.flags.args);
+        cargo.args(&build.flags.cmd.test_args());
         build.run(&mut cargo);
     }
 }
@@ -421,7 +380,7 @@ fn krate_android(build: &Build,
                               target = target,
                               test = test_file_name,
                               log = log,
-                              args = build.flags.args.join(" "));
+                              args = build.flags.cmd.test_args().join(" "));
 
         let output = output(Command::new("adb").arg("shell").arg(&program));
         println!("{}", output);

src/bootstrap/check.rs

@@ -25,17 +25,17 @@ pub fn clean(build: &Build) {
     rm_rf(build, &build.out.join("tmp"));
 
     for host in build.config.host.iter() {
-
-        let out = build.out.join(host);
-
-        rm_rf(build, &out.join("doc"));
-
-        for stage in 0..4 {
-            rm_rf(build, &out.join(format!("stage{}", stage)));
-            rm_rf(build, &out.join(format!("stage{}-std", stage)));
-            rm_rf(build, &out.join(format!("stage{}-rustc", stage)));
-            rm_rf(build, &out.join(format!("stage{}-tools", stage)));
-            rm_rf(build, &out.join(format!("stage{}-test", stage)));
+        let entries = match build.out.join(host).read_dir() {
+            Ok(iter) => iter,
+            Err(_) => continue,
+        };
+
+        for entry in entries {
+            let entry = t!(entry);
+            if entry.file_name().to_str() == Some("llvm") {
+                continue
+            }
+            t!(fs::remove_dir_all(&entry.path()));
         }
     }
 }