configure

fn configure(graph: *Graph, options: ConfigureOptions) !ScannedConfig

File

Code

fn configure(graph: *Graph, options: ConfigureOptions) !ScannedConfig {
    const configure_argv = options.configure_argv;
    const gpa = graph.cache.gpa;
    const io = graph.io;
    const arena = graph.arena;

    configure_argv[options.conf_argv_index_build_root] = options.build_root.directory.path orelse options.cwd_path;

    var http_client: std.http.Client = .{ .allocator = gpa, .io = io };
    defer http_client.deinit();

    var unlazy_set: Package.Fetch.JobQueue.UnlazySet = .{};
    var fork_set: Package.Fetch.JobQueue.ForkSet = .{};

    {
        // Populate fork_set.
        var group: Io.Group = .init;
        defer group.cancel(io);

        for (options.forks) |*fork|
            group.async(io, Fork.load, .{ io, gpa, fork, options.color });

        try group.await(io);

        for (options.forks) |*fork| {
            if (fork.failed) return error.AlreadyReported;
            try fork_set.put(arena, .{
                .path = fork.path,
                .manifest_ast = fork.manifest_ast,
                .manifest = fork.manifest,
                .uses = 0,
            }, {});
        }
    }
    defer Fork.deinitList(options.forks);

    var build_configurer_argv: std.ArrayList([]const u8) = .empty;
    defer build_configurer_argv.deinit(gpa);

    var dependencies_source: std.ArrayList(u8) = .empty;
    defer dependencies_source.deinit(gpa);

    const configurer_root_src_path: Cache.Path = .{
        .root_dir = graph.zig_lib_directory,
        .sub_path = "compiler/configurer.zig",
    };

    const root_build_src_path: Cache.Path = .{
        .root_dir = options.build_root.directory,
        .sub_path = options.build_root.build_zig_basename,
    };

    const configurer_exe_name = "configurer";

    try build_configurer_argv.appendSlice(gpa, &.{
        graph.zig_exe, "build-exe", //
        "--cache-dir", graph.local_cache_root.path orelse ".", //
        "--global-cache-dir", graph.global_cache_root.path orelse ".", //
        "--zig-lib-dir", graph.zig_lib_directory.path orelse ".", //
        "--name", configurer_exe_name, //
        "-fsingle-threaded", //
    });

    // Normally the build runner is compiled for the host target but here is
    // some code to help when debugging edits to the build runner so that you
    // can make sure it compiles successfully on other targets.
    const target_arch_os_abi: ?[]const u8 = if (options.debug_target) |triple| t: {
        try build_configurer_argv.appendSlice(gpa, &.{ "-target", triple });
        break :t triple;
    } else null;

    if (graph.libc_file) |libc_file| {
        try build_configurer_argv.appendSlice(gpa, &.{ "--libc", libc_file });
    }
    if (graph.reference_trace) |n| {
        try build_configurer_argv.append(gpa, try arena.print("-freference-trace={d}", .{n}));
    }
    if (graph.debug_compile_errors) {
        try build_configurer_argv.append(gpa, "--debug-compile-errors");
    }
    try build_configurer_argv.appendSlice(gpa, &.{
        "--dep", "@build", //
        "--dep", "@dependencies", //
        try arena.print("-Mroot={f}", .{configurer_root_src_path}), //
    });

    // In the loop below, after doing the fetch operation, the argv will be
    // truncated at this point, dependencies added, and then the
    // "--listen=-" arg appended at the end.
    const argv_deps_index = build_configurer_argv.items.len;

    const build_mod = try arena.create(CliModule);
    build_mod.* = .{
        .name = "@build",
        .root_path = try root_build_src_path.toString(arena),
    };

    const deps_mod = try arena.create(CliModule);
    deps_mod.* = .{
        .name = "@dependencies",
        .root_path = undefined,
    };

    // This loop is re-evaluated when the build script exits with an indication that it
    // could not continue due to missing lazy dependencies.
    const configuration_path: Path, var configuration_lock: ?Cache.Lock = cp: while (true) {
        build_mod.deps.clearRetainingCapacity();
        deps_mod.deps.clearRetainingCapacity();

        // Cache lookup for configure options. If we get a match, we can skip
        // execution of the configure script. If not, we get the file path to pass
        // to the configure process.
        //
        // In the hot path, we only check this cache, which means that also
        // configure source files need to go in here.
        var config_man_allocation: Cache.Manifest = undefined;
        const config_man: ?*Cache.Manifest = switch (options.cache_poison) {
            .pure, .disallowed, .ignored => m: {
                config_man_allocation = graph.cache.obtain();

                for (options.cached_passthru_configure) |i|
                    config_man_allocation.hash.addBytes(configure_argv[i]);

                if (target_arch_os_abi) |triple|
                    config_man_allocation.hash.addBytes(triple);

                // Prevents a `zig build` from getting a false positive cache hit following
                // a `zig build --cache-poison=ignored`.
                config_man_allocation.hash.add(options.cache_poison == .ignored);

                break :m &config_man_allocation;
            },
            .poisoned => null,
        };
        defer if (config_man) |man| man.deinit();

        // We want to release all the locks before executing the child process, so we make a nice
        // big block here to ensure the cleanup gets run when we extract out our argv.
        {
            {
                const fetch_prog_node = options.parent_progress_node.start("Fetch Packages", 0);
                defer fetch_prog_node.end();

                // Reset fork match counts.
                for (fork_set.keys()) |*fork| fork.uses = 0;

                var job_queue: Package.Fetch.JobQueue = .{
                    .io = io,
                    .http_client = &http_client,
                    .global_cache = graph.global_cache_root,
                    .local_storage = &.{
                        .cache_root = .{ .root_dir = graph.local_cache_root },
                        .pkg_root = options.pkg_root,
                    },
                    .recursive = true,
                    .debug_hash = false,
                    .unlazy_set = unlazy_set,
                    .fork_set = fork_set,
                    .mode = options.fetch_mode,
                    .prog_node = fetch_prog_node,
                    .read_only = options.system_pkg_dir_path != null,
                };
                defer job_queue.deinit();

                if (options.system_pkg_dir_path == null) {
                    try http_client.initDefaultProxies(arena, &graph.environ_map);
                }

                try job_queue.all_fetches.ensureUnusedCapacity(gpa, 1);
                try job_queue.table.ensureUnusedCapacity(gpa, 1);

                const phantom_package_root: Cache.Path = .{ .root_dir = options.build_root.directory };

                var fetch: Package.Fetch = .{
                    .arena = std.heap.ArenaAllocator.init(gpa),
                    .location = .{ .relative_path = phantom_package_root },
                    .location_tok = 0,
                    .hash_tok = .none,
                    .name_tok = 0,
                    .lazy_status = .eager,
                    .remote_package_root = phantom_package_root,
                    .parent_package_root = phantom_package_root,
                    .parent_manifest_ast = null,
                    .prog_node = fetch_prog_node,
                    .job_queue = &job_queue,
                    .omit_missing_hash_error = true,
                    .allow_missing_paths_field = false,
                    .use_latest_commit = false,

                    .package_root = undefined,
                    .error_bundle = undefined,
                    .manifest = undefined,
                    .manifest_ast = undefined,
                    .have_manifest = false,
                    .computed_hash = undefined,
                    .has_build_zig = true,
                    .oom_flag = false,
                    .latest_commit = null,

                    .cli_module = build_mod,
                };

                job_queue.all_fetches.appendAssumeCapacity(&fetch);

                job_queue.table.putAssumeCapacityNoClobber(
                    Package.Fetch.relativePathDigest(phantom_package_root, graph.global_cache_root),
                    &fetch,
                );

                job_queue.group.async(io, Package.Fetch.workerRun, .{ &fetch, "root" });
                try job_queue.group.await(io);

                {
                    // Ensure that forks were actually used. This is done
                    // before printing manifest errors because using a fork can
                    // prevent them.
                    var any_unused = false;
                    for (fork_set.keys()) |*fork| {
                        if (fork.uses == 0) {
                            log.err("fork {f} matched no {s} packages", .{
                                fork.path, fork.manifest.name,
                            });
                            any_unused = true;
                        } else {
                            log.info("fork {f} matched {d} {s} packages", .{
                                fork.path, fork.uses, fork.manifest.name,
                            });
                        }
                    }
                    if (any_unused) return error.FailedButCacheIntact;
                }

                try job_queue.consolidateErrors();

                if (fetch.error_bundle.root_list.items.len > 0) {
                    var errors = try fetch.error_bundle.toOwnedBundle("");
                    errors.renderToStderr(io, .{}, options.color) catch process.exit(1);
                    return error.FailedButCacheIntact;
                }

                if (options.fetch_only) {
                    _ = io.lockStderr(&.{}, .no_color) catch {};
                    process.exit(0);
                }

                // Create the dependencies.zig file for configurer to
                // obtain via `@import("@dependencies")`.
                {
                    {
                        dependencies_source.clearRetainingCapacity();
                        var source_writer: Io.Writer.Allocating = .fromArrayList(gpa, &dependencies_source);
                        defer dependencies_source = source_writer.toArrayList();
                        job_queue.createDependenciesSource(&source_writer.writer) catch |err| switch (err) {
                            error.WriteFailed => return error.OutOfMemory,
                        };
                    }
                    // Atomically create the file in a directory named after the hash of its contents.
                    var hh: Cache.HashHelper = .{};
                    hh.addBytes(builtin.zig_version_string);
                    hh.addBytes(dependencies_source.items);
                    const hex_digest = hh.final();
                    const dependencies_zig_path: Path = .{
                        .root_dir = graph.local_cache_root,
                        .sub_path = try arena.print("o/{s}/dependencies.zig", .{&hex_digest}),
                    };
                    var atomic_file = try dependencies_zig_path.root_dir.handle.createFileAtomic(
                        io,
                        dependencies_zig_path.sub_path,
                        .{ .make_path = true, .replace = true },
                    );
                    defer atomic_file.deinit(io);
                    atomic_file.file.writeStreamingAll(io, dependencies_source.items) catch |err|
                        fatal("writing dependencies.zig contents: {t}", .{err});
                    atomic_file.replace(io) catch |err|
                        fatal("replacing {f}: {t}", .{ dependencies_zig_path, err });

                    deps_mod.root_path = try dependencies_zig_path.toString(arena);
                }

                {
                    // Add a CliModule for each package's build.zig.
                    const hashes = job_queue.table.keys();
                    const fetches = job_queue.table.values();
                    try deps_mod.deps.ensureUnusedCapacity(arena, @intCast(hashes.len));
                    for (hashes, fetches) |*hash, f| {
                        if (f == &fetch) {
                            // The first one is a dummy package for the current project.
                            continue;
                        }
                        if (!f.has_build_zig)
                            continue;
                        const hash_slice = try arena.dupe(u8, hash.toSlice());

                        const m = try arena.create(CliModule);
                        m.* = .{
                            .root_path = try f.package_root.toString(arena),
                            .name = hash_slice,
                        };
                        deps_mod.deps.putAssumeCapacityNoClobber(hash_slice, m);
                        f.cli_module = m;
                    }

                    // Each build.zig module needs access to each of its
                    // dependencies' build.zig modules by name.
                    for (fetches) |f| {
                        const mod = f.cli_module orelse continue;
                        if (!f.have_manifest) continue;
                        const man = &f.manifest;
                        const dep_names = man.dependencies.keys();
                        try mod.deps.ensureUnusedCapacity(arena, @intCast(dep_names.len));
                        for (dep_names, man.dependencies.values()) |name, dep| {
                            const dep_digest = Package.Fetch.depDigest(
                                f.package_root,
                                graph.global_cache_root,
                                dep,
                            ) orelse continue;
                            const dep_mod = job_queue.table.get(dep_digest).?.cli_module orelse continue;
                            const name_cloned = try arena.dupe(u8, name);
                            mod.deps.putAssumeCapacityNoClobber(name_cloned, dep_mod);
                        }
                    }
                }

                // Lower module dependencies to CLI argv.
                build_configurer_argv.shrinkRetainingCapacity(argv_deps_index);
                for (deps_mod.deps.values()) |dep| {
                    try build_configurer_argv.ensureUnusedCapacity(gpa, 2 * dep.deps.count() + 1);
                    for (dep.deps.keys(), dep.deps.values()) |name, sub| {
                        build_configurer_argv.appendAssumeCapacity("--dep");
                        if (mem.eql(u8, name, sub.name)) {
                            build_configurer_argv.appendAssumeCapacity(sub.name);
                        } else {
                            build_configurer_argv.appendAssumeCapacity(try arena.print("{s}={s}", .{
                                name, sub.name,
                            }));
                        }
                    }
                    build_configurer_argv.appendAssumeCapacity(try arena.print("-M{s}={s}/{s}", .{
                        dep.name, dep.root_path, std.zig.build_zig_basename,
                    }));
                }
                try deps_mod.lower(arena, gpa, &build_configurer_argv);
                try build_mod.lower(arena, gpa, &build_configurer_argv);

                try build_configurer_argv.append(gpa, "--listen=-");
            }

            const compile_prog_node = options.parent_progress_node.start("Compile Configure Script", 0);
            defer compile_prog_node.end();

            if (config_man) |man| {
                if (try man.hit(compile_prog_node)) {
                    const digest = man.final();
                    break :cp .{
                        .{
                            .root_dir = graph.local_cache_root,
                            .sub_path = try arena.print("c/{s}", .{&digest}),
                        },
                        man.toOwnedLock(),
                    };
                }
            }
            const configure_exe_path: Path = if (std.zig.buildExeSubprocess(gpa, io, .{
                .argv = build_configurer_argv.items,
                .cache_root = graph.local_cache_root,
                .root_name = configurer_exe_name,
                .environ_map = &graph.environ_map,
                .cache_manifest = config_man,
                .arch_os_abi = target_arch_os_abi,
                .progress_node = compile_prog_node,
                .skip_log_cmdline_on_compile_errors = !graph.verbose,
            })) |r| r.path else |err| return err;
            defer gpa.free(configure_exe_path.sub_path);

            configure_argv[0] = try configure_exe_path.toString(arena);
        }

        if (!process.can_spawn) {
            fatal("cannot spawn command on {t}: {f}", .{ native_os, @as(std.zig.SubprocessCommand, .{
                .argv = configure_argv,
            }) });
        }

        const config_tmp_path: Path = .{
            .root_dir = graph.local_cache_root,
            .sub_path = try arena.print("tmp" ++ Dir.path.sep_str ++ "{x}", .{randInt(io, u64)}),
        };
        const config_tmp_file: Io.File = try config_tmp_path.root_dir.handle.createFile(
            io,
            config_tmp_path.sub_path,
            .{ .read = true, .exclusive = true },
        );
        defer config_tmp_file.close(io);

        const term = term: {
            const child_node = options.parent_progress_node.start("Run Configure Script", 0);
            defer child_node.end();
            var child = process.spawn(io, .{
                .argv = configure_argv,
                .stdout = .{ .file = config_tmp_file },
                .progress_node = child_node,
            }) catch |err| fatal("failed to spawn configure script {q}: {t}", .{ configure_argv[0], err });
            defer child.kill(io);
            break :term child.wait(io) catch |err|
                fatal("failed to wait configure script {q}: {t}", .{ configure_argv[0], err });
        };
        if (!term.success()) {
            // Failure to produce the configuration file.
            fatal("configure command {f}: {f}", .{ term, @as(std.zig.SubprocessCommand, .{
                .argv = configure_argv,
            }) });
        }
        // Even though the file is designed to be sent directly to make
        // runner, we must load it now because:
        // * If it contains additional file dependencies, we need to
        //   add them to `config_man` before obtaining the final digest.
        // * If it contains a set of lazy packages that need to be
        //   fetched, we need to fetch those now and re-run configure.
        var configuration = Configuration.loadFile(arena, io, config_tmp_file) catch |err|
            fatal("failed to load configuration file {f}: {t}", .{ config_tmp_path, err });

        if (configuration.unlazy_deps.len != 0) {
            var any_errors = false;
            for (configuration.unlazy_deps) |hash_string| {
                const hash = hash_string.slice(&configuration);
                assert(hash.len != 0);
                if (hash.len > Package.Hash.max_len) {
                    log.err("invalid digest (length {d} exceeds maximum): {q}", .{ hash.len, hash });
                    any_errors = true;
                    continue;
                }
                log.info("fetching lazy dependency {s}", .{hash});
                try unlazy_set.put(arena, .fromSlice(hash), {});
            }
            if (any_errors) return error.FailedButCacheIntact;
            if (options.system_pkg_dir_path) |p| {
                // In this mode, the system needs to provide these packages; they
                // cannot be fetched by Zig.
                const s = Dir.path.sep_str;
                for (unlazy_set.keys()) |*hash| {
                    log.err("lazy dependency package not found: {s}" ++ s ++ "{s}", .{ p, hash.toSlice() });
                }
                log.info("remote package fetching disabled due to --system mode", .{});
                log.info("dependencies might be avoidable depending on build configuration", .{});
                return error.FailedButCacheIntact;
            }
            continue :cp;
        }

        if (config_man) |man| for (configuration.path_deps) |path_dep| {
            switch (path_dep.flags.mode) {
                .directory => {}, // TODO
                .contents => try man.addPathPost(confPathDepToCachePath(graph, &configuration, path_dep)),
                .metadata => {}, // TODO
            }
        };

        // If it is poisoned, there is no point in moving it to cached
        // location. Just leave it in the tmp directory.
        if (configuration.poisoned) {
            break :cp .{ config_tmp_path, null };
        } else {
            const man = config_man.?;
            const digest = man.final();
            const final_path: Path = .{
                .root_dir = graph.local_cache_root,
                .sub_path = try arena.print("c/{s}", .{&digest}),
            };
            Io.Dir.rename(
                config_tmp_path.root_dir.handle,
                config_tmp_path.sub_path,
                final_path.root_dir.handle,
                final_path.sub_path,
                io,
            ) catch |err| retry: {
                const e = switch (err) {
                    error.FileNotFound => e: {
                        const dir_path = final_path.dirname().?;
                        dir_path.root_dir.handle.createDirPath(io, dir_path.sub_path) catch |e|
                            fatal("failed to create directory {f}: {t}", .{ dir_path, e });
                        if (Io.Dir.rename(
                            config_tmp_path.root_dir.handle,
                            config_tmp_path.sub_path,
                            final_path.root_dir.handle,
                            final_path.sub_path,
                            io,
                        )) |_| break :retry else |e| break :e e;
                    },
                    else => |e| e,
                };
                fatal("failed to rename configuration file from {f} into {f}: {t}", .{
                    config_tmp_path, final_path, e,
                });
            };
            man.writeManifest() catch |err| log.warn("failed to write cache manifest: {t}", .{err});
            break :cp .{ final_path, man.toOwnedLock() };
        }
    };
    // Hang on to the configuration file lock until we finish loading the configuration file.
    defer if (configuration_lock) |*l| l.release(io);

    switch (options.print_configuration) {
        .path => {
            initStdoutWriter(io).print("{f}\n", .{configuration_path}) catch
                fatal("failed printing cache file path: {t}", .{stdout_writer_allocation.err.?});
            stdout_writer_allocation.flush() catch |err|
                fatal("failed printing cache file path: {t}", .{err});
            _ = io.lockStderr(&.{}, .no_color) catch {};
            process.exit(0);
        },
        .none, .zon => {},
    }

    const configuration = c: {
        var file = configuration_path.root_dir.handle.openFile(io, configuration_path.sub_path, .{}) catch |err|
            fatal("failed to open configuration file {f}: {t}", .{ configuration_path, err });
        defer file.close(io);
        break :c Configuration.loadFile(arena, io, file) catch |err|
            fatal("failed to load configuration file {f}: {t}", .{ configuration_path, err });
    };
    // Technically if the configuration is marked as poisoned, we could
    // already delete the file now, but we leave it around in case the
    // maker process fails or crashes and it's helpful to be able to repeat
    // execution of the command line or otherwise inspect the configuration file.
    const c = &configuration;
    var top_level_steps: std.array_hash_map.String(Configuration.Step.Index) = .empty;
    for (configuration.steps, 0..) |*conf_step, step_index_usize| {
        if (conf_step.owner != .root) continue;
        const step_index: Configuration.Step.Index = @fromBackingInt(@intCast(step_index_usize));
        const flags = conf_step.flags(c);
        switch (flags.tag) {
            .top_level => {
                const name = step_index.ptr(c).name.slice(c);
                try top_level_steps.put(arena, name, step_index);
            },
            else => {},
        }
    }
    for (c.search_prefixes) |search_prefix| {
        try graph.search_prefixes.append(arena, search_prefix.slice(c));
    }
    return .{
        .configuration = configuration,
        .top_level_steps = top_level_steps,
        .path = configuration_path,
    };
}