make

pub fn make(
    run: *Run,
    run_index: Configuration.Step.Index,
    maker: *Maker,
    progress_node: std.Progress.Node,
) Step.ExtendedMakeError!void

File

Code

pub fn make(
    run: *Run,
    run_index: Configuration.Step.Index,
    maker: *Maker,
    progress_node: std.Progress.Node,
) Step.ExtendedMakeError!void {
    const graph = maker.graph;
    const gpa = maker.gpa;
    const step = maker.stepByIndex(run_index);
    const io = graph.io;
    const conf = &maker.scanned_config.configuration;
    const conf_step = run_index.ptr(conf);
    const conf_run = conf_step.extended.get(conf.extra).run;
    const cache_root = graph.local_cache_root;

    var arena_allocator: std.heap.ArenaAllocator = .init(gpa);
    defer arena_allocator.deinit();
    const arena = arena_allocator.allocator();

    var argv_list: std.ArrayList([]const u8) = .empty;
    defer argv_list.deinit(gpa);

    var output_placeholders: std.ArrayList(IndexedOutput) = .empty;
    defer output_placeholders.deinit(gpa);

    var man = graph.cache.obtain();
    defer man.deinit();

    if (conf_run.environ_map.value) |environ_map_index| {
        const environ_map = environ_map_index.get(conf);
        for (environ_map.keys.slice(conf), environ_map.values.slice(conf)) |key, value| {
            man.hash.addBytesZ(key.slice(conf));
            man.hash.addBytesZ(value.slice(conf));
        }
    }

    for (conf_run.preopen_names.slice, conf_run.preopen_paths.slice) |name, path| {
        man.hash.addBytesZ(name.slice(conf));
        const cwd_path = try maker.resolveLazyPathIndex(arena, path, run_index);
        man.hash.addBytes(try cwd_path.toString(arena));
    }

    man.hash.add(graph.fuzzing);
    man.hash.add(conf_run.flags.color);
    man.hash.add(conf_run.flags.disable_zig_progress);

    var any_dep_files = false;
    var any_output_args = false;
    var any_cli_positionals = false;

    for (conf_run.args.slice) |arg_index| {
        const arg = arg_index.get(conf);
        try argv_list.ensureUnusedCapacity(gpa, 1);
        switch (arg.flags.tag) {
            .string => {
                const prefix = arg.prefix.value.?.slice(conf);
                argv_list.appendAssumeCapacity(prefix);
                man.hash.addBytesZ(prefix);
            },
            .path_file => {
                const prefix = if (arg.prefix.value) |p| p.slice(conf) else "";
                const suffix = if (arg.suffix.value) |p| p.slice(conf) else "";
                const file_path = try maker.resolveLazyPathIndex(arena, arg.path.value.?, run_index);
                argv_list.appendAssumeCapacity(try mem.concat(arena, u8, &.{
                    prefix, try convertPathArg(arena, run_index, maker, file_path, arg.flags.make_absolute), suffix,
                }));
                man.hash.add(arg.flags.make_absolute);
                man.hash.addBytesZ(prefix);
                man.hash.addBytesZ(suffix);
                _ = try man.addFilePath(file_path, null);
            },
            .path_directory => {
                const prefix = if (arg.prefix.value) |p| p.slice(conf) else "";
                const suffix = if (arg.suffix.value) |p| p.slice(conf) else "";
                const file_path = try maker.resolveLazyPathIndex(arena, arg.path.value.?, run_index);
                const resolved_arg = try mem.concat(arena, u8, &.{
                    prefix, try convertPathArg(arena, run_index, maker, file_path, arg.flags.make_absolute), suffix,
                });
                argv_list.appendAssumeCapacity(resolved_arg);
                man.hash.add(arg.flags.make_absolute);
                man.hash.addBytes(resolved_arg);
            },
            .file_content => {
                const prefix = if (arg.prefix.value) |p| p.slice(conf) else "";
                const suffix = if (arg.suffix.value) |p| p.slice(conf) else "";
                const file_path = try maker.resolveLazyPathIndex(arena, arg.path.value.?, run_index);

                var result: std.Io.Writer.Allocating = .init(arena);
                result.writer.writeAll(prefix) catch return error.OutOfMemory;

                const file = file_path.root_dir.handle.openFile(io, file_path.sub_path, .{}) catch |err|
                    return step.fail(maker, "unable to open input file {f}: {t}", .{ file_path, err });
                defer file.close(io);

                var file_reader = file.reader(io, &.{});
                _ = file_reader.interface.streamRemaining(&result.writer) catch |err| switch (err) {
                    error.ReadFailed => switch (file_reader.err.?) {
                        error.Canceled => |e| return e,
                        else => |e| return step.fail(maker, "failed to read from {f}: {t}", .{ file_path, e }),
                    },
                    error.WriteFailed => return error.OutOfMemory,
                };
                result.writer.writeAll(suffix) catch return error.OutOfMemory;

                argv_list.appendAssumeCapacity(result.written());
                man.hash.addBytesZ(prefix);
                man.hash.addBytesZ(suffix);
                _ = try man.addFilePath(file_path, null);
            },
            .artifact => {
                const prefix = if (arg.prefix.value) |p| p.slice(conf) else "";
                const suffix = if (arg.suffix.value) |p| p.slice(conf) else "";
                const producer_index = arg.producer.value.?;
                const producer_step = producer_index.ptr(conf);
                const producer = producer_step.extended.get(conf.extra).compile;
                const producer_make_comp_step = maker.stepByIndex(producer_index);
                const producer_make_comp = &producer_make_comp_step.extended.compile;

                const file_path = producer_make_comp.installed_path orelse maker.generatedPath(producer.generated_bin.value.?).*;

                argv_list.appendAssumeCapacity(try mem.concat(arena, u8, &.{
                    prefix, try convertPathArg(arena, run_index, maker, file_path, arg.flags.make_absolute), suffix,
                }));

                man.hash.add(arg.flags.make_absolute);
                man.hash.addBytesZ(prefix);
                man.hash.addBytesZ(suffix);
                _ = try man.addFilePath(file_path, null);
            },
            .output_file, .output_directory => {
                const prefix = if (arg.prefix.value) |p| p.slice(conf) else "";
                const suffix = if (arg.suffix.value) |p| p.slice(conf) else "";
                const basename = arg.basename.value.?.slice(conf);

                man.hash.add(arg.flags.make_absolute);
                man.hash.addBytesZ(prefix);
                man.hash.addBytesZ(basename);
                man.hash.addBytesZ(suffix);
                man.hash.add(arg.flags.dep_file);

                any_dep_files = any_dep_files or arg.flags.dep_file;
                any_output_args = true;

                // Add a placeholder into the argument list because we need the
                // manifest hash to be updated with all arguments before the
                // object directory is computed.
                try output_placeholders.append(gpa, .{
                    .index = @intCast(argv_list.items.len),
                    .arg_index = arg_index,
                });
                argv_list.items.len += 1;
            },
            .passthru => {
                any_cli_positionals = true;
                if (maker.run_args) |run_args| {
                    try argv_list.appendSlice(gpa, run_args);
                    man.hash.addListOfBytes(run_args);
                }
            },
        }
    }

    man.hash.add(conf_run.flags.test_runner_mode);
    if (conf_run.flags.test_runner_mode) {
        const cache_dir_string = try convertPathArg(arena, run_index, maker, .{ .root_dir = cache_root }, false);

        try argv_list.ensureUnusedCapacity(gpa, 3);
        argv_list.appendAssumeCapacity(try arena.print("--cache-dir={s}", .{cache_dir_string}));
        argv_list.appendAssumeCapacity(try arena.print("--seed=0x{x}", .{graph.random_seed}));
        argv_list.appendAssumeCapacity("--listen=-");
    }

    switch (conf_run.stdin.u) {
        .bytes => |bytes| {
            man.hash.addBytes(bytes.slice(conf));
        },
        .lazy_path => |lazy_path| {
            const file_path = try maker.resolveLazyPathIndex(arena, lazy_path, run_index);
            _ = try man.addFilePath(file_path, null);
        },
        .none => {},
    }

    if (conf_run.captured_stdout.value) |captured| {
        man.hash.addBytes(captured.basename.slice(conf));
        man.hash.add(conf_run.flags.stdout_trim_whitespace);
    }

    if (conf_run.captured_stderr.value) |captured| {
        man.hash.addBytes(captured.basename.slice(conf));
        man.hash.add(conf_run.flags.stderr_trim_whitespace);
    }

    switch (conf_run.flags.stdio) {
        .infer_from_args, .inherit, .zig_test => {},
        .check => {
            man.hash.addBytes(if (conf_run.expect_stderr_exact.value) |bytes| bytes.slice(conf) else "");
            man.hash.addBytes(if (conf_run.expect_stdout_exact.value) |bytes| bytes.slice(conf) else "");
            for (conf_run.expect_stderr_match.slice) |bytes| man.hash.addBytes(bytes.slice(conf));
            for (conf_run.expect_stdout_match.slice) |bytes| man.hash.addBytes(bytes.slice(conf));
            man.hash.add(conf_run.flags2.expect_term_status);
            man.hash.addOptional(conf_run.expect_term_value.value);
        },
    }

    for (conf_run.file_inputs.slice) |lazy_path| {
        const file_path = try maker.resolveLazyPathIndex(arena, lazy_path, run_index);
        _ = try man.addFilePath(file_path, null);
    }

    if (conf_run.cwd.value) |lazy_path| {
        const cwd_path = try maker.resolveLazyPathIndex(arena, lazy_path, run_index);
        _ = man.hash.addBytes(try cwd_path.toString(arena));
    }

    // Whether the Run step has side effects *other than* updating the output arguments.
    // When fuzzing we need to always run the test runner to populate fuzz_tests.
    const has_side_effects = graph.fuzzing or conf_run.flags.has_side_effects or any_cli_positionals or
        switch (conf_run.flags.stdio) {
            .infer_from_args => !any_output_args and
                conf_run.captured_stdout.value == null and
                conf_run.captured_stderr.value == null,
            .inherit => true,
            .check, .zig_test => false,
        };

    if (!has_side_effects and try step.cacheHitWatched(maker, &man, progress_node)) {
        // Cache hit; skip running command.
        const digest = man.final();
        try populateGeneratedStdIo(maker, &conf_run, cache_root, &digest);
        try populateGeneratedPaths(maker, output_placeholders.items, cache_root, &digest);
        step.result_cached = true;
        return;
    }

    if (!any_dep_files) {
        // We already know the final output paths; use them directly.
        const digest = if (has_side_effects) man.hash.final() else man.final();
        const output_dir_path = "o" ++ Dir.path.sep_str ++ &digest;
        try populateGeneratedStdIo(maker, &conf_run, cache_root, &digest);
        try populateGeneratedPathsCreateDirs(arena, run_index, maker, output_dir_path, output_placeholders.items, argv_list.items);
        try runCommand(arena, run, run_index, maker, progress_node, argv_list.items, has_side_effects, output_dir_path, null);
        if (!has_side_effects) try step.writeManifestAndWatch(maker, &man);
        return;
    }

    // We do not know the final output paths yet; use temporary directory to run the command.
    var rand_int: u64 = undefined;
    io.random(@ptrCast(&rand_int));
    const tmp_dir_path = "tmp" ++ Dir.path.sep_str ++ std.fmt.hex(rand_int);

    try populateGeneratedPathsCreateDirs(arena, run_index, maker, tmp_dir_path, output_placeholders.items, argv_list.items);
    try runCommand(arena, run, run_index, maker, progress_node, argv_list.items, has_side_effects, tmp_dir_path, null);

    for (output_placeholders.items) |placeholder| {
        const arg = placeholder.arg_index.get(conf);
        switch (arg.flags.tag) {
            .output_file => if (arg.flags.dep_file) {
                const generated_path = maker.generatedPath(arg.generated.value.?).*;
                const result = if (has_side_effects)
                    man.addDepFile(generated_path.root_dir.handle, generated_path.sub_path)
                else
                    man.addDepFilePost(generated_path.root_dir.handle, generated_path.sub_path);
                result catch |err| switch (err) {
                    error.OutOfMemory, error.Canceled => |e| return e,
                    else => |e| return step.fail(maker, "failed adding to cache the file {f}: {t}", .{
                        generated_path, e,
                    }),
                };
            },
            .output_directory => continue,
            else => unreachable,
        }
    }

    const digest = if (has_side_effects) man.hash.final() else man.final();

    const any_output = output_placeholders.items.len > 0 or
        conf_run.captured_stdout.value != null or conf_run.captured_stderr.value != null;

    if (any_output) {
        // Rename into place.
        const tmp_path: Path = .{ .root_dir = cache_root, .sub_path = tmp_dir_path };
        const dst_path: Path = .{ .root_dir = cache_root, .sub_path = "o" ++ Dir.path.sep_str ++ &digest };
        Dir.rename(
            tmp_path.root_dir.handle,
            tmp_path.sub_path,
            dst_path.root_dir.handle,
            dst_path.sub_path,
            io,
        ) catch |err| switch (err) {
            error.DirNotEmpty => {
                dst_path.root_dir.handle.deleteTree(io, dst_path.sub_path) catch |del_err|
                    return step.fail(maker, "failed to remove tree {f}: {t}", .{ dst_path, del_err });

                Dir.rename(
                    tmp_path.root_dir.handle,
                    tmp_path.sub_path,
                    dst_path.root_dir.handle,
                    dst_path.sub_path,
                    io,
                ) catch |retry_err| return step.fail(maker, "failed to rename directory {f} to {f}: {t}", .{
                    tmp_path, dst_path, retry_err,
                });
            },
            else => return step.fail(maker, "failed to rename directory {f} to {f}: {t}", .{
                tmp_path, dst_path, err,
            }),
        };
    }

    if (!has_side_effects) try step.writeManifestAndWatch(maker, &man);

    try populateGeneratedStdIo(maker, &conf_run, cache_root, &digest);
    try populateGeneratedPaths(maker, output_placeholders.items, cache_root, &digest);

    // The utility functions that spawn the child process must unconditionally allocate
    // the failed command because at that point it is not known whether the step will
    // pass or fail based on the process termination. Here we free the memory since
    // the step has succeeded.
    step.clearFailedCommand(gpa);
}