evalZigTest

fn evalZigTest(
    run: *Run,
    run_index: Configuration.Step.Index,
    maker: *Maker,
    progress_node: std.Progress.Node,
    spawn_options: process.SpawnOptions,
    fuzz_context: ?FuzzContext,
) !void

File

Code

fn evalZigTest(
    run: *Run,
    run_index: Configuration.Step.Index,
    maker: *Maker,
    progress_node: std.Progress.Node,
    spawn_options: process.SpawnOptions,
    fuzz_context: ?FuzzContext,
) !void {
    if (fuzz_context != null) {
        try evalFuzzTest(run, run_index, progress_node, spawn_options, fuzz_context.?);
        return;
    }

    const graph = maker.graph;
    const gpa = maker.gpa;
    const io = graph.io;
    const step = maker.stepByIndex(run_index);

    // We will update this every time a child runs.
    step.result_peak_rss = 0;

    var test_results: Step.TestResults = .{
        .test_count = 0,
        .skip_count = 0,
        .fail_count = 0,
        .crash_count = 0,
        .timeout_count = 0,
        .leak_count = 0,
        .log_err_count = 0,
    };
    var test_metadata: ?TestMetadata = null;

    while (true) {
        var child = try process.spawn(io, spawn_options);
        var multi_reader_buffer: Io.File.MultiReader.Buffer(2) = undefined;
        var multi_reader: Io.File.MultiReader = undefined;
        multi_reader.init(gpa, io, multi_reader_buffer.toStreams(), &.{ child.stdout.?, child.stderr.? });
        var child_killed = false;
        defer if (!child_killed) {
            child.kill(io);
            multi_reader.deinit();
            step.result_peak_rss = @max(
                step.result_peak_rss,
                child.resource_usage_statistics.getMaxRss() orelse 0,
            );
        };

        switch (try waitZigTest(
            graph.arena,
            run,
            run_index,
            maker,
            &child,
            progress_node,
            &multi_reader,
            &test_metadata,
            &test_results,
        )) {
            .write_failed => |err| {
                // The runner unexpectedly closed a stdio pipe, which means a crash. Make sure we've captured
                // all available stderr to make our error output as useful as possible.
                const stderr_fr = multi_reader.fileReader(1);
                while (stderr_fr.interface.fillMore()) |_| {} else |e| switch (e) {
                    error.ReadFailed => return stderr_fr.err.?,
                    error.EndOfStream => {},
                }
                step.takeResultStderr(gpa, try multi_reader.toOwnedSlice(1));

                // Clean up everything and wait for the child to exit.
                child.stdin.?.close(io);
                child.stdin = null;
                multi_reader.deinit();
                child_killed = true;
                const term = try child.wait(io);
                step.result_peak_rss = @max(
                    step.result_peak_rss,
                    child.resource_usage_statistics.getMaxRss() orelse 0,
                );

                // The individual unit test results are irrelevant: the test runner itself broke!
                // Fail immediately without populating `s.test_results`.
                return step.fail(maker, "unable to write stdin ({t}); test process unexpectedly {f}", .{
                    err, fmtTerm(term),
                });
            },
            .no_poll => |no_poll| {
                // This might be a success (we requested exit and the child dutifully closed stdout) or
                // a crash of some kind. Either way, the child will terminate by itself -- wait for it.
                const stderr_owned = try multi_reader.toOwnedSlice(1);
                var keep_stderr_owned = false;
                defer if (!keep_stderr_owned) gpa.free(stderr_owned);

                // Clean up everything and wait for the child to exit.
                child.stdin.?.close(io);
                child.stdin = null;
                multi_reader.deinit();
                child_killed = true;
                const term = try child.wait(io);
                step.result_peak_rss = @max(
                    step.result_peak_rss,
                    child.resource_usage_statistics.getMaxRss() orelse 0,
                );

                if (no_poll.active_test_index) |test_index| {
                    // A test was running, so this is definitely a crash. Report it against that
                    // test, and continue to the next test.
                    test_metadata.?.ns_per_test[test_index] = no_poll.ns_elapsed;
                    test_results.crash_count += 1;
                    try step.addError(maker, "'{s}' {f}{s}{s}", .{
                        test_metadata.?.testName(test_index),
                        fmtTerm(term),
                        if (stderr_owned.len != 0) " with stderr:\n" else "",
                        std.mem.trim(u8, stderr_owned, "\n"),
                    });
                    continue;
                }

                // Report an error if the child terminated uncleanly or if we were still trying to run more tests.
                step.takeResultStderr(gpa, stderr_owned);
                keep_stderr_owned = true;

                const tests_done = test_metadata != null and test_metadata.?.next_index == std.math.maxInt(u32);
                if (!tests_done or !termMatches(.{ .exited = 0 }, term)) {
                    // The individual unit test results are irrelevant: the test runner itself broke!
                    // Fail immediately without populating `s.test_results`.
                    return step.fail(maker, "test process unexpectedly {f}", .{fmtTerm(term)});
                }

                // We're done with all of the tests! Commit the test results and return.
                step.test_results = test_results;
                if (test_metadata) |tm| {
                    run.cached_test_metadata = tm.toCachedTestMetadata();
                    if (maker.web_server) |ws| {
                        if (graph.time_report) {
                            ws.updateTimeReportRunTest(
                                run_index,
                                &run.cached_test_metadata.?,
                                tm.ns_per_test,
                            );
                        }
                    }
                }
                return;
            },
            .timeout => |timeout| {
                const stderr_owned = try multi_reader.toOwnedSlice(1);
                var keep_stderr_owned = false;
                defer if (!keep_stderr_owned) gpa.free(stderr_owned);

                if (timeout.active_test_index) |test_index| {
                    // A test was running. Report the timeout against that test, and continue on to
                    // the next test.
                    test_metadata.?.ns_per_test[test_index] = timeout.ns_elapsed;
                    test_results.timeout_count += 1;
                    try step.addError(maker, "'{s}' timed out after {f}{s}{s}", .{
                        test_metadata.?.testName(test_index),
                        Io.Duration{ .nanoseconds = timeout.ns_elapsed },
                        if (stderr_owned.len != 0) " with stderr:\n" else "",
                        std.mem.trim(u8, stderr_owned, "\n"),
                    });
                    continue;
                }
                // Just log an error and let the child be killed.
                step.takeResultStderr(gpa, stderr_owned);
                keep_stderr_owned = true;

                // The individual unit test results in `results` are irrelevant: the test runner
                // is broken! Fail immediately without populating `s.test_results`.
                return step.fail(maker, "test runner failed to respond for {f}", .{
                    Io.Duration{ .nanoseconds = timeout.ns_elapsed },
                });
            },
        }
        comptime unreachable;
    }
}