computeHash

fn computeHash(f: *Fetch, pkg_path: Path, filter: Filter) RunError!ComputedHash

File

Code

fn computeHash(f: *Fetch, pkg_path: Path, filter: Filter) RunError!ComputedHash {
    const io = f.job_queue.io;
    // All the path name strings need to be in memory for sorting.
    const arena = f.arena.allocator();
    const gpa = f.arena.child_allocator;
    const eb = &f.error_bundle;
    const root_dir = pkg_path.root_dir.handle;

    // Collect all files, recursively, then sort.
    var all_files = std.array_list.Managed(*HashedFile).init(gpa);
    defer all_files.deinit();

    var deleted_files = std.array_list.Managed(*DeletedFile).init(gpa);
    defer deleted_files.deinit();

    // Track directories which had any files deleted from them so that empty directories
    // can be deleted.
    var sus_dirs: std.array_hash_map.String(void) = .empty;
    defer sus_dirs.deinit(gpa);

    var walker = try root_dir.walk(gpa);
    defer walker.deinit();

    // Total number of bytes of file contents included in the package.
    var total_size: u64 = 0;

    {
        // The final hash will be a hash of each file hashed independently. This
        // allows hashing in parallel.
        var group: Io.Group = .init;
        defer group.cancel(io);

        while (walker.next(io) catch |err| {
            try eb.addRootErrorMessage(.{ .msg = try eb.printString(
                "unable to walk temporary directory '{f}': {t}",
                .{ pkg_path, err },
            ) });
            return error.FetchFailed;
        }) |entry| {
            if (entry.kind == .directory) continue;

            const entry_pkg_path = stripRoot(entry.path, pkg_path.sub_path);
            if (!filter.includePath(entry_pkg_path)) {
                // Delete instead of including in hash calculation.
                const fs_path = try arena.dupe(u8, entry.path);

                // Also track the parent directory in case it becomes empty.
                if (fs.path.dirname(fs_path)) |parent|
                    try sus_dirs.put(gpa, parent, {});

                const deleted_file = try arena.create(DeletedFile);
                deleted_file.* = .{
                    .fs_path = fs_path,
                    .failure = undefined, // to be populated by the worker
                };
                group.async(io, workerDeleteFile, .{ io, root_dir, deleted_file });
                try deleted_files.append(deleted_file);
                continue;
            }

            const kind: HashedFile.Kind = switch (entry.kind) {
                .directory => unreachable,
                .file => .file,
                .sym_link => .link,
                else => return f.fail(f.location_tok, try eb.printString(
                    "package contains '{s}' which has illegal file type '{t}'",
                    .{ entry.path, entry.kind },
                )),
            };

            if (std.mem.eql(u8, entry_pkg_path, std.zig.build_zig_basename))
                f.has_build_zig = true;

            const fs_path = try arena.dupe(u8, entry.path);
            const hashed_file = try arena.create(HashedFile);
            hashed_file.* = .{
                .fs_path = fs_path,
                .normalized_path = try normalizePathAlloc(arena, entry_pkg_path),
                .kind = kind,
                .hash = undefined, // to be populated by the worker
                .failure = undefined, // to be populated by the worker
                .size = undefined, // to be populated by the worker
            };
            group.async(io, workerHashFile, .{ io, root_dir, hashed_file });
            try all_files.append(hashed_file);
        }

        try group.await(io);
    }

    {
        // Sort by length, descending, so that child directories get removed first.
        sus_dirs.sortUnstable(@as(struct {
            keys: []const []const u8,
            pub fn lessThan(ctx: @This(), a_index: usize, b_index: usize) bool {
                return ctx.keys[b_index].len < ctx.keys[a_index].len;
            }
        }, .{ .keys = sus_dirs.keys() }));

        // During this loop, more entries will be added, so we must loop by index.
        var i: usize = 0;
        while (i < sus_dirs.count()) : (i += 1) {
            const sus_dir = sus_dirs.keys()[i];
            root_dir.deleteDir(io, sus_dir) catch |err| switch (err) {
                error.DirNotEmpty => continue,
                error.FileNotFound => continue,
                else => |e| {
                    try eb.addRootErrorMessage(.{ .msg = try eb.printString(
                        "unable to delete empty directory '{s}': {s}",
                        .{ sus_dir, @errorName(e) },
                    ) });
                    return error.FetchFailed;
                },
            };
            if (fs.path.dirname(sus_dir)) |parent| {
                try sus_dirs.put(gpa, parent, {});
            }
        }
    }

    std.mem.sortUnstable(*HashedFile, all_files.items, {}, HashedFile.lessThan);

    var hasher = Package.Hash.Algo.init(.{});
    var any_failures = false;
    for (all_files.items) |hashed_file| {
        hashed_file.failure catch |err| {
            any_failures = true;
            try eb.addRootErrorMessage(.{
                .msg = try eb.printString("unable to hash '{s}': {s}", .{
                    hashed_file.fs_path, @errorName(err),
                }),
            });
        };
        hasher.update(&hashed_file.hash);
        total_size += hashed_file.size;
    }
    for (deleted_files.items) |deleted_file| {
        deleted_file.failure catch |err| {
            any_failures = true;
            try eb.addRootErrorMessage(.{
                .msg = try eb.printString("failed to delete excluded path '{s}' from package: {s}", .{
                    deleted_file.fs_path, @errorName(err),
                }),
            });
        };
    }

    if (any_failures) return error.FetchFailed;

    if (f.job_queue.debug_hash) {
        assert(!f.job_queue.recursive);
        // Print something to stdout that can be text diffed to figure out why
        // the package hash is different.
        dumpHashInfo(io, all_files.items) catch |err|
            std.process.fatal("unable to write to stdout: {t}", .{err});
    }

    return .{
        .digest = hasher.finalResult(),
        .total_size = total_size,
    };
}