Manifest

pub const Manifest = struct

File

Code

pub const Manifest = struct {
    cache: *Cache,
    /// Current state for incremental hashing.
    hash: HashHelper,
    manifest_file: ?Io.File,
    manifest_dirty: bool,
    /// Set this flag to true before calling hit() in order to indicate that
    /// upon a cache hit, the code using the cache will not modify the files
    /// within the cache directory. This allows multiple processes to utilize
    /// the same cache directory at the same time.
    want_shared_lock: bool = true,
    have_exclusive_lock: bool = false,
    // Indicate that we want isProblematicTimestamp to perform a filesystem write in
    // order to obtain a problematic timestamp for the next call. Calls after that
    // will then use the same timestamp, to avoid unnecessary filesystem writes.
    want_refresh_timestamp: bool = true,
    files: Files = .{},
    hex_digest: HexDigest,
    diagnostic: Diagnostic = .none,
    /// Keeps track of the last time we performed a file system write to observe
    /// what time the file system thinks it is, according to its own granularity.
    recent_problematic_timestamp: Io.Timestamp = .zero,

    pub const Diagnostic = union(enum) {
        none,
        manifest_create: Io.File.OpenError,
        manifest_read: Io.File.Reader.Error,
        manifest_lock: Io.File.LockError,
        file_open: FileOp,
        file_stat: FileOp,
        file_read: FileOp,
        file_hash: FileOp,

        pub const FileOp = struct {
            file_index: usize,
            err: anyerror,
        };
    };

    pub const Files = std.array_hash_map.Custom(File, void, FilesContext, false);

    pub const FilesContext = struct {
        pub fn hash(fc: FilesContext, file: File) u32 {
            _ = fc;
            return file.prefixed_path.hash();
        }

        pub fn eql(fc: FilesContext, a: File, b: File, b_index: usize) bool {
            _ = fc;
            _ = b_index;
            return a.prefixed_path.eql(b.prefixed_path);
        }
    };

    const FilesAdapter = struct {
        pub fn eql(context: @This(), a: PrefixedPath, b: File, b_index: usize) bool {
            _ = context;
            _ = b_index;
            return a.eql(b.prefixed_path);
        }

        pub fn hash(context: @This(), key: PrefixedPath) u32 {
            _ = context;
            return key.hash();
        }
    };

    /// Add a file as a dependency of process being cached. When `hit` is
    /// called, the file's contents will be checked to ensure that it matches
    /// the contents from previous times.
    ///
    /// Max file size will be used to determine the amount of space the file contents
    /// are allowed to take up in memory. If max_file_size is null, then the contents
    /// will not be loaded into memory.
    ///
    /// Returns the index of the entry in the `files` array list. You can use it
    /// to access the contents of the file after calling `hit()` like so:
    ///
    /// ```
    /// var file_contents = cache_hash.files.keys()[file_index].contents.?;
    /// ```
    pub fn addFilePath(m: *Manifest, file_path: Path, max_file_size: ?usize) !usize {
        return addOpenedFile(m, file_path, null, max_file_size);
    }

    /// Same as `addFilePath` except the file has already been opened.
    pub fn addOpenedFile(m: *Manifest, path: Path, handle: ?Io.File, max_file_size: ?usize) !usize {
        const gpa = m.cache.gpa;
        try m.files.ensureUnusedCapacity(gpa, 1);
        const resolved_path = try std.fs.path.resolve(gpa, &.{
            path.root_dir.path orelse ".",
            path.subPathOrDot(),
        });
        errdefer gpa.free(resolved_path);
        const prefixed_path = try m.cache.findPrefixResolved(resolved_path);
        return addFileInner(m, prefixed_path, handle, max_file_size);
    }

    /// Deprecated; use `addFilePath`.
    pub fn addFile(self: *Manifest, file_path: []const u8, max_file_size: ?usize) !usize {
        assert(self.manifest_file == null);

        const gpa = self.cache.gpa;
        try self.files.ensureUnusedCapacity(gpa, 1);
        const prefixed_path = try self.cache.findPrefix(file_path);
        errdefer gpa.free(prefixed_path.sub_path);

        return addFileInner(self, prefixed_path, null, max_file_size);
    }

    fn addFileInner(self: *Manifest, prefixed_path: PrefixedPath, handle: ?Io.File, max_file_size: ?usize) usize {
        const gop = self.files.getOrPutAssumeCapacityAdapted(prefixed_path, FilesAdapter{});
        if (gop.found_existing) {
            self.cache.gpa.free(prefixed_path.sub_path);
            gop.key_ptr.updateMaxSize(max_file_size);
            gop.key_ptr.updateHandle(handle);
            return gop.index;
        }
        gop.key_ptr.* = .{
            .prefixed_path = prefixed_path,
            .contents = null,
            .max_file_size = max_file_size,
            .stat = undefined,
            .bin_digest = undefined,
            .handle = handle,
        };

        self.hash.add(prefixed_path.prefix);
        self.hash.addBytes(prefixed_path.sub_path);

        return gop.index;
    }

    /// Deprecated, use `addOptionalFilePath`.
    pub fn addOptionalFile(self: *Manifest, optional_file_path: ?[]const u8) !void {
        self.hash.add(optional_file_path != null);
        const file_path = optional_file_path orelse return;
        _ = try self.addFile(file_path, null);
    }

    pub fn addOptionalFilePath(self: *Manifest, optional_file_path: ?Path) !void {
        self.hash.add(optional_file_path != null);
        const file_path = optional_file_path orelse return;
        _ = try self.addFilePath(file_path, null);
    }

    pub fn addListOfFiles(self: *Manifest, list_of_files: []const []const u8) !void {
        self.hash.add(list_of_files.len);
        for (list_of_files) |file_path| {
            _ = try self.addFile(file_path, null);
        }
    }

    pub fn addDepFile(self: *Manifest, dir: Io.Dir, dep_file_sub_path: []const u8) !void {
        assert(self.manifest_file == null);
        return self.addDepFileMaybePost(dir, dep_file_sub_path);
    }

    pub const HitError = error{
        /// Unable to check the cache for a reason that has been recorded into
        /// the `diagnostic` field.
        CacheCheckFailed,
        /// A cache manifest file exists however it could not be parsed.
        InvalidFormat,
        OutOfMemory,
        Canceled,
    };

    /// Check the cache to see if the input exists in it. If it exists, returns `true`.
    /// A hex encoding of its hash is available by calling `final`.
    ///
    /// This function will also acquire an exclusive lock to the manifest file. This means
    /// that a process holding a Manifest will block any other process attempting to
    /// acquire the lock. If `want_shared_lock` is `true`, a cache hit guarantees the
    /// manifest file to be locked in shared mode, and a cache miss guarantees the manifest
    /// file to be locked in exclusive mode.
    ///
    /// The lock on the manifest file is released when `deinit` is called. As another
    /// option, one may call `toOwnedLock` to obtain a smaller object which can represent
    /// the lock. `deinit` is safe to call whether or not `toOwnedLock` has been called.
    pub fn hit(man: *Manifest, parent_progress_node: std.Progress.Node) HitError!bool {
        const node = parent_progress_node.start("Reusing Cache Artifacts", 0);
        defer node.end();
        return hitInner(man);
    }

    pub fn hitInner(self: *Manifest) HitError!bool {
        assert(self.manifest_file == null);

        self.diagnostic = .none;

        const ext = ".txt";
        var manifest_file_path: [hex_digest_len + ext.len]u8 = undefined;

        var bin_digest: BinDigest = undefined;
        self.hash.hasher.final(&bin_digest);

        self.hex_digest = binToHex(bin_digest);

        @memcpy(manifest_file_path[0..self.hex_digest.len], &self.hex_digest);
        manifest_file_path[hex_digest_len..][0..ext.len].* = ext.*;

        const io = self.cache.io;

        // We'll try to open the cache with an exclusive lock, but if that would block
        // and `want_shared_lock` is set, a shared lock might be sufficient, so we'll
        // open with a shared lock instead.
        while (true) {
            if (self.cache.manifest_dir.createFile(io, &manifest_file_path, .{
                .read = true,
                .truncate = false,
                .lock = .exclusive,
                .lock_nonblocking = self.want_shared_lock,
            })) |manifest_file| {
                self.manifest_file = manifest_file;
                self.have_exclusive_lock = true;
                break;
            } else |err| switch (err) {
                error.WouldBlock => {
                    self.manifest_file = self.cache.manifest_dir.openFile(io, &manifest_file_path, .{
                        .mode = .read_write,
                        .lock = .shared,
                    }) catch |e| {
                        self.diagnostic = .{ .manifest_create = e };
                        return error.CacheCheckFailed;
                    };
                    break;
                },
                error.FileNotFound => {
                    // There are no dir components, so the only possibility
                    // should be that the directory behind the handle has been
                    // deleted, however we have observed on macOS two processes
                    // racing to do openat() with O_CREAT manifest in ENOENT.
                    //
                    // As a workaround, we retry with exclusive=true which
                    // disambiguates by returning EEXIST, indicating original
                    // failure was a race, or ENOENT, indicating deletion of
                    // the directory of our open handle.
                    if (!builtin.os.tag.isDarwin()) {
                        self.diagnostic = .{ .manifest_create = error.FileNotFound };
                        return error.CacheCheckFailed;
                    }

                    if (self.cache.manifest_dir.createFile(io, &manifest_file_path, .{
                        .read = true,
                        .truncate = false,
                        .lock = .exclusive,
                        .lock_nonblocking = self.want_shared_lock,
                        .exclusive = true,
                    })) |manifest_file| {
                        self.manifest_file = manifest_file;
                        self.have_exclusive_lock = true;
                        break;
                    } else |excl_err| switch (excl_err) {
                        error.WouldBlock, error.PathAlreadyExists => continue,
                        error.FileNotFound => {
                            self.diagnostic = .{ .manifest_create = error.FileNotFound };
                            return error.CacheCheckFailed;
                        },
                        error.Canceled => |e| return e,
                        else => |e| {
                            self.diagnostic = .{ .manifest_create = e };
                            return error.CacheCheckFailed;
                        },
                    }
                },
                error.Canceled => |e| return e,
                else => |e| {
                    self.diagnostic = .{ .manifest_create = e };
                    return error.CacheCheckFailed;
                },
            }
        }

        self.want_refresh_timestamp = true;

        const input_file_count = self.files.entries.len;

        // We're going to construct a second hash. Its input will begin with the digest we've
        // already computed (`bin_digest`), and then it'll have the digests of each input file,
        // including "post" files (see `addFilePost`). If this is a hit, we learn the set of "post"
        // files from the manifest on disk. If this is a miss, we'll learn those from future calls
        // to `addFilePost` etc. As such, the state of `self.hash.hasher` after this function
        // depends on whether this is a hit or a miss.
        //
        // If we return `true` indicating a cache hit, then `self.hash.hasher` must already include
        // the digests of the "post" files, so the caller can call `final`. Otherwise, on a cache
        // miss, `self.hash.hasher` will include the digests of all non-"post" files -- that is,
        // the ones we've already been told about. The rest will be discovered through calls to
        // `addFilePost` etc, which will update the hasher. After all files are added, the user can
        // use `final`, and will at some point `writeManifest` the file list to disk.

        self.hash.hasher = hasher_init;
        self.hash.hasher.update(&bin_digest);

        hit: {
            const file_digests_populated: usize = digests: {
                switch (try self.hitWithCurrentLock()) {
                    .hit => break :hit,
                    .miss => |m| if (!try self.upgradeToExclusiveLock()) {
                        break :digests m.file_digests_populated;
                    },
                }
                // We've just had a miss with the shared lock, and upgraded to an exclusive lock. Someone
                // else might have modified the digest, so we need to check again before deciding to miss.
                // Before trying again, we must reset `self.hash.hasher` and `self.files`.
                // This is basically just the first half of `unhit`.
                self.hash.hasher = hasher_init;
                self.hash.hasher.update(&bin_digest);
                while (self.files.count() != input_file_count) {
                    var file = self.files.pop().?;
                    file.key.deinit(self.cache.gpa);
                }
                switch (try self.hitWithCurrentLock()) {
                    .hit => break :hit,
                    .miss => |m| break :digests m.file_digests_populated,
                }
            };

            // This is a guaranteed cache miss. We're almost ready to return `false`, but there's a
            // little bookkeeping to do first. The first `file_digests_populated` entries in `files`
            // have their `bin_digest` populated; there may be some left in `input_file_count` which
            // we'll need to populate ourselves. Other than that, this is basically `unhit`.
            self.manifest_dirty = true;
            self.hash.hasher = hasher_init;
            self.hash.hasher.update(&bin_digest);
            while (self.files.count() != input_file_count) {
                var file = self.files.pop().?;
                file.key.deinit(self.cache.gpa);
            }
            for (self.files.keys(), 0..) |*file, idx| {
                if (idx < file_digests_populated) {
                    // `bin_digest` is already populated by `hitWithCurrentLock`, so we can use it directly.
                    self.hash.hasher.update(&file.bin_digest);
                } else {
                    self.populateFileHash(file) catch |err| {
                        self.diagnostic = .{ .file_hash = .{
                            .file_index = idx,
                            .err = err,
                        } };
                        return error.CacheCheckFailed;
                    };
                }
            }
            return false;
        }

        if (self.want_shared_lock) {
            self.downgradeToSharedLock() catch |err| {
                self.diagnostic = .{ .manifest_lock = err };
                return error.CacheCheckFailed;
            };
        }

        return true;
    }

    /// Assumes that `self.hash.hasher` has been updated only with the original digest and that
    /// `self.files` contains only the original input files.
    fn hitWithCurrentLock(self: *Manifest) HitError!union(enum) {
        hit,
        miss: struct {
            file_digests_populated: usize,
        },
    } {
        const gpa = self.cache.gpa;
        const io = self.cache.io;
        const input_file_count = self.files.entries.len;
        var tiny_buffer: [1]u8 = undefined; // allows allocRemaining to detect limit exceeded
        var manifest_reader = self.manifest_file.?.reader(io, &tiny_buffer); // Reads positionally from zero.
        const limit: std.Io.Limit = .limited(manifest_file_size_max);
        const file_contents = manifest_reader.interface.allocRemaining(gpa, limit) catch |err| switch (err) {
            error.OutOfMemory => |e| return e,
            error.StreamTooLong => return error.OutOfMemory,
            error.ReadFailed => {
                self.diagnostic = .{ .manifest_read = manifest_reader.err.? };
                return error.CacheCheckFailed;
            },
        };
        defer gpa.free(file_contents);

        var any_file_changed = false;
        var line_iter = mem.tokenizeScalar(u8, file_contents, '\n');
        var idx: usize = 0;
        const header_valid = valid: {
            const line = line_iter.next() orelse break :valid false;
            break :valid std.mem.eql(u8, line, manifest_header);
        };
        if (!header_valid) {
            return .{ .miss = .{ .file_digests_populated = 0 } };
        }
        while (line_iter.next()) |line| {
            defer idx += 1;

            var iter = mem.tokenizeScalar(u8, line, ' ');
            const size = iter.next() orelse return error.InvalidFormat;
            const inode = iter.next() orelse return error.InvalidFormat;
            const mtime_nsec_str = iter.next() orelse return error.InvalidFormat;
            const digest_str = iter.next() orelse return error.InvalidFormat;
            const prefix_str = iter.next() orelse return error.InvalidFormat;
            const file_path = iter.rest();

            const stat_size = fmt.parseInt(u64, size, 10) catch return error.InvalidFormat;
            const stat_inode = fmt.parseInt(Io.File.INode, inode, 10) catch return error.InvalidFormat;
            const stat_mtime = fmt.parseInt(i64, mtime_nsec_str, 10) catch return error.InvalidFormat;
            const file_bin_digest = b: {
                if (digest_str.len != hex_digest_len) return error.InvalidFormat;
                var bd: BinDigest = undefined;
                _ = fmt.hexToBytes(&bd, digest_str) catch return error.InvalidFormat;
                break :b bd;
            };

            const prefix = fmt.parseInt(u8, prefix_str, 10) catch return error.InvalidFormat;
            if (prefix >= self.cache.prefixes_len) return error.InvalidFormat;

            if (file_path.len == 0) return error.InvalidFormat;

            const cache_hash_file = f: {
                const prefixed_path: PrefixedPath = .{
                    .prefix = prefix,
                    .sub_path = file_path, // expires with file_contents
                };
                if (idx < input_file_count) {
                    const file = &self.files.keys()[idx];
                    if (!file.prefixed_path.eql(prefixed_path))
                        return error.InvalidFormat;

                    file.stat = .{
                        .size = stat_size,
                        .inode = stat_inode,
                        .mtime = .{ .nanoseconds = stat_mtime },
                    };
                    file.bin_digest = file_bin_digest;
                    break :f file;
                }
                const gop = try self.files.getOrPutAdapted(gpa, prefixed_path, FilesAdapter{});
                errdefer _ = self.files.pop();
                if (!gop.found_existing) {
                    gop.key_ptr.* = .{
                        .prefixed_path = .{
                            .prefix = prefix,
                            .sub_path = try gpa.dupe(u8, file_path),
                        },
                        .contents = null,
                        .max_file_size = null,
                        .handle = null,
                        .stat = .{
                            .size = stat_size,
                            .inode = stat_inode,
                            .mtime = .{ .nanoseconds = stat_mtime },
                        },
                        .bin_digest = file_bin_digest,
                    };
                }
                break :f gop.key_ptr;
            };

            const pp = cache_hash_file.prefixed_path;
            const dir = self.cache.prefixes()[pp.prefix].handle;
            const this_file = dir.openFile(io, pp.sub_path, .{ .mode = .read_only }) catch |err| switch (err) {
                error.FileNotFound => {
                    // Every digest before this one has been populated successfully.
                    return .{ .miss = .{ .file_digests_populated = idx } };
                },
                error.Canceled => |e| return e,
                else => |e| {
                    self.diagnostic = .{ .file_open = .{
                        .file_index = idx,
                        .err = e,
                    } };
                    return error.CacheCheckFailed;
                },
            };
            defer this_file.close(io);

            const actual_stat = this_file.stat(io) catch |err| {
                self.diagnostic = .{ .file_stat = .{
                    .file_index = idx,
                    .err = err,
                } };
                return error.CacheCheckFailed;
            };
            const size_match = actual_stat.size == cache_hash_file.stat.size;
            const mtime_match = actual_stat.mtime.nanoseconds == cache_hash_file.stat.mtime.nanoseconds;
            const inode_match = actual_stat.inode == cache_hash_file.stat.inode;

            if (!size_match or !mtime_match or !inode_match) {
                cache_hash_file.stat = .{
                    .size = actual_stat.size,
                    .mtime = actual_stat.mtime,
                    .inode = actual_stat.inode,
                };

                if (try self.isProblematicTimestamp(cache_hash_file.stat.mtime)) {
                    // The actual file has an unreliable timestamp, force it to be hashed
                    cache_hash_file.stat.mtime = .zero;
                    cache_hash_file.stat.inode = 0;
                }

                var actual_digest: BinDigest = undefined;
                hashFile(io, this_file, &actual_digest) catch |err| {
                    self.diagnostic = .{ .file_read = .{
                        .file_index = idx,
                        .err = err,
                    } };
                    return error.CacheCheckFailed;
                };

                if (!mem.eql(u8, &cache_hash_file.bin_digest, &actual_digest)) {
                    cache_hash_file.bin_digest = actual_digest;
                    // keep going until we have the input file digests
                    any_file_changed = true;
                }
            }

            if (!any_file_changed) {
                self.hash.hasher.update(&cache_hash_file.bin_digest);
            }
        }

        // If the manifest was somehow missing one of our input files, or if any file hash has changed,
        // then this is a cache miss. However, we have successfully populated some or all of the file
        // digests.
        if (any_file_changed or idx < input_file_count) {
            return .{ .miss = .{ .file_digests_populated = idx } };
        }

        return .hit;
    }

    /// Reset `self.hash.hasher` to the state it should be in after `hit` returns `false`.
    /// The hasher contains the original input digest, and all original input file digests (i.e.
    /// not including post files).
    /// Assumes that `bin_digest` is populated for all files up to `input_file_count`. As such,
    /// this is not necessarily safe to call within `hit`.
    pub fn unhit(self: *Manifest, bin_digest: BinDigest, input_file_count: usize) void {
        // Reset the hash.
        self.hash.hasher = hasher_init;
        self.hash.hasher.update(&bin_digest);

        // Remove files not in the initial hash.
        while (self.files.count() != input_file_count) {
            var file = self.files.pop().?;
            file.key.deinit(self.cache.gpa);
        }

        for (self.files.keys()) |file| {
            self.hash.hasher.update(&file.bin_digest);
        }
    }

    fn isProblematicTimestamp(man: *Manifest, timestamp: Io.Timestamp) error{Canceled}!bool {
        const io = man.cache.io;

        // If the file_time is prior to the most recent problematic timestamp
        // then we don't need to access the filesystem.
        if (timestamp.nanoseconds < man.recent_problematic_timestamp.nanoseconds)
            return false;

        // Next we will check the globally shared Cache timestamp, which is accessed
        // from multiple threads.
        try man.cache.mutex.lock(io);
        defer man.cache.mutex.unlock(io);

        // Save the global one to our local one to avoid locking next time.
        man.recent_problematic_timestamp = man.cache.recent_problematic_timestamp;
        if (timestamp.nanoseconds < man.recent_problematic_timestamp.nanoseconds)
            return false;

        // This flag prevents multiple filesystem writes for the same hit() call.
        if (man.want_refresh_timestamp) {
            man.want_refresh_timestamp = false;

            var file = man.cache.manifest_dir.createFile(io, "timestamp", .{
                .read = true,
                .truncate = true,
            }) catch |err| switch (err) {
                error.Canceled => |e| return e,
                else => return true,
            };
            defer file.close(io);

            // Save locally and also save globally (we still hold the global lock).
            const stat = file.stat(io) catch |err| switch (err) {
                error.Canceled => |e| return e,
                else => return true,
            };
            man.recent_problematic_timestamp = stat.mtime;
            man.cache.recent_problematic_timestamp = man.recent_problematic_timestamp;
        }

        return timestamp.nanoseconds >= man.recent_problematic_timestamp.nanoseconds;
    }

    fn populateFileHash(self: *Manifest, ch_file: *File) !void {
        const io = self.cache.io;

        if (ch_file.handle) |handle| {
            return populateFileHashHandle(self, ch_file, handle);
        } else {
            const pp = ch_file.prefixed_path;
            const dir = self.cache.prefixes()[pp.prefix].handle;
            const handle = try dir.openFile(io, pp.sub_path, .{});
            defer handle.close(io);
            return populateFileHashHandle(self, ch_file, handle);
        }
    }

    fn populateFileHashHandle(self: *Manifest, ch_file: *File, io_file: Io.File) !void {
        const io = self.cache.io;
        const gpa = self.cache.gpa;

        const actual_stat = try io_file.stat(io);
        ch_file.stat = .{
            .size = actual_stat.size,
            .mtime = actual_stat.mtime,
            .inode = actual_stat.inode,
        };

        if (try self.isProblematicTimestamp(ch_file.stat.mtime)) {
            // The actual file has an unreliable timestamp, force it to be hashed
            ch_file.stat.mtime = .zero;
            ch_file.stat.inode = 0;
        }

        if (ch_file.max_file_size) |max_file_size| {
            if (ch_file.stat.size > max_file_size) return error.FileTooBig;

            // Hash while reading from disk, to keep the contents in the cpu
            // cache while doing hashing.
            const contents = try gpa.alloc(u8, @intCast(ch_file.stat.size));
            errdefer gpa.free(contents);

            var hasher = hasher_init;
            var off: usize = 0;
            while (true) {
                const bytes_read = try io_file.readPositional(io, &.{contents[off..]}, off);
                if (bytes_read == 0) break;
                hasher.update(contents[off..][0..bytes_read]);
                off += bytes_read;
            }
            hasher.final(&ch_file.bin_digest);

            ch_file.contents = contents;
        } else {
            try hashFile(io, io_file, &ch_file.bin_digest);
        }

        self.hash.hasher.update(&ch_file.bin_digest);
    }

    /// Add a file as a dependency of process being cached, after the initial hash has been
    /// calculated. This is useful for processes that don't know all the files that
    /// are depended on ahead of time. For example, a source file that can import other files
    /// will need to be recompiled if the imported file is changed.
    pub fn addFilePostFetch(self: *Manifest, file_path: []const u8, max_file_size: usize) ![]const u8 {
        assert(self.manifest_file != null);

        const gpa = self.cache.gpa;
        const prefixed_path = try self.cache.findPrefix(file_path);
        errdefer gpa.free(prefixed_path.sub_path);

        const gop = try self.files.getOrPutAdapted(gpa, prefixed_path, FilesAdapter{});
        errdefer _ = self.files.pop();

        if (gop.found_existing) {
            gpa.free(prefixed_path.sub_path);
            return gop.key_ptr.contents.?;
        }

        gop.key_ptr.* = .{
            .prefixed_path = prefixed_path,
            .max_file_size = max_file_size,
            .stat = undefined,
            .bin_digest = undefined,
            .contents = null,
            .handle = null,
        };

        self.files.lockPointers();
        defer self.files.unlockPointers();

        try self.populateFileHash(gop.key_ptr);
        return gop.key_ptr.contents.?;
    }

    /// Add a file as a dependency of process being cached, after the initial hash has been
    /// calculated.
    ///
    /// This is useful for processes that don't know the all the files that are
    /// depended on ahead of time. For example, a source file that can import
    /// other files will need to be recompiled if the imported file is changed.
    pub fn addFilePost(man: *Manifest, file_path: []const u8) !void {
        assert(man.manifest_file != null);
        const gpa = man.cache.gpa;
        const prefixed_path = try man.cache.findPrefix(file_path);
        var keep = false;
        defer if (!keep) gpa.free(prefixed_path.sub_path);
        keep = try addPrefixedPathPost(man, prefixed_path);
    }

    pub fn addPathPost(man: *Manifest, path: Path) !void {
        assert(man.manifest_file != null);
        const gpa = man.cache.gpa;
        const prefixed_path: PrefixedPath = try man.cache.findPrefixPath(path);
        var keep = false;
        defer if (!keep) gpa.free(prefixed_path.sub_path);
        keep = try addPrefixedPathPost(man, prefixed_path);
    }

    /// Low level function. `prefixed_path` references cloned memory. Returns
    /// whether or not `prefixed_path.sub_path` should be kept.
    pub fn addPrefixedPathPost(man: *Manifest, prefixed_path: PrefixedPath) !bool {
        assert(man.manifest_file != null);
        const gpa = man.cache.gpa;

        const gop = try man.files.getOrPutAdapted(gpa, prefixed_path, FilesAdapter{});
        errdefer _ = man.files.pop();

        if (gop.found_existing) return false;

        gop.key_ptr.* = .{
            .prefixed_path = prefixed_path,
            .max_file_size = null,
            .handle = null,
            .stat = undefined,
            .bin_digest = undefined,
            .contents = null,
        };

        man.files.lockPointers();
        defer man.files.unlockPointers();

        try man.populateFileHash(gop.key_ptr);
        return true;
    }

    /// Like `addFilePost` but when the file contents have already been loaded from disk.
    pub fn addFilePostContents(
        self: *Manifest,
        file_path: []const u8,
        bytes: []const u8,
        stat: File.Stat,
    ) !void {
        assert(self.manifest_file != null);
        const gpa = self.cache.gpa;

        const prefixed_path = try self.cache.findPrefix(file_path);
        errdefer gpa.free(prefixed_path.sub_path);

        const gop = try self.files.getOrPutAdapted(gpa, prefixed_path, FilesAdapter{});
        errdefer _ = self.files.pop();

        if (gop.found_existing) {
            gpa.free(prefixed_path.sub_path);
            return;
        }

        const new_file = gop.key_ptr;

        new_file.* = .{
            .prefixed_path = prefixed_path,
            .max_file_size = null,
            .handle = null,
            .stat = stat,
            .bin_digest = undefined,
            .contents = null,
        };

        if (try self.isProblematicTimestamp(new_file.stat.mtime)) {
            // The actual file has an unreliable timestamp, force it to be hashed
            new_file.stat.mtime = .zero;
            new_file.stat.inode = 0;
        }

        {
            var hasher = hasher_init;
            hasher.update(bytes);
            hasher.final(&new_file.bin_digest);
        }

        self.hash.hasher.update(&new_file.bin_digest);
    }

    pub fn addDepFilePost(self: *Manifest, dir: Io.Dir, dep_file_sub_path: []const u8) !void {
        assert(self.manifest_file != null);
        return self.addDepFileMaybePost(dir, dep_file_sub_path);
    }

    fn addDepFileMaybePost(self: *Manifest, dir: Io.Dir, dep_file_sub_path: []const u8) !void {
        const gpa = self.cache.gpa;
        const io = self.cache.io;
        const dep_file_contents = try dir.readFileAlloc(io, dep_file_sub_path, gpa, .limited(manifest_file_size_max));
        defer gpa.free(dep_file_contents);

        var error_buf: std.ArrayList(u8) = .empty;
        defer error_buf.deinit(gpa);

        var resolve_buf: std.ArrayList(u8) = .empty;
        defer resolve_buf.deinit(gpa);

        var it: DepTokenizer = .{ .bytes = dep_file_contents };
        while (it.next()) |token| {
            switch (token) {
                // We don't care about targets, we only want the prereqs
                // Clang is invoked in single-source mode but other programs may not
                .target, .target_must_resolve => {},
                .prereq => |file_path| if (self.manifest_file == null) {
                    _ = try self.addFile(file_path, null);
                } else try self.addFilePost(file_path),
                .prereq_must_resolve => {
                    resolve_buf.clearRetainingCapacity();
                    try token.resolve(gpa, &resolve_buf);
                    if (self.manifest_file == null) {
                        _ = try self.addFile(resolve_buf.items, null);
                    } else try self.addFilePost(resolve_buf.items);
                },
                else => |err| {
                    try err.printError(gpa, &error_buf);
                    log.err("failed parsing {s}: {s}", .{ dep_file_sub_path, error_buf.items });
                    return error.InvalidDepFile;
                },
            }
        }
    }

    /// Returns a binary hash of the inputs.
    pub fn finalBin(self: *Manifest) BinDigest {
        assert(self.manifest_file != null);

        // We don't close the manifest file yet, because we want to
        // keep it locked until the API user is done using it.
        // We also don't write out the manifest yet, because until
        // cache_release is called we still might be working on creating
        // the artifacts to cache.

        var bin_digest: BinDigest = undefined;
        self.hash.hasher.final(&bin_digest);
        return bin_digest;
    }

    /// Returns a hex encoded hash of the inputs.
    pub fn final(self: *Manifest) HexDigest {
        const bin_digest = self.finalBin();
        return binToHex(bin_digest);
    }

    /// If `want_shared_lock` is true, this function automatically downgrades the
    /// lock from exclusive to shared.
    pub fn writeManifest(self: *Manifest) !void {
        assert(self.have_exclusive_lock);
        const io = self.cache.io;
        const manifest_file = self.manifest_file.?;
        if (self.manifest_dirty) {
            self.manifest_dirty = false;

            var buffer: [4000]u8 = undefined;
            var fw = manifest_file.writer(io, &buffer);
            writeDirtyManifestToStream(self, &fw) catch |err| switch (err) {
                error.WriteFailed => return fw.err.?,
                else => |e| return e,
            };
        }

        if (self.want_shared_lock) {
            try self.downgradeToSharedLock();
        }
    }

    fn writeDirtyManifestToStream(self: *Manifest, fw: *Io.File.Writer) !void {
        try fw.interface.writeAll(manifest_header ++ "\n");
        for (self.files.keys()) |file| {
            try fw.interface.print("{d} {d} {d} {x} {d} {s}\n", .{
                file.stat.size,
                file.stat.inode,
                file.stat.mtime,
                &file.bin_digest,
                file.prefixed_path.prefix,
                file.prefixed_path.sub_path,
            });
        }
        try fw.end();
    }

    fn downgradeToSharedLock(self: *Manifest) !void {
        if (!self.have_exclusive_lock) return;
        const io = self.cache.io;

        if (std.process.can_spawn or !builtin.single_threaded) {
            const manifest_file = self.manifest_file.?;
            try manifest_file.downgradeLock(io);
        }

        self.have_exclusive_lock = false;
    }

    fn upgradeToExclusiveLock(self: *Manifest) error{CacheCheckFailed}!bool {
        if (self.have_exclusive_lock) return false;
        assert(self.manifest_file != null);
        const io = self.cache.io;

        if (std.process.can_spawn or !builtin.single_threaded) {
            const manifest_file = self.manifest_file.?;
            // Here we intentionally have a period where the lock is released, in case there are
            // other processes holding a shared lock.
            manifest_file.unlock(io);
            manifest_file.lock(io, .exclusive) catch |err| {
                self.diagnostic = .{ .manifest_lock = err };
                return error.CacheCheckFailed;
            };
        }
        self.have_exclusive_lock = true;
        return true;
    }

    /// Obtain only the data needed to maintain a lock on the manifest file.
    /// The `Manifest` remains safe to deinit.
    /// Don't forget to call `writeManifest` before this!
    pub fn toOwnedLock(self: *Manifest) Lock {
        defer self.manifest_file = null;
        return .{ .manifest_file = self.manifest_file.? };
    }

    /// Releases the manifest file and frees any memory the Manifest was using.
    /// `Manifest.hit` must be called first.
    /// Don't forget to call `writeManifest` before this!
    pub fn deinit(self: *Manifest) void {
        const io = self.cache.io;

        if (self.manifest_file) |file| {
            if (builtin.os.tag == .windows) {
                // See Lock.release for why this is required on Windows
                file.unlock(io);
            }

            file.close(io);
        }
        for (self.files.keys()) |*file| {
            file.deinit(self.cache.gpa);
        }
        self.files.deinit(self.cache.gpa);
    }

    pub fn populateFileSystemInputs(man: *Manifest, buf: *std.ArrayList(u8)) Allocator.Error!void {
        assert(@typeInfo(std.zig.Server.Message.PathPrefix).@"enum".field_names.len == man.cache.prefixes_len);
        buf.clearRetainingCapacity();
        const gpa = man.cache.gpa;
        const files = man.files.keys();
        if (files.len > 0) {
            for (files) |file| {
                try buf.ensureUnusedCapacity(gpa, file.prefixed_path.sub_path.len + 2);
                buf.appendAssumeCapacity(file.prefixed_path.prefix + 1);
                buf.appendSliceAssumeCapacity(file.prefixed_path.sub_path);
                buf.appendAssumeCapacity(0);
            }
            // The null byte is a separator, not a terminator.
            buf.items.len -= 1;
        }
    }

    pub fn populateOtherManifest(man: *Manifest, other: *Manifest, prefix_map: [4]u8) Allocator.Error!void {
        const gpa = other.cache.gpa;
        assert(@typeInfo(std.zig.Server.Message.PathPrefix).@"enum".field_names.len == man.cache.prefixes_len);
        assert(man.cache.prefixes_len == 4);
        for (man.files.keys()) |file| {
            const prefixed_path: PrefixedPath = .{
                .prefix = prefix_map[file.prefixed_path.prefix],
                .sub_path = try gpa.dupe(u8, file.prefixed_path.sub_path),
            };
            errdefer gpa.free(prefixed_path.sub_path);

            const gop = try other.files.getOrPutAdapted(gpa, prefixed_path, FilesAdapter{});
            errdefer _ = other.files.pop();

            if (gop.found_existing) {
                gpa.free(prefixed_path.sub_path);
                continue;
            }

            gop.key_ptr.* = .{
                .prefixed_path = prefixed_path,
                .max_file_size = file.max_file_size,
                .handle = file.handle,
                .stat = file.stat,
                .bin_digest = file.bin_digest,
                .contents = null,
            };

            other.hash.hasher.update(&gop.key_ptr.bin_digest);
        }
    }
}