parseTokens

pub fn parseTokens(
    gpa: Allocator,
    source: [:0]const u8,
    tokens: Ast.TokenList.Slice,
    options: ParseOptions,
) Allocator.Error!Ast {
    var parser: Parse = .{
        .source = source,
        .gpa = gpa,
        .tokens = tokens,
        .errors = .empty,
        .nodes = .empty,
        .extra_data = .empty,
        .scratch = .empty,
        .tok_i = 0,
        .recover = options.recover,
    };
    defer parser.errors.deinit(gpa);
    defer parser.nodes.deinit(gpa);
    defer parser.extra_data.deinit(gpa);
    defer parser.scratch.deinit(gpa);

    // Empirically, Zig source code has a 2:1 ratio of tokens to AST nodes.
    // Make sure at least 1 so we can use appendAssumeCapacity on the root node below.
    const estimated_node_count = (tokens.len + 2) / 2;
    try parser.nodes.ensureTotalCapacity(gpa, estimated_node_count);

    switch (options.mode) {
        .zig => try parser.parseRoot(),
        .zon => try parser.parseZon(),
    }

    try parser.extra_data.shrinkToLen(gpa);
    try parser.errors.shrinkToLen(gpa);

    // TODO experiment with compacting the MultiArrayList slices here
    return .{
        .source = source,
        .mode = options.mode,
        .tokens = tokens,
        .nodes = parser.nodes.toOwnedSlice(),
        .extra_data = parser.extra_data.toOwnedSliceAssert(),
        .errors = parser.errors.toOwnedSliceAssert(),
    };
}