Token

pub const Token = struct

File

Code

pub const Token = struct {
    id: Id,
    start: usize,
    end: usize,
    line_number: usize,

    pub const Id = enum {
        literal,
        number,
        quoted_ascii_string,
        quoted_wide_string,
        operator,
        begin,
        end,
        comma,
        open_paren,
        close_paren,
        /// This Id is only used for errors, the Lexer will never return one
        /// of these from a `next` call.
        preprocessor_command,
        invalid,
        eof,

        pub fn nameForErrorDisplay(self: Id) []const u8 {
            return switch (self) {
                .literal => "<literal>",
                .number => "<number>",
                .quoted_ascii_string => "<quoted ascii string>",
                .quoted_wide_string => "<quoted wide string>",
                .operator => "<operator>",
                .begin => "<'{' or BEGIN>",
                .end => "<'}' or END>",
                .comma => ",",
                .open_paren => "(",
                .close_paren => ")",
                .preprocessor_command => "<preprocessor command>",
                .invalid => unreachable,
                .eof => "<eof>",
            };
        }
    };

    pub fn slice(self: Token, buffer: []const u8) []const u8 {
        return buffer[self.start..self.end];
    }

    /// Returns 0-based column
    pub fn calculateColumn(token: Token, source: []const u8, tab_columns: usize, maybe_line_start: ?usize) usize {
        const line_start = maybe_line_start orelse token.getLineStartForColumnCalc(source);

        var i: usize = line_start;
        var column: usize = 0;
        while (i < token.start) : (i += 1) {
            column += columnWidth(column, source[i], tab_columns);
        }
        return column;
    }

    // TODO: More testing is needed to determine if this can be merged with getLineStartForErrorDisplay
    //       (the TODO in currentIndexFormsLineEndingPair should be taken into account as well)
    pub fn getLineStartForColumnCalc(token: Token, source: []const u8) usize {
        const line_start = line_start: {
            if (token.start != 0) {
                // start checking at the byte before the token
                var index = token.start - 1;
                while (true) {
                    if (source[index] == '\n') break :line_start @min(source.len - 1, index + 1);
                    if (index != 0) index -= 1 else break;
                }
            }
            break :line_start 0;
        };
        return line_start;
    }

    pub fn getLineStartForErrorDisplay(token: Token, source: []const u8) usize {
        const line_start = line_start: {
            if (token.start != 0) {
                // start checking at the byte before the token
                var index = token.start - 1;
                while (true) {
                    if (source[index] == '\r' or source[index] == '\n') break :line_start @min(source.len - 1, index + 1);
                    if (index != 0) index -= 1 else break;
                }
            }
            break :line_start 0;
        };
        return line_start;
    }

    pub fn getLineForErrorDisplay(token: Token, source: []const u8, maybe_line_start: ?usize) []const u8 {
        const line_start = maybe_line_start orelse token.getLineStartForErrorDisplay(source);

        var line_end = line_start;
        while (line_end < source.len and source[line_end] != '\r' and source[line_end] != '\n') : (line_end += 1) {}
        return source[line_start..line_end];
    }

    pub fn isStringLiteral(token: Token) bool {
        return token.id == .quoted_ascii_string or token.id == .quoted_wide_string;
    }
}