feature. See also
. The project being documented here (as the example) is the Zig library itself.
lex.Lexer
pub const Lexer = struct
File
Code
pub const Lexer = struct {
const Self = @This();
buffer: []const u8,
index: usize,
line_handler: LineHandler,
at_start_of_line: bool = true,
error_context_token: ?Token = null,
current_code_page: SupportedCodePage,
default_code_page: SupportedCodePage,
source_mappings: ?*SourceMappings,
max_string_literal_codepoints: u15,
seen_pragma_code_pages: u2 = 0,
last_pragma_code_page_token: ?Token = null,
pub const Error = LexError;
pub const LexerOptions = struct {
default_code_page: SupportedCodePage = .windows1252,
source_mappings: ?*SourceMappings = null,
max_string_literal_codepoints: u15 = default_max_string_literal_codepoints,
};
pub fn init(buffer: []const u8, options: LexerOptions) Self {
return Self{
.buffer = buffer,
.index = 0,
.current_code_page = options.default_code_page,
.default_code_page = options.default_code_page,
.source_mappings = options.source_mappings,
.max_string_literal_codepoints = options.max_string_literal_codepoints,
.line_handler = .{ .buffer = buffer },
};
}
pub fn dump(self: *Self, token: *const Token) void {
std.debug.print("{s}:{d}: {f}\n", .{
@tagName(token.id), token.line_number, std.ascii.hexEscape(token.slice(self.buffer), .lower),
});
}
pub const LexMethod = enum {
whitespace_delimiter_only,
normal,
normal_expect_operator,
};
pub fn next(self: *Self, comptime method: LexMethod) LexError!Token {
switch (method) {
.whitespace_delimiter_only => return self.nextWhitespaceDelimeterOnly(),
.normal => return self.nextNormal(),
.normal_expect_operator => return self.nextNormalWithContext(.expect_operator),
}
}
const StateWhitespaceDelimiterOnly = enum {
start,
literal,
preprocessor,
semicolon,
};
pub fn nextWhitespaceDelimeterOnly(self: *Self) LexError!Token {
const start_index = self.index;
var result = Token{
.id = .eof,
.start = start_index,
.end = undefined,
.line_number = self.line_handler.line_number,
};
var state = StateWhitespaceDelimiterOnly.start;
while (self.current_code_page.codepointAt(self.index, self.buffer)) |codepoint| : (self.index += codepoint.byte_len) {
const c = codepoint.value;
try self.checkForIllegalCodepoint(codepoint, false);
switch (state) {
.start => switch (c) {
'\r', '\n' => {
result.start = self.index + 1;
result.line_number = self.incrementLineNumber();
},
' ', '\t', '\x05'...'\x08', '\x0B'...'\x0C', '\x0E'...'\x1F' => {
result.start = self.index + 1;
},
// can be intermixed with other whitespace). Who knows why.
// TODO: This should either be removed, or it should also include
// the codepoints listed in disjoint_code_page.zig
'\xA0' => if (self.at_start_of_line) {
result.start = self.index + codepoint.byte_len;
} else {
state = .literal;
self.at_start_of_line = false;
},
'#' => {
if (self.at_start_of_line) {
state = .preprocessor;
} else {
state = .literal;
}
self.at_start_of_line = false;
},
';' => {
state = .semicolon;
self.at_start_of_line = false;
},
else => {
state = .literal;
self.at_start_of_line = false;
},
},
.literal => switch (c) {
'\r', '\n', ' ', '\t', '\x05'...'\x08', '\x0B'...'\x0C', '\x0E'...'\x1F' => {
result.id = .literal;
break;
},
else => {},
},
.preprocessor => switch (c) {
'\r', '\n' => {
try self.evaluatePreprocessorCommand(result.start, self.index);
result.start = self.index + 1;
state = .start;
result.line_number = self.incrementLineNumber();
},
else => {},
},
.semicolon => switch (c) {
'\r', '\n' => {
result.start = self.index + 1;
state = .start;
result.line_number = self.incrementLineNumber();
},
else => {},
},
}
} else {
switch (state) {
.start => {},
.semicolon => {
result.start = self.index;
},
.literal => {
result.id = .literal;
},
.preprocessor => {
try self.evaluatePreprocessorCommand(result.start, self.index);
result.start = self.index;
},
}
}
result.end = self.index;
std.debug.assert(result.id != .eof or result.start == result.end);
return result;
}
const StateNormal = enum {
start,
literal_or_quoted_wide_string,
quoted_ascii_string,
quoted_wide_string,
quoted_ascii_string_escape,
quoted_wide_string_escape,
quoted_ascii_string_maybe_end,
quoted_wide_string_maybe_end,
literal,
number_literal,
preprocessor,
semicolon,
e,
en,
b,
be,
beg,
begi,
};
pub fn nextNormal(self: *Self) LexError!Token {
return self.nextNormalWithContext(.any);
}
pub fn nextNormalWithContext(self: *Self, context: enum { expect_operator, any }) LexError!Token {
const start_index = self.index;
var result = Token{
.id = .eof,
.start = start_index,
.end = undefined,
.line_number = self.line_handler.line_number,
};
var state = StateNormal.start;
// length for its 'string literal too long' errors; it isn't easily
// explained or intuitive (it's sort-of pre-parsed byte length but with
// a few of exceptions/edge cases).
//
// It also behaves strangely with non-ASCII codepoints, e.g. even though the default
// limit is 4097, you can only have 4094 € codepoints (1 UTF-16 code unit each),
// and 2048 𐐷 codepoints (2 UTF-16 code units each).
//
// TODO: Understand this more, bring it more in line with how the Win32 limits work.
// Alternatively, do something that makes more sense but may be more permissive.
var string_literal_length: usize = 0;
// there are tons of tab stop characters within a string literal.
var string_literal_column: usize = 0;
var string_literal_collapsing_whitespace: bool = false;
var still_could_have_exponent: bool = true;
var exponent_index: ?usize = null;
while (self.current_code_page.codepointAt(self.index, self.buffer)) |codepoint| : (self.index += codepoint.byte_len) {
const c = codepoint.value;
const in_string_literal = switch (state) {
.quoted_ascii_string,
.quoted_wide_string,
.quoted_ascii_string_escape,
.quoted_wide_string_escape,
.quoted_ascii_string_maybe_end,
.quoted_wide_string_maybe_end,
=>
// then we want to treat the current codepoint as 'not in a string literal'
// for the purposes of detecting illegal codepoints. This means that we will
// error on illegal-outside-string-literal characters that are outside string
// literals from the perspective of a C preprocessor, but that may be
// inside string literals from the perspective of the RC lexer. For example,
// "hello
// @"
// will be treated as a single string literal by the RC lexer but the Win32
// preprocessor will consider this an unclosed string literal followed by
// the character @ and ", and will therefore error since the Win32 RC preprocessor
// errors on the @ character outside string literals.
//
// By doing this here, we can effectively emulate the Win32 RC preprocessor behavior
// at lex-time, and avoid the need for a separate step that checks for this edge-case
// specifically.
result.line_number == self.line_handler.line_number,
else => false,
};
try self.checkForIllegalCodepoint(codepoint, in_string_literal);
switch (state) {
.start => switch (c) {
'\r', '\n' => {
result.start = self.index + 1;
result.line_number = self.incrementLineNumber();
},
' ', '\t', '\x05'...'\x08', '\x0B'...'\x0C', '\x0E'...'\x1F' => {
result.start = self.index + 1;
},
// can be intermixed with other whitespace). Who knows why.
'\xA0' => if (self.at_start_of_line) {
result.start = self.index + codepoint.byte_len;
} else {
state = .literal;
self.at_start_of_line = false;
},
'L', 'l' => {
state = .literal_or_quoted_wide_string;
self.at_start_of_line = false;
},
'E', 'e' => {
state = .e;
self.at_start_of_line = false;
},
'B', 'b' => {
state = .b;
self.at_start_of_line = false;
},
'"' => {
state = .quoted_ascii_string;
self.at_start_of_line = false;
string_literal_collapsing_whitespace = false;
string_literal_length = 0;
var dummy_token = Token{
.start = self.index,
.end = self.index,
.line_number = self.line_handler.line_number,
.id = .invalid,
};
string_literal_column = dummy_token.calculateColumn(self.buffer, 8, null);
},
'+', '&', '|' => {
self.index += 1;
result.id = .operator;
self.at_start_of_line = false;
break;
},
'-' => {
if (context == .expect_operator) {
self.index += 1;
result.id = .operator;
self.at_start_of_line = false;
break;
} else {
state = .number_literal;
still_could_have_exponent = true;
exponent_index = null;
self.at_start_of_line = false;
}
},
'0'...'9', '~' => {
state = .number_literal;
still_could_have_exponent = true;
exponent_index = null;
self.at_start_of_line = false;
},
'#' => {
if (self.at_start_of_line) {
state = .preprocessor;
} else {
state = .literal;
}
self.at_start_of_line = false;
},
';' => {
state = .semicolon;
self.at_start_of_line = false;
},
'{', '}' => {
self.index += 1;
result.id = if (c == '{') .begin else .end;
self.at_start_of_line = false;
break;
},
'(', ')' => {
self.index += 1;
result.id = if (c == '(') .open_paren else .close_paren;
self.at_start_of_line = false;
break;
},
',' => {
self.index += 1;
result.id = .comma;
self.at_start_of_line = false;
break;
},
else => {
if (isNonAsciiDigit(c)) {
self.error_context_token = .{
.id = .number,
.start = result.start,
.end = self.index + 1,
.line_number = self.line_handler.line_number,
};
return error.InvalidDigitCharacterInNumberLiteral;
}
state = .literal;
self.at_start_of_line = false;
},
},
.preprocessor => switch (c) {
'\r', '\n' => {
try self.evaluatePreprocessorCommand(result.start, self.index);
result.start = self.index + 1;
state = .start;
result.line_number = self.incrementLineNumber();
},
else => {},
},
// the next line.
.semicolon => switch (c) {
'\r', '\n' => {
result.start = self.index + 1;
state = .start;
result.line_number = self.incrementLineNumber();
},
else => {},
},
.number_literal => switch (c) {
' ', '\t', '\x05'...'\x08', '\x0B'...'\x0C', '\x0E'...'\x1F',
'\r', '\n', '"', ',', '{', '}', '+', '-', '|', '&', '~', '(', ')',
'\'', ';', '=',
=> {
result.id = .number;
break;
},
'0'...'9' => {
if (exponent_index) |exp_i| {
if (self.index - 1 == exp_i) {
// the Win32 RC compiler.
self.error_context_token = .{
.id = .number,
.start = result.start,
.end = self.index + 1,
.line_number = self.line_handler.line_number,
};
return error.InvalidNumberWithExponent;
}
}
},
'e', 'E' => {
if (still_could_have_exponent) {
exponent_index = self.index;
still_could_have_exponent = false;
}
},
else => {
if (isNonAsciiDigit(c)) {
self.error_context_token = .{
.id = .number,
.start = result.start,
.end = self.index + 1,
.line_number = self.line_handler.line_number,
};
return error.InvalidDigitCharacterInNumberLiteral;
}
still_could_have_exponent = false;
},
},
.literal_or_quoted_wide_string => switch (c) {
' ', '\t', '\x05'...'\x08', '\x0B'...'\x0C', '\x0E'...'\x1F',
'\r', '\n', ',', '{', '}', '+', '-', '|', '&', '~', '(', ')',
'\'', ';', '=',
=> {
result.id = .literal;
break;
},
'"' => {
state = .quoted_wide_string;
string_literal_collapsing_whitespace = false;
string_literal_length = 0;
var dummy_token = Token{
.start = self.index,
.end = self.index,
.line_number = self.line_handler.line_number,
.id = .invalid,
};
string_literal_column = dummy_token.calculateColumn(self.buffer, 8, null);
},
else => {
state = .literal;
},
},
.literal => switch (c) {
' ', '\t', '\x05'...'\x08', '\x0B'...'\x0C', '\x0E'...'\x1F',
'\r', '\n', '"', ',', '{', '}', '+', '-', '|', '&', '~', '(', ')',
'\'', ';', '=',
=> {
result.id = .literal;
break;
},
else => {},
},
.e => switch (c) {
'N', 'n' => {
state = .en;
},
else => {
state = .literal;
self.index -= 1;
},
},
.en => switch (c) {
'D', 'd' => {
result.id = .end;
self.index += 1;
break;
},
else => {
state = .literal;
self.index -= 1;
},
},
.b => switch (c) {
'E', 'e' => {
state = .be;
},
else => {
state = .literal;
self.index -= 1;
},
},
.be => switch (c) {
'G', 'g' => {
state = .beg;
},
else => {
state = .literal;
self.index -= 1;
},
},
.beg => switch (c) {
'I', 'i' => {
state = .begi;
},
else => {
state = .literal;
self.index -= 1;
},
},
.begi => switch (c) {
'N', 'n' => {
result.id = .begin;
self.index += 1;
break;
},
else => {
state = .literal;
self.index -= 1;
},
},
.quoted_ascii_string, .quoted_wide_string => switch (c) {
'"' => {
string_literal_column += 1;
state = if (state == .quoted_ascii_string) .quoted_ascii_string_maybe_end else .quoted_wide_string_maybe_end;
},
'\\' => {
string_literal_length += 1;
string_literal_column += 1;
state = if (state == .quoted_ascii_string) .quoted_ascii_string_escape else .quoted_wide_string_escape;
},
'\r' => {
string_literal_column = 0;
// Increment line number but don't affect the result token's line number
_ = self.incrementLineNumber();
},
'\n' => {
string_literal_column = 0;
if (!string_literal_collapsing_whitespace) {
string_literal_length += 2;
string_literal_collapsing_whitespace = true;
}
// Increment line number but don't affect the result token's line number
_ = self.incrementLineNumber();
},
'\t', ' ', '\x0b', '\x0c' => {
if (!string_literal_collapsing_whitespace) {
// needed to reach the next 8-column tab stop.
const width = columnWidth(string_literal_column, @intCast(c), 8);
string_literal_length += width;
string_literal_column += width;
}
},
else => {
string_literal_collapsing_whitespace = false;
string_literal_length += 1;
string_literal_column += 1;
},
},
.quoted_ascii_string_escape, .quoted_wide_string_escape => switch (c) {
'"' => {
self.error_context_token = .{
.id = .invalid,
.start = self.index - 1,
.end = self.index + 1,
.line_number = self.line_handler.line_number,
};
return error.FoundCStyleEscapedQuote;
},
else => {
string_literal_length += 1;
string_literal_column += 1;
state = if (state == .quoted_ascii_string_escape) .quoted_ascii_string else .quoted_wide_string;
},
},
.quoted_ascii_string_maybe_end, .quoted_wide_string_maybe_end => switch (c) {
'"' => {
state = if (state == .quoted_ascii_string_maybe_end) .quoted_ascii_string else .quoted_wide_string;
// Since we did not increment on the first " (because it could have been
// the end of the quoted string), we increment here
string_literal_length += 1;
string_literal_column += 1;
},
else => {
result.id = if (state == .quoted_ascii_string_maybe_end) .quoted_ascii_string else .quoted_wide_string;
break;
},
},
}
} else {
switch (state) {
.start => {},
.semicolon => {
result.start = self.index;
},
.literal_or_quoted_wide_string, .literal, .e, .en, .b, .be, .beg, .begi => {
result.id = .literal;
},
.preprocessor => {
try self.evaluatePreprocessorCommand(result.start, self.index);
result.start = self.index;
},
.number_literal => {
result.id = .number;
},
.quoted_ascii_string_maybe_end, .quoted_wide_string_maybe_end => {
result.id = if (state == .quoted_ascii_string_maybe_end) .quoted_ascii_string else .quoted_wide_string;
},
.quoted_ascii_string,
.quoted_wide_string,
.quoted_ascii_string_escape,
.quoted_wide_string_escape,
=> {
self.error_context_token = .{
.id = .eof,
.start = self.index,
.end = self.index,
.line_number = self.line_handler.line_number,
};
return LexError.UnfinishedStringLiteral;
},
}
}
result.end = self.index;
if (result.id == .quoted_ascii_string or result.id == .quoted_wide_string) {
if (string_literal_length > self.max_string_literal_codepoints) {
self.error_context_token = result;
return LexError.StringLiteralTooLong;
}
}
std.debug.assert(result.id != .eof or result.start == result.end);
return result;
}
fn incrementLineNumber(self: *Self) usize {
_ = self.line_handler.incrementLineNumber(self.index);
self.at_start_of_line = true;
return self.line_handler.line_number;
}
fn checkForIllegalCodepoint(self: *Self, codepoint: code_pages.Codepoint, in_string_literal: bool) LexError!void {
const err = switch (codepoint.value) {
// 0x1A = Substitute (treated as EOF)
// NOTE: 0x1A gets treated as EOF by the clang preprocessor so after a .rc file
// is run through the clang preprocessor it will no longer have 0x1A characters in it.
// 0x7F = DEL (treated as a context-specific terminator by the Windows RC compiler)
0x00, 0x1A, 0x7F => error.IllegalByte,
// 0x04 is valid but behaves strangely (sort of acts as a 'skip the next character' instruction)
0x01...0x04 => if (!in_string_literal) error.IllegalByteOutsideStringLiterals else return,
// fatal error RC1116: RC terminating after preprocessor errors) if they are ever used
// outside of string literals. Not exactly sure why this would be the case, though.
// TODO: Make sure there aren't any exceptions
'@', '`' => if (!in_string_literal) error.IllegalByteOutsideStringLiterals else return,
// there are edge cases where it leads to cryptic 'compiler limit : macro definition too big'
// errors (e.g. a BOM within a number literal). By making this illegal we avoid having to
// deal with a lot of edge cases and remove the potential footgun of the bytes of a BOM
// being 'missing' when included in a string literal (the Windows RC compiler acts as
// if the codepoint was never part of the string literal).
'\u{FEFF}' => error.IllegalByteOrderMark,
// RC compiler (but without the cryptic errors). Silently dropping bytes still seems like
// enough of a footgun with no real use-cases that it's still worth erroring instead of
// emulating the RC compiler's behavior, though.
'\u{E000}' => error.IllegalPrivateUseCharacter,
// and miscompilations when used within string literals. We avoid the miscompilation
// within string literals and emit a warning, but outside of string literals it makes
// more sense to just disallow these codepoints.
0x900, 0xA00, 0xA0D, 0x2000, 0xD00, 0xFFFE, 0xFFFF => if (!in_string_literal) error.IllegalCodepointOutsideStringLiterals else return,
else => return,
};
self.error_context_token = .{
.id = .invalid,
.start = self.index,
.end = self.index + codepoint.byte_len,
.line_number = self.line_handler.line_number,
};
return err;
}
fn evaluatePreprocessorCommand(self: *Self, start: usize, end: usize) !void {
const token = Token{
.id = .preprocessor_command,
.start = start,
.end = end,
.line_number = self.line_handler.line_number,
};
errdefer self.error_context_token = token;
const full_command = self.buffer[start..end];
const code_page = (parsePragmaCodePage(full_command) catch |err| switch (err) {
error.NotPragma, error.NotCodePagePragma => return,
else => |e| return e,
}) orelse self.default_code_page;
// > This pragma is not supported in an included resource file (.rc)
//
// Even though the Win32 behavior is to just ignore such directives silently,
// this is an error in the lexer to allow for emitting warnings/errors when
// such directives are found if that's wanted. The intention is for the lexer
// to still be able to work correctly after this error is returned.
if (self.source_mappings) |source_mappings| {
if (!source_mappings.isRootFile(token.line_number)) {
return error.CodePagePragmaInIncludedFile;
}
}
self.seen_pragma_code_pages +|= 1;
self.last_pragma_code_page_token = token;
self.current_code_page = code_page;
}
pub fn getErrorDetails(self: Self, lex_err: LexError) ErrorDetails {
const err = switch (lex_err) {
error.UnfinishedStringLiteral => ErrorDetails.Error.unfinished_string_literal,
error.StringLiteralTooLong => return .{
.err = .string_literal_too_long,
.code_page = self.current_code_page,
.token = self.error_context_token.?,
.extra = .{ .number = self.max_string_literal_codepoints },
},
error.InvalidNumberWithExponent => ErrorDetails.Error.invalid_number_with_exponent,
error.InvalidDigitCharacterInNumberLiteral => ErrorDetails.Error.invalid_digit_character_in_number_literal,
error.IllegalByte => ErrorDetails.Error.illegal_byte,
error.IllegalByteOutsideStringLiterals => ErrorDetails.Error.illegal_byte_outside_string_literals,
error.IllegalCodepointOutsideStringLiterals => ErrorDetails.Error.illegal_codepoint_outside_string_literals,
error.IllegalByteOrderMark => ErrorDetails.Error.illegal_byte_order_mark,
error.IllegalPrivateUseCharacter => ErrorDetails.Error.illegal_private_use_character,
error.FoundCStyleEscapedQuote => ErrorDetails.Error.found_c_style_escaped_quote,
error.CodePagePragmaMissingLeftParen => ErrorDetails.Error.code_page_pragma_missing_left_paren,
error.CodePagePragmaMissingRightParen => ErrorDetails.Error.code_page_pragma_missing_right_paren,
error.CodePagePragmaInvalidCodePage => ErrorDetails.Error.code_page_pragma_invalid_code_page,
error.CodePagePragmaNotInteger => ErrorDetails.Error.code_page_pragma_not_integer,
error.CodePagePragmaOverflow => ErrorDetails.Error.code_page_pragma_overflow,
error.CodePagePragmaUnsupportedCodePage => ErrorDetails.Error.code_page_pragma_unsupported_code_page,
error.CodePagePragmaInIncludedFile => ErrorDetails.Error.code_page_pragma_in_included_file,
};
return .{
.err = err,
.code_page = self.current_code_page,
.token = self.error_context_token.?,
};
}
}
