Valid escapes:
"" -> "
\a, \A => 0x08 (not 0x07 like in C)
\n => 0x0A
\r => 0x0D
\t, \T => 0x09
\ =>
\nnn => byte with numeric value given by nnn interpreted as octal
(wraps on overflow, number of digits can be 1-3 for ASCII strings
and 1-7 for wide strings)
\xhh => byte with numeric value given by hh interpreted as hex
(number of digits can be 0-2 for ASCII strings and 0-4 for
wide strings)
<\r+> =>
<[\r\n\t ]+> => <nothing>
Special cases: <\t> => 1-8 spaces, dependent on columns in the source rc file itself <\r> => <nothing> <\n+><\w+?\n?> => <space><\n>
Special, especially weird case: "" => " NOTE: This leads to footguns because the preprocessor can start parsing things out-of-sync with the RC compiler, expanding macros within string literals, etc. This parse function handles this case the same as the Windows RC compiler, but " within a string literal is treated as an error by the lexer, so the relevant branches should never actually be hit during this function.
pub const IterativeStringParser = structpub const IterativeStringParser = struct {
source: []const u8,
code_page: SupportedCodePage,
/// The type of the string inferred by the prefix (L"" or "")
/// This is what matters for things like the maximum digits in an
/// escape sequence, whether or not invalid escape sequences are skipped, etc.
declared_string_type: StringType,
pending_codepoint: ?u21 = null,
num_pending_spaces: u8 = 0,
index: usize = 0,
column: usize = 0,
diagnostics: ?DiagnosticsContext = null,
seen_tab: bool = false,
const State = enum {
normal,
quote,
newline,
escaped,
escaped_cr,
escaped_newlines,
escaped_octal,
escaped_hex,
};
pub fn init(bytes: SourceBytes, options: StringParseOptions) IterativeStringParser {
const declared_string_type: StringType = switch (bytes.slice[0]) {
'L', 'l' => .wide,
else => .ascii,
};
var source = bytes.slice[1 .. bytes.slice.len - 1]; // remove ""
var column = options.start_column + 1; // for the removed "
if (declared_string_type == .wide) {
source = source[1..]; // remove L
column += 1; // for the removed L
}
return .{
.source = source,
.code_page = bytes.code_page,
.declared_string_type = declared_string_type,
.column = column,
.diagnostics = options.diagnostics,
};
}
pub const ParsedCodepoint = struct {
codepoint: u21,
/// Note: If this is true, `codepoint` will have an effective maximum value
/// of 0xFFFF, as `codepoint` is calculated using wrapping arithmetic on a u16.
/// If the value needs to be truncated to a smaller integer (e.g. for ASCII string
/// literals), then that must be done by the caller.
from_escaped_integer: bool = false,
/// Denotes that the codepoint is:
/// - Escaped (has a \ in front of it), and
/// - Has a value >= U+10000, meaning it would be encoded as a surrogate
/// pair in UTF-16, and
/// - Is part of a wide string literal
///
/// Normally in wide string literals, invalid escapes are omitted
/// during parsing (the codepoints are not returned at all during
/// the `next` call), but this is a special case in which the
/// escape only applies to the high surrogate pair of the codepoint.
///
/// TODO: Maybe just return the low surrogate codepoint by itself in this case.
escaped_surrogate_pair: bool = false,
};
pub fn next(self: *IterativeStringParser) std.mem.Allocator.Error!?ParsedCodepoint {
const result = try self.nextUnchecked();
if (self.diagnostics != null and result != null and !result.?.from_escaped_integer) {
switch (result.?.codepoint) {
0x0900, 0x0A00, 0x0A0D, 0x2000, 0x0D00 => {
const err: ErrorDetails.Error = if (result.?.codepoint == 0xD00)
.rc_would_miscompile_codepoint_skip
else
.rc_would_miscompile_codepoint_whitespace;
try self.diagnostics.?.diagnostics.append(ErrorDetails{
.err = err,
.type = .warning,
.code_page = self.code_page,
.token = self.diagnostics.?.token,
.extra = .{ .number = result.?.codepoint },
});
},
0xFFFE, 0xFFFF => {
try self.diagnostics.?.diagnostics.append(ErrorDetails{
.err = .rc_would_miscompile_codepoint_bom,
.type = .warning,
.code_page = self.code_page,
.token = self.diagnostics.?.token,
.extra = .{ .number = result.?.codepoint },
});
try self.diagnostics.?.diagnostics.append(ErrorDetails{
.err = .rc_would_miscompile_codepoint_bom,
.type = .note,
.code_page = self.code_page,
.token = self.diagnostics.?.token,
.print_source_line = false,
.extra = .{ .number = result.?.codepoint },
});
},
else => {},
}
}
return result;
}
pub fn nextUnchecked(self: *IterativeStringParser) std.mem.Allocator.Error!?ParsedCodepoint {
if (self.num_pending_spaces > 0) {
// Ensure that we don't get into this predicament so we can ensure that
// the order of processing any pending stuff doesn't matter
std.debug.assert(self.pending_codepoint == null);
self.num_pending_spaces -= 1;
return .{ .codepoint = ' ' };
}
if (self.pending_codepoint) |pending_codepoint| {
self.pending_codepoint = null;
return .{ .codepoint = pending_codepoint };
}
if (self.index >= self.source.len) return null;
var state: State = .normal;
var string_escape_n: u16 = 0;
var string_escape_i: u8 = 0;
const max_octal_escape_digits: u8 = switch (self.declared_string_type) {
.ascii => 3,
.wide => 7,
};
const max_hex_escape_digits: u8 = switch (self.declared_string_type) {
.ascii => 2,
.wide => 4,
};
var backtrack: bool = undefined;
while (self.code_page.codepointAt(self.index, self.source)) |codepoint| : ({
if (!backtrack) self.index += codepoint.byte_len;
}) {
backtrack = false;
const c = codepoint.value;
defer {
if (!backtrack) {
if (c == '\t') {
self.column += columnsUntilTabStop(self.column, 8);
} else {
self.column += codepoint.byte_len;
}
}
}
switch (state) {
.normal => switch (c) {
'\\' => state = .escaped,
'"' => state = .quote,
'\r' => {},
'\n' => state = .newline,
'\t' => {
// Only warn about a tab getting converted to spaces once per string
if (self.diagnostics != null and !self.seen_tab) {
try self.diagnostics.?.diagnostics.append(ErrorDetails{
.err = .tab_converted_to_spaces,
.type = .warning,
.code_page = self.code_page,
.token = self.diagnostics.?.token,
});
try self.diagnostics.?.diagnostics.append(ErrorDetails{
.err = .tab_converted_to_spaces,
.type = .note,
.code_page = self.code_page,
.token = self.diagnostics.?.token,
.print_source_line = false,
});
self.seen_tab = true;
}
const cols = columnsUntilTabStop(self.column, 8);
self.num_pending_spaces = @intCast(cols - 1);
self.index += codepoint.byte_len;
return .{ .codepoint = ' ' };
},
else => {
self.index += codepoint.byte_len;
return .{ .codepoint = c };
},
},
.quote => switch (c) {
'"' => {
// "" => "
self.index += codepoint.byte_len;
return .{ .codepoint = '"' };
},
else => unreachable, // this is a bug in the lexer
},
.newline => switch (c) {
'\r', ' ', '\t', '\n', '\x0b', '\x0c', '\xa0' => {},
else => {
// we intentionally avoid incrementing self.index
// to handle the current char in the next call,
// and we set backtrack so column count is handled correctly
backtrack = true;
// <space><newline>
self.pending_codepoint = '\n';
return .{ .codepoint = ' ' };
},
},
.escaped => switch (c) {
'\r' => state = .escaped_cr,
'\n' => state = .escaped_newlines,
'0'...'7' => {
string_escape_n = std.fmt.charToDigit(@intCast(c), 8) catch unreachable;
string_escape_i = 1;
state = .escaped_octal;
},
'x', 'X' => {
string_escape_n = 0;
string_escape_i = 0;
state = .escaped_hex;
},
else => {
switch (c) {
'a', 'A' => {
self.index += codepoint.byte_len;
// might be a bug in RC, but matches its behavior
return .{ .codepoint = '\x08' };
},
'n' => {
self.index += codepoint.byte_len;
return .{ .codepoint = '\n' };
},
'r' => {
self.index += codepoint.byte_len;
return .{ .codepoint = '\r' };
},
't', 'T' => {
self.index += codepoint.byte_len;
return .{ .codepoint = '\t' };
},
'\\' => {
self.index += codepoint.byte_len;
return .{ .codepoint = '\\' };
},
'"' => {
// \" is a special case that doesn't get the \ included,
backtrack = true;
},
else => switch (self.declared_string_type) {
.wide => {
// All invalid escape sequences are skipped in wide strings,
// but there is a special case around \<tab> where the \
// is skipped but the tab character is processed.
// It's actually a bit weirder than that, though, since
// the preprocessor is the one that does the <tab> -> spaces
// conversion, so it goes something like this:
//
// Before preprocessing: L"\<tab>"
// After preprocessing: L"\ "
//
// So the parser only sees an escaped space character followed
// by some other number of spaces >= 0.
//
// However, our preprocessor keeps tab characters intact, so we emulate
// the above behavior by skipping the \ and then outputting one less
// space than normal for the <tab> character.
if (c == '\t') {
// Only warn about a tab getting converted to spaces once per string
if (self.diagnostics != null and !self.seen_tab) {
try self.diagnostics.?.diagnostics.append(ErrorDetails{
.err = .tab_converted_to_spaces,
.type = .warning,
.code_page = self.code_page,
.token = self.diagnostics.?.token,
});
try self.diagnostics.?.diagnostics.append(ErrorDetails{
.err = .tab_converted_to_spaces,
.type = .note,
.code_page = self.code_page,
.token = self.diagnostics.?.token,
.print_source_line = false,
});
self.seen_tab = true;
}
const cols = columnsUntilTabStop(self.column, 8);
// If the tab character would only be converted to a single space,
// then we can just skip both the \ and the <tab> and move on.
if (cols > 1) {
self.num_pending_spaces = @intCast(cols - 2);
self.index += codepoint.byte_len;
return .{ .codepoint = ' ' };
}
}
// There's a second special case when the codepoint would be encoded
// as a surrogate pair in UTF-16, as the escape 'applies' to the
// high surrogate pair only in this instance. This is a side-effect
// of the Win32 RC compiler preprocessor outputting UTF-16 and the
// compiler itself seemingly working on code units instead of code points
// in this particular instance.
//
// We emulate this behavior by emitting the codepoint, but with a marker
// that indicates that it needs to be handled specially.
if (c >= 0x10000 and c != code_pages.Codepoint.invalid) {
self.index += codepoint.byte_len;
return .{ .codepoint = c, .escaped_surrogate_pair = true };
}
},
.ascii => {
// we intentionally avoid incrementing self.index
// to handle the current char in the next call,
// and we set backtrack so column count is handled correctly
backtrack = true;
return .{ .codepoint = '\\' };
},
},
}
state = .normal;
},
},
.escaped_cr => switch (c) {
'\r' => {},
'\n' => state = .escaped_newlines,
else => {
// we intentionally avoid incrementing self.index
// to handle the current char in the next call,
// and we set backtrack so column count is handled correctly
backtrack = true;
return .{ .codepoint = '\\' };
},
},
.escaped_newlines => switch (c) {
'\r', '\n', '\t', ' ', '\x0b', '\x0c', '\xa0' => {},
else => {
// backtrack so that we handle the current char properly
backtrack = true;
state = .normal;
},
},
.escaped_octal => switch (c) {
'0'...'7' => {
// Note: We use wrapping arithmetic on a u16 here since there's been no observed
// string parsing scenario where an escaped integer with a value >= the u16
// max is interpreted as anything but the truncated u16 value.
string_escape_n *%= 8;
string_escape_n +%= std.fmt.charToDigit(@intCast(c), 8) catch unreachable;
string_escape_i += 1;
if (string_escape_i == max_octal_escape_digits) {
self.index += codepoint.byte_len;
return .{ .codepoint = string_escape_n, .from_escaped_integer = true };
}
},
else => {
// we intentionally avoid incrementing self.index
// to handle the current char in the next call,
// and we set backtrack so column count is handled correctly
backtrack = true;
// write out whatever byte we have parsed so far
return .{ .codepoint = string_escape_n, .from_escaped_integer = true };
},
},
.escaped_hex => switch (c) {
'0'...'9', 'a'...'f', 'A'...'F' => {
string_escape_n *= 16;
string_escape_n += std.fmt.charToDigit(@intCast(c), 16) catch unreachable;
string_escape_i += 1;
if (string_escape_i == max_hex_escape_digits) {
self.index += codepoint.byte_len;
return .{ .codepoint = string_escape_n, .from_escaped_integer = true };
}
},
else => {
// we intentionally avoid incrementing self.index
// to handle the current char in the next call,
// and we set backtrack so column count is handled correctly
backtrack = true;
// write out whatever byte we have parsed so far
// (even with 0 actual digits, \x alone parses to 0)
const escaped_value = string_escape_n;
return .{ .codepoint = escaped_value, .from_escaped_integer = true };
},
},
}
}
switch (state) {
.normal, .escaped_newlines => {},
.newline => {
// <space><newline>
self.pending_codepoint = '\n';
return .{ .codepoint = ' ' };
},
.escaped, .escaped_cr => return .{ .codepoint = '\\' },
.escaped_octal, .escaped_hex => {
return .{ .codepoint = string_escape_n, .from_escaped_integer = true };
},
.quote => unreachable, // this is a bug in the lexer
}
return null;
}
}