pub fn peek(self: *Self) ?[]const T self: *Self3348
3349
3350
3351
3352
3353
3354
3355
3356
3357
3358
3359
3360
3361
3362
3363
3364 pub fn peek(self: *Self) ?[]const T {
// move to beginning of token
while (self.index < self.buffer.len and self.isDelimiter(self.index)) : (self.index += switch (delimiter_type) {
.sequence => self.delimiter.len,
.any, .scalar => 1,
}) {}
const start = self.index;
if (start == self.buffer.len) {
return null;
}
// move to end of token
var end = start;
while (end < self.buffer.len and !self.isDelimiter(end)) : (end += 1) {}
return self.buffer[start..end];
}
Returns a slice of the current token, or null if tokenization is complete. Does not advance to the next token.