Utf8

pub const Utf8 = struct

File

Code

pub const Utf8 = struct {
    /// Implements decoding with rejection of ill-formed UTF-8 sequences based on section
    /// D92 of Chapter 3 of the Unicode standard (Table 3-7 specifically).
    ///
    /// Note: This does not match "U+FFFD Substitution of Maximal Subparts", but instead
    ///       matches the behavior of the Windows RC compiler.
    pub const WellFormedDecoder = struct {
        /// Like std.unicode.utf8ByteSequenceLength, but:
        /// - Rejects non-well-formed first bytes, i.e. C0-C1, F5-FF
        /// - Returns an optional value instead of an error union
        pub fn sequenceLength(first_byte: u8) ?u3 {
            return switch (first_byte) {
                0x00...0x7F => 1,
                0xC2...0xDF => 2,
                0xE0...0xEF => 3,
                0xF0...0xF4 => 4,
                else => null,
            };
        }

        fn isContinuationByte(byte: u8) bool {
            return switch (byte) {
                0x80...0xBF => true,
                else => false,
            };
        }

        pub fn decode(bytes: []const u8) Codepoint {
            std.debug.assert(bytes.len > 0);
            const first_byte = bytes[0];
            const expected_len = sequenceLength(first_byte) orelse {
                return .{ .value = Codepoint.invalid, .byte_len = 1 };
            };
            if (expected_len == 1) return .{ .value = first_byte, .byte_len = 1 };

            var value: u21 = first_byte & 0b00011111;
            var byte_index: u8 = 1;
            while (byte_index < @min(bytes.len, expected_len)) : (byte_index += 1) {
                const byte = bytes[byte_index];
                // See Table 3-7 of D92 in Chapter 3 of the Unicode Standard
                const valid: bool = switch (byte_index) {
                    1 => switch (first_byte) {
                        0xE0 => switch (byte) {
                            0xA0...0xBF => true,
                            else => false,
                        },
                        0xED => switch (byte) {
                            0x80...0x9F => true,
                            else => false,
                        },
                        0xF0 => switch (byte) {
                            0x90...0xBF => true,
                            else => false,
                        },
                        0xF4 => switch (byte) {
                            0x80...0x8F => true,
                            else => false,
                        },
                        else => switch (byte) {
                            0x80...0xBF => true,
                            else => false,
                        },
                    },
                    else => switch (byte) {
                        0x80...0xBF => true,
                        else => false,
                    },
                };

                if (!valid) {
                    var len = byte_index;
                    // Only include the byte in the invalid sequence if it's in the range
                    // of a continuation byte. All other values should not be included in the
                    // invalid sequence.
                    if (isContinuationByte(byte)) len += 1;
                    return .{ .value = Codepoint.invalid, .byte_len = len };
                }

                value <<= 6;
                value |= byte & 0b00111111;
            }
            if (byte_index != expected_len) {
                return .{ .value = Codepoint.invalid, .byte_len = byte_index };
            }
            return .{ .value = value, .byte_len = expected_len };
        }
    };
}