diff --git a/src/main.c b/src/main.c index b486db5..0a10aa4 100644 --- a/src/main.c +++ b/src/main.c @@ -451,10 +451,13 @@ void string_test_misc(void) bool prefix1 = str_starts_with(str1, S("He")); bool prefix2 = str_starts_with(str1, S("")); bool prefix3 = str_starts_with(str1, S("false")); + bool prefix4 = str_starts_with(str1, S("Hello World")); + bool suffix1 = str_ends_with(str1, S("orld")); bool suffix2 = str_ends_with(str1, S("")); bool suffix3 = str_ends_with(str1, S("false")); - printf("matches: %d %d %d %d %d %d\n", prefix1, prefix2, prefix3, suffix1, suffix2, suffix3); + bool suffix4 = str_ends_with(str1, S("Hello World")); + printf("matches: %d %d %d %d %d %d %d %d\n", prefix1, prefix2, prefix3, prefix4, suffix1, suffix2, suffix3, suffix4); Str str2 = S(" Hello World\t \n"); Str str2_trim = str_ascii_trim(str2); @@ -465,7 +468,6 @@ void string_test_misc(void) Str str3_trim = str_ascii_trim(str3); printf("str3_trim: '%.*s'\n", str3_trim.len, str3_trim.str); - Str str4 = S(" \t\t "); Str str4_trim = str_ascii_trim(str4); printf("str4_trim: '%.*s'\n", str4_trim.len, str4_trim.str); @@ -499,6 +501,34 @@ void string_test_join(void) } +void print_runes(Str str) +{ + U32 offset = 0; + U32 len = str_rune_count(str); + printf("rune len %d\n", len); + while (offset < str.len) { + RuneDecode d = str_utf8_decode(str, offset); + printf("Rune: %d valid: %d\n", d.rune, d.valid); + offset += d.width; + } +} + +void string_test_utf8(void) +{ + Str str1 = S("Test 1ø2 ☺"); + + TRACEF("str1:\n"); + print_runes(str1); + + Str str2 = str_empty(); + TRACEF("str2:\n"); + print_runes(str2); + + Str str3 = S("1Å 2\xc6" "3\xc3"); + TRACEF("str3:\n"); + print_runes(str3); +} + int main(int argc, char *argv[]) { arena_test_1(); @@ -515,6 +545,7 @@ int main(int argc, char *argv[]) string_split_test_1(); string_test_misc(); string_test_join(); + string_test_utf8(); } diff --git a/src/string.c b/src/string.c index 49fdc9b..27fda30 100644 --- a/src/string.c +++ b/src/string.c @@ -37,12 +37,6 @@ Str str_push(Allocator *alloc, Str str) return str_create(mem, str.len); } -static INLINE bool char_is_ascii_whitespace(U8 byte) -{ - return byte == ' ' || byte == '\t' || byte == '\v' || byte == '\f' || byte == '\r' || byte == '\n'; - -} - Str str_ascii_trim_left(Str str) { U8 *start = str.str; @@ -129,7 +123,6 @@ void str_split(Str str, Str separators, StrVec *vec) Str str_join(Allocator *allocator, StrVec strings, Str separator) { - U32 joined_str_len = 0; for (U32 i = 0; i < strings.len; i++) { @@ -163,3 +156,226 @@ Str str_join(Allocator *allocator, StrVec strings, Str separator) return str_create(joined_str, joined_str_len); } + + +/* + * Allowed range of the first continuation byte for each range class. + * + * The normal continuation-byte interval is 0x80..0xBF. + */ +static const Utf8Range utf8_ranges[] = { + [UTF8_RANGE_NONE] = { 0x00, 0x00 }, + [UTF8_RANGE_80_BF] = { 0x80, 0xBF }, + [UTF8_RANGE_A0_BF] = { 0xA0, 0xBF }, + [UTF8_RANGE_80_9F] = { 0x80, 0x9F }, + [UTF8_RANGE_90_BF] = { 0x90, 0xBF }, + [UTF8_RANGE_80_8F] = { 0x80, 0x8F }, +}; + + +/* + * Classification of every possible first byte. + * + * The zero-initialized entries are invalid UTF-8 leading bytes: + * + * 80..BF continuation bytes cannot start a sequence + * C0..C1 would produce overlong two-byte encodings + * F5..FF would encode values above U+10FFFF or are otherwise invalid + * + * Valid leading-byte ranges: + * + * 00..7F ASCII + * C2..DF 2 bytes + * E0..EF 3 bytes + * F0..F4 4 bytes + * + * E0, ED, F0, and F4 receive special range classes because their second + * byte is constrained beyond the normal 80..BF continuation-byte range. + * + */ +static const U8 utf8_lead[256] = { + [0x00 ... 0x7F] = UTF8_DESC(1, UTF8_RANGE_NONE), + + [0xC2 ... 0xDF] = UTF8_DESC(2, UTF8_RANGE_80_BF), + + [0xE0] = UTF8_DESC(3, UTF8_RANGE_A0_BF), + + [0xE1 ... 0xEC] = UTF8_DESC(3, UTF8_RANGE_80_BF), + + [0xED] = UTF8_DESC(3, UTF8_RANGE_80_9F), + + [0xEE ... 0xEF] = UTF8_DESC(3, UTF8_RANGE_80_BF), + + [0xF0] = UTF8_DESC(4, UTF8_RANGE_90_BF), + + [0xF1 ... 0xF3] = UTF8_DESC(4, UTF8_RANGE_80_BF), + + [0xF4] = UTF8_DESC(4, UTF8_RANGE_80_8F), +}; + +/* + * Decode one Unicode scalar value from UTF-8. + * + * Valid input: + * + * returns the decoded Rune + * valid = true + * width = number of UTF-8 bytes consumed + * + * Invalid input: + * + * returns U+FFFD + * valid = false + * width = 1 + * + * Consuming one byte on malformed input guarantees forward progress and + * allows decoding to resume immediately after the offending byte. + * + * End of input: + * + * returns RUNE_EOF + * valid = false + * width = 0 + */ +RuneDecode _str_utf8_decode_full(Str str, U32 offset) +{ + const uint8_t b0 = str.str[offset]; + const uint8_t desc = utf8_lead[b0]; + const uint8_t width = UTF8_DESC_LENGTH(desc); + /* + * width == 0 covers: + * + * 80..BF + * C0..C1 + * F5..FF + */ + if (width == 0) + goto invalid; + + /* + * We must have the complete candidate sequence before accessing any + * continuation bytes below. + */ + if (str.len - offset < width) + goto invalid; + + /* + * Validate the first continuation byte. + * + * Its descriptor-selected range handles both the normal continuation + * range and the E0 / ED / F0 / F4 boundary cases. + */ + const Utf8Range range = utf8_ranges[UTF8_DESC_RANGE(desc)]; + const uint8_t b1 = str.str[offset + 1]; + + if (b1 < range.lo || b1 > range.hi) + goto invalid; + + /* + * Decode paths are intentionally unrolled. + * + * Besides avoiding a loop in this hot function, each case combines + * continuation-byte validation with scalar construction. The compiler + * therefore sees fixed-width operations for every UTF-8 length. + */ + switch (width) { + case 2: { + /* + * 110xxxxx 10xxxxxx + * + * C2..DF in the lead table already guarantees: + * + * - exactly a legal two-byte leader + * - no C0/C1 overlong encoding + * + * b1 has already been checked against 80..BF. + */ + const Rune rune = + ((Rune)(b0 & 0x1F) << 6) | + (Rune)(b1 & 0x3F); + + return (RuneDecode) { + .rune = rune, + .width = 2, + .valid = true, + }; + } + + case 3: { + /* + * 1110xxxx 10xxxxxx 10xxxxxx + * + * b1 has already been validated using the descriptor-specific + * range. Only b2 still needs a continuation-byte check. + */ + const uint8_t b2 = str.str[offset + 2]; + + if ((b2 & 0xC0) != 0x80) + goto invalid; + + const Rune rune = + ((Rune)(b0 & 0x0F) << 12) | + ((Rune)(b1 & 0x3F) << 6) | + (Rune)(b2 & 0x3F); + + return (RuneDecode) { + .rune = rune, + .width = 3, + .valid = true, + }; + } + + case 4: { + /* + * 11110xxx 10xxxxxx 10xxxxxx 10xxxxxx + * + * b1 has already been validated using the descriptor-specific + * range. b2 and b3 use the ordinary continuation-byte form. + */ + const uint8_t b2 = str.str[offset + 2]; + const uint8_t b3 = str.str[offset + 3]; + + /* + * Validate both remaining continuation bytes with one expression. + * + * For a valid continuation byte: + * + * byte & 0xC0 == 0x80 + * + * XOR with 0x80 therefore gives zero in the high two bits. + * ORing the two transformed bytes lets us test both at once. + */ + if ((((b2 ^ 0x80) | (b3 ^ 0x80)) & 0xC0) != 0) + goto invalid; + + const Rune rune = + ((Rune)(b0 & 0x07) << 18) | + ((Rune)(b1 & 0x3F) << 12) | + ((Rune)(b2 & 0x3F) << 6) | + (Rune)(b3 & 0x3F); + + return (RuneDecode) { + .rune = rune, + .width = 4, + .valid = true, + }; + } + + default: + /* + * The descriptor table can only produce widths 0..4. + * Width 1 was handled by the ASCII fast path. + */ + goto invalid; + } + + +invalid: + return (RuneDecode) { + .rune = RUNE_ERROR, + .width = 1, + .valid = false, + }; +} + + diff --git a/src/string.h b/src/string.h index 1b998bd..36658cf 100644 --- a/src/string.h +++ b/src/string.h @@ -111,3 +111,255 @@ struct StrVec { void str_split(Str str, Str separators, StrVec *vec); Str str_join(Allocator *allocator, StrVec strings, Str separator); + + + +static INLINE bool char_is_ascii_whitespace(U8 byte) +{ + return byte == ' ' || byte == '\t' || byte == '\v' || byte == '\f' || byte == '\r' || byte == '\n'; + +} + +static INLINE bool char_is_digit(U8 byte) +{ + return byte >= '0' && byte <= '9'; +} + + + +// UTF-8/Unicode + +typedef S32 Rune; + + +/* + * Unicode replacement character. + * + * Returned for malformed UTF-8 + */ +#define RUNE_ERROR ((Rune)0xFFFD) + +#define RUNE_EOF ((Rune)-1) +#define RUNE_INVALID (-1) + +// Maximum valid Unicode code point. +#define RUNE_MAX 0x10FFFF +#define RUNE_SURROGATE_MIN 0xD800 +#define RUNE_SURROGATE_MAX 0xDFFF +#define RUNE_1_MAX ((1 << 7)- 1) +#define RUNE_2_MAX ((1 << 11)- 1) +#define RUNE_3_MAX ((1 << 16)- 1) + +typedef struct { + Rune rune; + + /* + * Number of input bytes consumed. + * + * valid UTF-8 : 1..4 + * invalid : 1 + * end of input: 0 + */ + U8 width; + + /* + * True only when rune was decoded from valid UTF-8. + * + * This is required because U+FFFD itself may legitimately occur in + * valid input, so rune == RUNE_ERROR does not imply an error. + */ + bool valid; +} RuneDecode; + + +/* + * Each entry in utf8_lead[] is an 8-bit descriptor: + * + * 7 6 5 4 3 2 1 0 + * +---------+-----+ + * | range | len | + * +---------+-----+ + * + * len: + * Number of bytes in the UTF-8 sequence. + * + * 0 = byte cannot start a valid UTF-8 sequence + * 1 = ASCII + * 2 = two-byte UTF-8 sequence + * 3 = three-byte UTF-8 sequence + * 4 = four-byte UTF-8 sequence + * + * range: + * Selects the legal range for byte 1, the first continuation byte. + * + * This range is enough to encode all of UTF-8's special boundary rules: + * + * E0 A0..BF prevents overlong 3-byte encodings + * ED 80..9F excludes UTF-16 surrogate code points + * F0 90..BF prevents overlong 4-byte encodings + * F4 80..8F limits Unicode to U+10FFFF + * + * All continuation bytes after byte 1 simply use the normal 80..BF range. + */ + +#define UTF8_LENGTH_BITS 3u +#define UTF8_LENGTH_MASK ((1u << UTF8_LENGTH_BITS) - 1u) + +#define UTF8_DESC(length, range) \ + ((U8)((length) | ((U8)(range) << UTF8_LENGTH_BITS))) + +#define UTF8_DESC_LENGTH(desc) \ + ((U8)((desc) & UTF8_LENGTH_MASK)) + +#define UTF8_DESC_RANGE(desc) \ + ((U8)((desc) >> UTF8_LENGTH_BITS)) + + +/* + * Classes of allowed values for the first continuation byte. + */ +enum Utf8RangeClass { + /* + * No continuation byte exists. + * + * Used only by ASCII descriptors. + */ + UTF8_RANGE_NONE = 0, + + /* + * Normal continuation-byte range. + */ + UTF8_RANGE_80_BF, + + /* + * E0 xx xx: + * + * E0 80..9F would encode values below U+0800 and therefore be + * non-canonical overlong UTF-8. + */ + UTF8_RANGE_A0_BF, + + /* + * ED xx xx: + * + * ED A0..BF would encode U+D800..U+DFFF, which are UTF-16 surrogate + * code points and are not Unicode scalar values. + */ + UTF8_RANGE_80_9F, + + /* + * F0 xx xx xx: + * + * F0 80..8F would encode values below U+10000 and therefore be + * non-canonical overlong UTF-8. + */ + UTF8_RANGE_90_BF, + + /* + * F4 xx xx xx: + * + * F4 90..BF would encode values above Unicode's maximum scalar value, + * U+10FFFF. + */ + UTF8_RANGE_80_8F, +}; + + +typedef struct { + U8 lo; + U8 hi; +} Utf8Range; + +// Helper: +RuneDecode _str_utf8_decode_full(Str str, U32 offset); +/* + * Decode one Unicode scalar value from UTF-8. + * + * Valid input: + * + * returns the decoded Rune + * valid = true + * width = number of UTF-8 bytes consumed + * + * Invalid input: + * + * returns U+FFFD + * valid = false + * width = 1 + * + * Consuming one byte on malformed input guarantees forward progress and + * allows decoding to resume immediately after the offending byte. + * + * End of input: + * + * returns RUNE_EOF + * valid = false + * width = 0 + */ +static INLINE RuneDecode str_utf8_decode(Str str, U32 offset) +{ + if (str.len == 0 || str.len <= offset) { + return (RuneDecode) { + .rune = RUNE_EOF, + .width = 0, + .valid = false, + }; + } + + const uint8_t b0 = str.str[offset]; + + + /* + * Handle ascii, helps inlining for ascii cases + * + * Testing b0 directly also avoids consulting utf8_ranges[] for the + * overwhelmingly common single-byte case. + */ + if (b0 < 0x80) { + return (RuneDecode) { + .rune = (Rune)b0, + .width = 1, + .valid = true, + }; + } + + return _str_utf8_decode_full(str, offset); +} + +// Length of bytes to encode the Rune in utf-8. +// returns -1 if the Rune is not a valid value to encode in UTF-8. +static INLINE S32 rune_utf8_len(Rune rune) { + + if (rune < 0) { + return RUNE_INVALID; + } else if (rune <= RUNE_1_MAX) { + return 1; + } else if (rune <= RUNE_2_MAX) { + return 2; + } else if (rune > RUNE_SURROGATE_MIN && rune <= RUNE_SURROGATE_MAX) { + return RUNE_INVALID; + } else if (rune <= RUNE_3_MAX) { + return 3; + } else if (rune <= RUNE_MAX) { + return 4; + } + + return RUNE_INVALID; +} + +// Number of unicode Runes in str. Invalid UTF-8 is counted as 1 Rune. +// (1 invalud UTF-8 byte will be decoded to 1 Rune of RUNE_ERROR +static INLINE U32 str_rune_count(Str str) +{ + U32 offset = 0; + U32 runes = 0; + + RuneDecode d = str_utf8_decode(str, offset); + while (d.rune != RUNE_EOF) { + offset += d.width; + runes++; + d = str_utf8_decode(str, offset); + } + + return runes; +}