#include "string.h" Str str_empty(void) { static const U8 empty_; static const Str str = {(U8 *)&empty_, 0}; return str; } Str str_slice(Str str, U32 start, U32 len) { Assert(start <= str.len); Assert(len <= str.len - start); Str new_str = str_create(&str.str[start], len); return new_str; } Str str_concat(Allocator *alloc, Str first, Str second) { Str str; Assert(first.len <= U32_Max - second.len); str.len = first.len + second.len; str.str = allocator_push(alloc, str.len, 1);; MemCopy(str.str, first.str, first.len); MemCopy(&str.str[first.len], second.str, second.len); return str; } Str str_push(Allocator *alloc, Str str) { U8 *mem = (U8 *)allocator_push(alloc, str.len, 1); MemCopy(mem, str.str, str.len); return str_create(mem, str.len); } Str str_ascii_trim_left(Str str) { U8 *start = str.str; U8 *end = str.str + str.len; while (start != end) { if (!char_is_ascii_whitespace(*start)) { break; } start++; } return str_create(start, (U32)(end - start)); } Str str_ascii_trim_right(Str str) { U8 *start = str.str; U8 *end = str.str + str.len; while (start != end) { if (!char_is_ascii_whitespace(*(end - 1))) { break; } end--; } return str_create(start, (U32)(end - start)); } Str str_ascii_trim(Str str) { Str new_str = str_ascii_trim_left(str); new_str = str_ascii_trim_right(new_str); return new_str; } S64 str_index(Str haystack, Str needle) { if (needle.len == 0) { return 0; } if (needle.len == 1) { return str_find_byte(haystack, needle.str[0]); } if (needle.len > haystack.len) { return -1; } U8 *found = MemFind(haystack.str, haystack.len, needle.str, needle.len); if (found != NULL) { return (S64)(found - haystack.str); } return -1; } static INLINE S16 char_hex_to_binary(U8 c) { if (c >='0' && c <='9') return c - '0'; else if (c >= 'A' && c <= 'F') return c - 'A' + 10; else if (c >= 'a' && c <= 'f') return c - 'a' + 10; return -1; } static const char hex_chars[] = "0123456789ABCDEF"; Str str_hex_encode(Str binary_data, Str hex_result) { Assert(hex_result.len >= binary_data.len * 2); U32 hex_len = binary_data.len * 2; for (U32 i = 0, t = 0; i < binary_data.len; ++i,t+=2) { U8 high_nibble; U8 low_nibble; high_nibble = (binary_data.str[i] & 0xF0) >> 4 ; low_nibble = binary_data.str[i] & 0x0F ; hex_result.str[t] = hex_chars[high_nibble]; hex_result.str[t+1] = hex_chars[low_nibble]; } hex_result.len = hex_len; return hex_result; } Str str_hex_decode(Str hex_str, Str result) { Assert(result.len >= hex_str.len/2); U32 dest_pos = 0; U32 source_pos = 0; while (source_pos + 1 < hex_str.len) { if (char_is_ascii_whitespace(hex_str.str[source_pos])) { source_pos++; continue; } S16 high_nibble; S16 low_nibble; high_nibble = char_hex_to_binary(hex_str.str[source_pos]); low_nibble = char_hex_to_binary(hex_str.str[source_pos + 1]); if (high_nibble == -1 || low_nibble == -1) { result.len = 0; return result; } result.str[dest_pos] = (high_nibble << 4 ) | low_nibble; source_pos += 2; dest_pos++; } result.len = dest_pos; return result; } // split @str on any byte in @separators, place split strings in @vec void str_split(Str str, Str separators, StrVec *vec) { if (vec->max_len == 0 || str.len == 0) { return; } U32 start_pos = 0; for (U32 i = 0; i < str.len; i++) { U8 byte = str.str[i]; if (str_find_byte(separators, byte) != -1) { if (vec->len >= vec->max_len) { break; } Str sub_str = str_slice(str, start_pos, i - start_pos); vec->strings[vec->len] = sub_str; vec->len++; start_pos = i + 1; } } if (start_pos <= str.len && vec->len < vec->max_len) { Str sub_str = str_slice(str, start_pos, str.len - start_pos); vec->strings[vec->len] = sub_str; vec->len++; } } Str str_join(Allocator *allocator, StrVec strings, Str separator) { U32 joined_str_len = 0; for (U32 i = 0; i < strings.len; i++) { U32 part_len = strings.strings[i].len; Assert(joined_str_len <= U32_Max - part_len); // overflow joined_str_len += part_len; } if (strings.len > 1) { U32 separator_count = strings.len - 1; Assert(separator.len <= U32_Max / separator_count); // overflow Assert(joined_str_len <= U32_Max - separator.len * separator_count); // overflow joined_str_len += separator.len * separator_count; } U8 *joined_str = allocator_push(allocator, joined_str_len, 1); if (strings.len > 0) { U32 pos = 0; Str *str = &strings.strings[0]; MemCopy(&joined_str[pos], str->str, str->len); pos += str->len; for (U32 i = 1; i < strings.len; i++) { str = &strings.strings[i]; MemCopy(&joined_str[pos], separator.str, separator.len); pos += separator.len; MemCopy(&joined_str[pos], str->str, str->len); pos += str->len; } } return str_create(joined_str, joined_str_len); } /* * Each entry in utf8_lead[] is an 8-bit descriptor: * * 7 6 5 4 3 2 1 0 * +---------+-----+ * | range | len | * +---------+-----+ * * len: * Number of bytes in the UTF-8 sequence. * * 0 = byte cannot start a valid UTF-8 sequence * 1 = ASCII * 2 = two-byte UTF-8 sequence * 3 = three-byte UTF-8 sequence * 4 = four-byte UTF-8 sequence * * range: * Selects the legal range for byte 1, the first continuation byte. * * This range is enough to encode all of UTF-8's special boundary rules: * * E0 A0..BF prevents overlong 3-byte encodings * ED 80..9F excludes UTF-16 surrogate code points * F0 90..BF prevents overlong 4-byte encodings * F4 80..8F limits Unicode to U+10FFFF * * All continuation bytes after byte 1 simply use the normal 80..BF range. */ #define UTF8_LENGTH_BITS 3u #define UTF8_LENGTH_MASK ((1u << UTF8_LENGTH_BITS) - 1u) #define UTF8_DESC(length, range) \ ((U8)((length) | ((U8)(range) << UTF8_LENGTH_BITS))) #define UTF8_DESC_LENGTH(desc) \ ((U8)((desc) & UTF8_LENGTH_MASK)) #define UTF8_DESC_RANGE(desc) \ ((U8)((desc) >> UTF8_LENGTH_BITS)) /* * Classes of allowed values for the first continuation byte. */ enum Utf8RangeClass { /* * No continuation byte exists. * * Used only by ASCII descriptors. */ UTF8_RANGE_NONE = 0, /* * Normal continuation-byte range. */ UTF8_RANGE_80_BF, /* * E0 xx xx: * * E0 80..9F would encode values below U+0800 and therefore be * non-canonical overlong UTF-8. */ UTF8_RANGE_A0_BF, /* * ED xx xx: * * ED A0..BF would encode U+D800..U+DFFF, which are UTF-16 surrogate * code points and are not Unicode scalar values. */ UTF8_RANGE_80_9F, /* * F0 xx xx xx: * * F0 80..8F would encode values below U+10000 and therefore be * non-canonical overlong UTF-8. */ UTF8_RANGE_90_BF, /* * F4 xx xx xx: * * F4 90..BF would encode values above Unicode's maximum scalar value, * U+10FFFF. */ UTF8_RANGE_80_8F, }; typedef struct { U8 lo; U8 hi; } Utf8Range; /* * Allowed range of the first continuation byte for each range class. * * The normal continuation-byte interval is 0x80..0xBF. */ static const Utf8Range utf8_ranges[] = { [UTF8_RANGE_NONE] = { 0x00, 0x00 }, [UTF8_RANGE_80_BF] = { 0x80, 0xBF }, [UTF8_RANGE_A0_BF] = { 0xA0, 0xBF }, [UTF8_RANGE_80_9F] = { 0x80, 0x9F }, [UTF8_RANGE_90_BF] = { 0x90, 0xBF }, [UTF8_RANGE_80_8F] = { 0x80, 0x8F }, }; /* * Each entry in utf8_lead[] is an 8-bit descriptor: * * 7 6 5 4 3 2 1 0 * +---------+-----+ * | range | len | * +---------+-----+ * * len: * Number of bytes in the UTF-8 sequence. * * 0 = byte cannot start a valid UTF-8 sequence * 1 = ASCII * 2 = two-byte UTF-8 sequence * 3 = three-byte UTF-8 sequence * 4 = four-byte UTF-8 sequence * * range: * Selects the legal range for byte 1, the first continuation byte. * * This range is enough to encode all of UTF-8's special boundary rules: * * E0 A0..BF prevents overlong 3-byte encodings * ED 80..9F excludes UTF-16 surrogate code points * F0 90..BF prevents overlong 4-byte encodings * F4 80..8F limits Unicode to U+10FFFF * * All continuation bytes after byte 1 simply use the normal 80..BF range. */ /* * Classification of every possible first byte. * * The zero-initialized entries are invalid UTF-8 leading bytes: * * 80..BF continuation bytes cannot start a sequence * C0..C1 would produce overlong two-byte encodings * F5..FF would encode values above U+10FFFF or are otherwise invalid * * Valid leading-byte ranges: * * 00..7F ASCII * C2..DF 2 bytes * E0..EF 3 bytes * F0..F4 4 bytes * * E0, ED, F0, and F4 receive special range classes because their second * byte is constrained beyond the normal 80..BF continuation-byte range. * */ static const U8 utf8_lead[256] = { [0x00 ... 0x7F] = UTF8_DESC(1, UTF8_RANGE_NONE), [0xC2 ... 0xDF] = UTF8_DESC(2, UTF8_RANGE_80_BF), [0xE0] = UTF8_DESC(3, UTF8_RANGE_A0_BF), [0xE1 ... 0xEC] = UTF8_DESC(3, UTF8_RANGE_80_BF), [0xED] = UTF8_DESC(3, UTF8_RANGE_80_9F), [0xEE ... 0xEF] = UTF8_DESC(3, UTF8_RANGE_80_BF), [0xF0] = UTF8_DESC(4, UTF8_RANGE_90_BF), [0xF1 ... 0xF3] = UTF8_DESC(4, UTF8_RANGE_80_BF), [0xF4] = UTF8_DESC(4, UTF8_RANGE_80_8F), }; /* * Decode one Unicode scalar value from UTF-8. * * Valid input: * * returns the decoded Rune * valid = true * width = number of UTF-8 bytes consumed * * Invalid input: * * returns U+FFFD * valid = false * width = 1 * * Consuming one byte on malformed input guarantees forward progress and * allows decoding to resume immediately after the offending byte. * * End of input: * * returns RUNE_EOF * valid = false * width = 0 */ RuneDecode _str_utf8_decode_full(Str str, U32 offset) { if (str.len == 0 || str.len <= offset) { return (RuneDecode) { .rune = RUNE_EOF, .width = 0, .valid = false, }; } U8 b0 = str.str[offset]; U8 desc = utf8_lead[b0]; U8 width = UTF8_DESC_LENGTH(desc); /* * width == 0 covers: * * 80..BF * C0..C1 * F5..FF */ if (width == 0) goto invalid; /* * We must have the complete candidate sequence before accessing any * continuation bytes below. */ if (str.len - offset < width) goto invalid; /* * Validate the first continuation byte. * * Its descriptor-selected range handles both the normal continuation * range and the E0 / ED / F0 / F4 boundary cases. */ Utf8Range range = utf8_ranges[UTF8_DESC_RANGE(desc)]; U8 b1 = str.str[offset + 1]; if (b1 < range.lo || b1 > range.hi) goto invalid; /* * Decode paths are intentionally unrolled. * * Besides avoiding a loop in this hot function, each case combines * continuation-byte validation with scalar construction. The compiler * therefore sees fixed-width operations for every UTF-8 length. */ switch (width) { case 2: { /* * 110xxxxx 10xxxxxx * * C2..DF in the lead table already guarantees: * * - exactly a legal two-byte leader * - no C0/C1 overlong encoding * * b1 has already been checked against 80..BF. */ Rune rune = ((Rune)(b0 & 0x1F) << 6) | (Rune)(b1 & 0x3F); return (RuneDecode) { .rune = rune, .width = 2, .valid = true, }; } case 3: { /* * 1110xxxx 10xxxxxx 10xxxxxx * * b1 has already been validated using the descriptor-specific * range. Only b2 still needs a continuation-byte check. */ U8 b2 = str.str[offset + 2]; if ((b2 & 0xC0) != 0x80) goto invalid; Rune rune = ((Rune)(b0 & 0x0F) << 12) | ((Rune)(b1 & 0x3F) << 6) | (Rune)(b2 & 0x3F); return (RuneDecode) { .rune = rune, .width = 3, .valid = true, }; } case 4: { /* * 11110xxx 10xxxxxx 10xxxxxx 10xxxxxx * * b1 has already been validated using the descriptor-specific * range. b2 and b3 use the ordinary continuation-byte form. */ U8 b2 = str.str[offset + 2]; U8 b3 = str.str[offset + 3]; /* * Validate both remaining continuation bytes with one expression. * * For a valid continuation byte: * * byte & 0xC0 == 0x80 * * XOR with 0x80 therefore gives zero in the high two bits. * ORing the two transformed bytes lets us test both at once. */ if ((((b2 ^ 0x80) | (b3 ^ 0x80)) & 0xC0) != 0) goto invalid; Rune rune = ((Rune)(b0 & 0x07) << 18) | ((Rune)(b1 & 0x3F) << 12) | ((Rune)(b2 & 0x3F) << 6) | (Rune)(b3 & 0x3F); return (RuneDecode) { .rune = rune, .width = 4, .valid = true, }; } default: /* * The descriptor table can only produce widths 0..4. * Width 1 was handled by the ASCII fast path. */ goto invalid; } invalid: return (RuneDecode) { .rune = RUNE_ERROR, .width = 1, .valid = false, }; }