noselasd/nob
1
0
Fork 0
Code Issues Pull Requests 1 Actions Packages Projects Releases Wiki Activity

Basic UTF-8 functions

Browse Source
This commit is contained in:
noselasd
2026-08-10 23:32:08 +02:00
parent 426534e066
commit cb009866ff
3 changed files with 508 additions and 9 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/string.c
+223 -7
View File
@@ -37,12 +37,6 @@ Str str_push(Allocator *alloc, Str str)
return str_create(mem, str.len);
}
static INLINE bool char_is_ascii_whitespace(U8 byte)
{
return byte == ' ' || byte == '\t' || byte == '\v' || byte == '\f' || byte == '\r' || byte == '\n';
}
Str str_ascii_trim_left(Str str)
{
U8 *start = str.str;
@@ -129,7 +123,6 @@ void str_split(Str str, Str separators, StrVec *vec)
Str str_join(Allocator *allocator, StrVec strings, Str separator)
{
U32 joined_str_len = 0;
for (U32 i = 0; i < strings.len; i++) {
@@ -163,3 +156,226 @@ Str str_join(Allocator *allocator, StrVec strings, Str separator)
return str_create(joined_str, joined_str_len);
}
/*
* Allowed range of the first continuation byte for each range class.
*
* The normal continuation-byte interval is 0x80..0xBF.
*/
static const Utf8Range utf8_ranges[] = {
[UTF8_RANGE_NONE] = { 0x00, 0x00 },
[UTF8_RANGE_80_BF] = { 0x80, 0xBF },
[UTF8_RANGE_A0_BF] = { 0xA0, 0xBF },
[UTF8_RANGE_80_9F] = { 0x80, 0x9F },
[UTF8_RANGE_90_BF] = { 0x90, 0xBF },
[UTF8_RANGE_80_8F] = { 0x80, 0x8F },
};
/*
* Classification of every possible first byte.
*
* The zero-initialized entries are invalid UTF-8 leading bytes:
*
* 80..BF continuation bytes cannot start a sequence
* C0..C1 would produce overlong two-byte encodings
* F5..FF would encode values above U+10FFFF or are otherwise invalid
*
* Valid leading-byte ranges:
*
* 00..7F ASCII
* C2..DF 2 bytes
* E0..EF 3 bytes
* F0..F4 4 bytes
*
* E0, ED, F0, and F4 receive special range classes because their second
* byte is constrained beyond the normal 80..BF continuation-byte range.
*
*/
static const U8 utf8_lead[256] = {
[0x00 ... 0x7F] = UTF8_DESC(1, UTF8_RANGE_NONE),
[0xC2 ... 0xDF] = UTF8_DESC(2, UTF8_RANGE_80_BF),
[0xE0] = UTF8_DESC(3, UTF8_RANGE_A0_BF),
[0xE1 ... 0xEC] = UTF8_DESC(3, UTF8_RANGE_80_BF),
[0xED] = UTF8_DESC(3, UTF8_RANGE_80_9F),
[0xEE ... 0xEF] = UTF8_DESC(3, UTF8_RANGE_80_BF),
[0xF0] = UTF8_DESC(4, UTF8_RANGE_90_BF),
[0xF1 ... 0xF3] = UTF8_DESC(4, UTF8_RANGE_80_BF),
[0xF4] = UTF8_DESC(4, UTF8_RANGE_80_8F),
};
/*
* Decode one Unicode scalar value from UTF-8.
*
* Valid input:
*
* returns the decoded Rune
* valid = true
* width = number of UTF-8 bytes consumed
*
* Invalid input:
*
* returns U+FFFD
* valid = false
* width = 1
*
* Consuming one byte on malformed input guarantees forward progress and
* allows decoding to resume immediately after the offending byte.
*
* End of input:
*
* returns RUNE_EOF
* valid = false
* width = 0
*/
RuneDecode _str_utf8_decode_full(Str str, U32 offset)
{
const uint8_t b0 = str.str[offset];
const uint8_t desc = utf8_lead[b0];
const uint8_t width = UTF8_DESC_LENGTH(desc);
/*
* width == 0 covers:
*
* 80..BF
* C0..C1
* F5..FF
*/
if (width == 0)
goto invalid;
/*
* We must have the complete candidate sequence before accessing any
* continuation bytes below.
*/
if (str.len - offset < width)
goto invalid;
/*
* Validate the first continuation byte.
*
* Its descriptor-selected range handles both the normal continuation
* range and the E0 / ED / F0 / F4 boundary cases.
*/
const Utf8Range range = utf8_ranges[UTF8_DESC_RANGE(desc)];
const uint8_t b1 = str.str[offset + 1];
if (b1 < range.lo || b1 > range.hi)
goto invalid;
/*
* Decode paths are intentionally unrolled.
*
* Besides avoiding a loop in this hot function, each case combines
* continuation-byte validation with scalar construction. The compiler
* therefore sees fixed-width operations for every UTF-8 length.
*/
switch (width) {
case 2: {
/*
* 110xxxxx 10xxxxxx
*
* C2..DF in the lead table already guarantees:
*
* - exactly a legal two-byte leader
* - no C0/C1 overlong encoding
*
* b1 has already been checked against 80..BF.
*/
const Rune rune =
((Rune)(b0 & 0x1F) << 6) |
(Rune)(b1 & 0x3F);
return (RuneDecode) {
.rune = rune,
.width = 2,
.valid = true,
};
}
case 3: {
/*
* 1110xxxx 10xxxxxx 10xxxxxx
*
* b1 has already been validated using the descriptor-specific
* range. Only b2 still needs a continuation-byte check.
*/
const uint8_t b2 = str.str[offset + 2];
if ((b2 & 0xC0) != 0x80)
goto invalid;
const Rune rune =
((Rune)(b0 & 0x0F) << 12) |
((Rune)(b1 & 0x3F) << 6) |
(Rune)(b2 & 0x3F);
return (RuneDecode) {
.rune = rune,
.width = 3,
.valid = true,
};
}
case 4: {
/*
* 11110xxx 10xxxxxx 10xxxxxx 10xxxxxx
*
* b1 has already been validated using the descriptor-specific
* range. b2 and b3 use the ordinary continuation-byte form.
*/
const uint8_t b2 = str.str[offset + 2];
const uint8_t b3 = str.str[offset + 3];
/*
* Validate both remaining continuation bytes with one expression.
*
* For a valid continuation byte:
*
* byte & 0xC0 == 0x80
*
* XOR with 0x80 therefore gives zero in the high two bits.
* ORing the two transformed bytes lets us test both at once.
*/
if ((((b2 ^ 0x80) | (b3 ^ 0x80)) & 0xC0) != 0)
goto invalid;
const Rune rune =
((Rune)(b0 & 0x07) << 18) |
((Rune)(b1 & 0x3F) << 12) |
((Rune)(b2 & 0x3F) << 6) |
(Rune)(b3 & 0x3F);
return (RuneDecode) {
.rune = rune,
.width = 4,
.valid = true,
};
}
default:
/*
* The descriptor table can only produce widths 0..4.
* Width 1 was handled by the ASCII fast path.
*/
goto invalid;
}
invalid:
return (RuneDecode) {
.rune = RUNE_ERROR,
.width = 1,
.valid = false,
};
}