From 02638b20b91f8b1657b410f0e589db306b396a5a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Nils=20O=2E=20Sel=C3=A5sdal?= Date: Sun, 16 Aug 2026 20:19:24 +0200 Subject: [PATCH] Move utf stuff from header file --- src/string.c | 129 +++++++++++++++++++++++++++++++++++++++++++++++++++ src/string.h | 103 +--------------------------------------- 2 files changed, 131 insertions(+), 101 deletions(-) diff --git a/src/string.c b/src/string.c index 084a256..98a2499 100644 --- a/src/string.c +++ b/src/string.c @@ -224,6 +224,105 @@ Str str_join(Allocator *allocator, StrVec strings, Str separator) } + +/* + * Each entry in utf8_lead[] is an 8-bit descriptor: + * + * 7 6 5 4 3 2 1 0 + * +---------+-----+ + * | range | len | + * +---------+-----+ + * + * len: + * Number of bytes in the UTF-8 sequence. + * + * 0 = byte cannot start a valid UTF-8 sequence + * 1 = ASCII + * 2 = two-byte UTF-8 sequence + * 3 = three-byte UTF-8 sequence + * 4 = four-byte UTF-8 sequence + * + * range: + * Selects the legal range for byte 1, the first continuation byte. + * + * This range is enough to encode all of UTF-8's special boundary rules: + * + * E0 A0..BF prevents overlong 3-byte encodings + * ED 80..9F excludes UTF-16 surrogate code points + * F0 90..BF prevents overlong 4-byte encodings + * F4 80..8F limits Unicode to U+10FFFF + * + * All continuation bytes after byte 1 simply use the normal 80..BF range. + */ + +#define UTF8_LENGTH_BITS 3u +#define UTF8_LENGTH_MASK ((1u << UTF8_LENGTH_BITS) - 1u) + +#define UTF8_DESC(length, range) \ + ((U8)((length) | ((U8)(range) << UTF8_LENGTH_BITS))) + +#define UTF8_DESC_LENGTH(desc) \ + ((U8)((desc) & UTF8_LENGTH_MASK)) + +#define UTF8_DESC_RANGE(desc) \ + ((U8)((desc) >> UTF8_LENGTH_BITS)) + + +/* + * Classes of allowed values for the first continuation byte. + */ +enum Utf8RangeClass { + /* + * No continuation byte exists. + * + * Used only by ASCII descriptors. + */ + UTF8_RANGE_NONE = 0, + + /* + * Normal continuation-byte range. + */ + UTF8_RANGE_80_BF, + + /* + * E0 xx xx: + * + * E0 80..9F would encode values below U+0800 and therefore be + * non-canonical overlong UTF-8. + */ + UTF8_RANGE_A0_BF, + + /* + * ED xx xx: + * + * ED A0..BF would encode U+D800..U+DFFF, which are UTF-16 surrogate + * code points and are not Unicode scalar values. + */ + UTF8_RANGE_80_9F, + + /* + * F0 xx xx xx: + * + * F0 80..8F would encode values below U+10000 and therefore be + * non-canonical overlong UTF-8. + */ + UTF8_RANGE_90_BF, + + /* + * F4 xx xx xx: + * + * F4 90..BF would encode values above Unicode's maximum scalar value, + * U+10FFFF. + */ + UTF8_RANGE_80_8F, +}; + + +typedef struct { + U8 lo; + U8 hi; +} Utf8Range; + /* * Allowed range of the first continuation byte for each range class. * @@ -239,6 +338,36 @@ static const Utf8Range utf8_ranges[] = { }; + +/* + * Each entry in utf8_lead[] is an 8-bit descriptor: + * + * 7 6 5 4 3 2 1 0 + * +---------+-----+ + * | range | len | + * +---------+-----+ + * + * len: + * Number of bytes in the UTF-8 sequence. + * + * 0 = byte cannot start a valid UTF-8 sequence + * 1 = ASCII + * 2 = two-byte UTF-8 sequence + * 3 = three-byte UTF-8 sequence + * 4 = four-byte UTF-8 sequence + * + * range: + * Selects the legal range for byte 1, the first continuation byte. + * + * This range is enough to encode all of UTF-8's special boundary rules: + * + * E0 A0..BF prevents overlong 3-byte encodings + * ED 80..9F excludes UTF-16 surrogate code points + * F0 90..BF prevents overlong 4-byte encodings + * F4 80..8F limits Unicode to U+10FFFF + * + * All continuation bytes after byte 1 simply use the normal 80..BF range. + */ /* * Classification of every possible first byte. * diff --git a/src/string.h b/src/string.h index 6f44746..5dcc72c 100644 --- a/src/string.h +++ b/src/string.h @@ -198,111 +198,12 @@ typedef struct { /* * True only when rune was decoded from valid UTF-8. * - * This is required because U+FFFD itself may legitimately occur in - * valid input, so rune == RUNE_ERROR does not imply an error. + * U+FFFD itself may legitimately occur in valid + * input, so rune == RUNE_ERROR does not imply an error. */ bool valid; } RuneDecode; - -/* - * Each entry in utf8_lead[] is an 8-bit descriptor: - * - * 7 6 5 4 3 2 1 0 - * +---------+-----+ - * | range | len | - * +---------+-----+ - * - * len: - * Number of bytes in the UTF-8 sequence. - * - * 0 = byte cannot start a valid UTF-8 sequence - * 1 = ASCII - * 2 = two-byte UTF-8 sequence - * 3 = three-byte UTF-8 sequence - * 4 = four-byte UTF-8 sequence - * - * range: - * Selects the legal range for byte 1, the first continuation byte. - * - * This range is enough to encode all of UTF-8's special boundary rules: - * - * E0 A0..BF prevents overlong 3-byte encodings - * ED 80..9F excludes UTF-16 surrogate code points - * F0 90..BF prevents overlong 4-byte encodings - * F4 80..8F limits Unicode to U+10FFFF - * - * All continuation bytes after byte 1 simply use the normal 80..BF range. - */ - -#define UTF8_LENGTH_BITS 3u -#define UTF8_LENGTH_MASK ((1u << UTF8_LENGTH_BITS) - 1u) - -#define UTF8_DESC(length, range) \ - ((U8)((length) | ((U8)(range) << UTF8_LENGTH_BITS))) - -#define UTF8_DESC_LENGTH(desc) \ - ((U8)((desc) & UTF8_LENGTH_MASK)) - -#define UTF8_DESC_RANGE(desc) \ - ((U8)((desc) >> UTF8_LENGTH_BITS)) - - -/* - * Classes of allowed values for the first continuation byte. - */ -enum Utf8RangeClass { - /* - * No continuation byte exists. - * - * Used only by ASCII descriptors. - */ - UTF8_RANGE_NONE = 0, - - /* - * Normal continuation-byte range. - */ - UTF8_RANGE_80_BF, - - /* - * E0 xx xx: - * - * E0 80..9F would encode values below U+0800 and therefore be - * non-canonical overlong UTF-8. - */ - UTF8_RANGE_A0_BF, - - /* - * ED xx xx: - * - * ED A0..BF would encode U+D800..U+DFFF, which are UTF-16 surrogate - * code points and are not Unicode scalar values. - */ - UTF8_RANGE_80_9F, - - /* - * F0 xx xx xx: - * - * F0 80..8F would encode values below U+10000 and therefore be - * non-canonical overlong UTF-8. - */ - UTF8_RANGE_90_BF, - - /* - * F4 xx xx xx: - * - * F4 90..BF would encode values above Unicode's maximum scalar value, - * U+10FFFF. - */ - UTF8_RANGE_80_8F, -}; - - -typedef struct { - U8 lo; - U8 hi; -} Utf8Range; - // Helper: RuneDecode _str_utf8_decode_full(Str str, U32 offset); /*