Basic UTF-8 functions

src/main.c

@@ -451,10 +451,13 @@ void string_test_misc(void)
     bool prefix1 = str_starts_with(str1, S("He"));
     bool prefix2 = str_starts_with(str1, S(""));
     bool prefix3 = str_starts_with(str1, S("false"));
+    bool prefix4 = str_starts_with(str1, S("Hello World"));
+
     bool suffix1 = str_ends_with(str1, S("orld"));
     bool suffix2 = str_ends_with(str1, S(""));
     bool suffix3 = str_ends_with(str1, S("false"));
-    printf("matches: %d %d %d %d %d %d\n", prefix1, prefix2, prefix3, suffix1, suffix2, suffix3);
+    bool suffix4 = str_ends_with(str1, S("Hello World"));
+    printf("matches: %d %d %d %d %d %d %d %d\n", prefix1, prefix2, prefix3, prefix4, suffix1, suffix2, suffix3, suffix4);
 
     Str str2 = S(" Hello World\t \n");
     Str str2_trim = str_ascii_trim(str2);
@@ -465,7 +468,6 @@ void string_test_misc(void)
     Str str3_trim = str_ascii_trim(str3);
     printf("str3_trim: '%.*s'\n", str3_trim.len, str3_trim.str);
 
-
     Str str4 = S("   \t\t    ");
     Str str4_trim = str_ascii_trim(str4);
     printf("str4_trim: '%.*s'\n", str4_trim.len, str4_trim.str);
@@ -499,6 +501,34 @@ void string_test_join(void)
 
 }
 
+void print_runes(Str str)
+{
+    U32 offset = 0;
+    U32 len = str_rune_count(str);
+    printf("rune len %d\n", len);
+    while (offset < str.len) {
+        RuneDecode d = str_utf8_decode(str, offset);
+        printf("Rune: %d  valid: %d\n", d.rune, d.valid);
+        offset += d.width;
+    }
+}
+
+void string_test_utf8(void)
+{
+    Str str1 = S("Test 1ø2 ☺");
+
+    TRACEF("str1:\n");
+    print_runes(str1);
+
+    Str str2 = str_empty();
+    TRACEF("str2:\n");
+    print_runes(str2);
+
+    Str str3 = S("1Å 2\xc6" "3\xc3");
+    TRACEF("str3:\n");
+    print_runes(str3);
+}
+
 int main(int argc, char *argv[])
 {
     arena_test_1();
@@ -515,6 +545,7 @@ int main(int argc, char *argv[])
     string_split_test_1();
     string_test_misc();
     string_test_join();
+    string_test_utf8();
 
 }
 

src/string.c

@@ -37,12 +37,6 @@ Str str_push(Allocator *alloc, Str str)
     return str_create(mem, str.len);
 }
 
-static INLINE bool char_is_ascii_whitespace(U8 byte)
-{
-    return byte == ' ' || byte == '\t' || byte == '\v' || byte == '\f' || byte == '\r' || byte == '\n';
-
-}
-
 Str str_ascii_trim_left(Str str)
 {
     U8 *start = str.str;
@@ -129,7 +123,6 @@ void str_split(Str str, Str separators, StrVec *vec)
 
 Str str_join(Allocator *allocator, StrVec strings, Str separator)
 {
-
     U32 joined_str_len = 0;
 
     for (U32 i = 0; i < strings.len; i++) {
@@ -163,3 +156,226 @@ Str str_join(Allocator *allocator, StrVec strings, Str separator)
 
     return str_create(joined_str, joined_str_len);
 }
+
+
+/*
+ * Allowed range of the first continuation byte for each range class.
+ *
+ * The normal continuation-byte interval is 0x80..0xBF.
+ */
+static const Utf8Range utf8_ranges[] = {
+    [UTF8_RANGE_NONE]  = { 0x00, 0x00 },
+    [UTF8_RANGE_80_BF] = { 0x80, 0xBF },
+    [UTF8_RANGE_A0_BF] = { 0xA0, 0xBF },
+    [UTF8_RANGE_80_9F] = { 0x80, 0x9F },
+    [UTF8_RANGE_90_BF] = { 0x90, 0xBF },
+    [UTF8_RANGE_80_8F] = { 0x80, 0x8F },
+};
+
+
+/*
+ * Classification of every possible first byte.
+ *
+ * The zero-initialized entries are invalid UTF-8 leading bytes:
+ *
+ *     80..BF  continuation bytes cannot start a sequence
+ *     C0..C1  would produce overlong two-byte encodings
+ *     F5..FF  would encode values above U+10FFFF or are otherwise invalid
+ *
+ * Valid leading-byte ranges:
+ *
+ *     00..7F  ASCII
+ *     C2..DF  2 bytes
+ *     E0..EF  3 bytes
+ *     F0..F4  4 bytes
+ *
+ * E0, ED, F0, and F4 receive special range classes because their second
+ * byte is constrained beyond the normal 80..BF continuation-byte range.
+ *
+ */
+static const U8 utf8_lead[256] = {
+    [0x00 ... 0x7F] = UTF8_DESC(1, UTF8_RANGE_NONE),
+
+    [0xC2 ... 0xDF] = UTF8_DESC(2, UTF8_RANGE_80_BF),
+
+    [0xE0]          = UTF8_DESC(3, UTF8_RANGE_A0_BF),
+
+    [0xE1 ... 0xEC] = UTF8_DESC(3, UTF8_RANGE_80_BF),
+
+    [0xED]          = UTF8_DESC(3, UTF8_RANGE_80_9F),
+
+    [0xEE ... 0xEF] = UTF8_DESC(3, UTF8_RANGE_80_BF),
+
+    [0xF0]          = UTF8_DESC(4, UTF8_RANGE_90_BF),
+
+    [0xF1 ... 0xF3] = UTF8_DESC(4, UTF8_RANGE_80_BF),
+
+    [0xF4]          = UTF8_DESC(4, UTF8_RANGE_80_8F),
+};
+
+/*
+ * Decode one Unicode scalar value from UTF-8.
+ *
+ * Valid input:
+ *
+ *     returns the decoded Rune
+ *     valid = true
+ *     width = number of UTF-8 bytes consumed
+ *
+ * Invalid input:
+ *
+ *     returns U+FFFD
+ *     valid = false
+ *     width = 1
+ *
+ * Consuming one byte on malformed input guarantees forward progress and
+ * allows decoding to resume immediately after the offending byte.
+ *
+ * End of input:
+ *
+ *     returns RUNE_EOF
+ *     valid = false
+ *     width = 0
+ */
+RuneDecode _str_utf8_decode_full(Str str, U32 offset)
+{
+    const uint8_t b0 = str.str[offset];
+    const uint8_t desc = utf8_lead[b0];
+    const uint8_t width = UTF8_DESC_LENGTH(desc);
+    /*
+     * width == 0 covers:
+     *
+     *     80..BF
+     *     C0..C1
+     *     F5..FF
+     */
+    if (width == 0)
+        goto invalid;
+
+    /*
+     * We must have the complete candidate sequence before accessing any
+     * continuation bytes below.
+     */
+    if (str.len - offset < width)
+        goto invalid;
+
+    /*
+     * Validate the first continuation byte.
+     *
+     * Its descriptor-selected range handles both the normal continuation
+     * range and the E0 / ED / F0 / F4 boundary cases.
+     */
+    const Utf8Range range = utf8_ranges[UTF8_DESC_RANGE(desc)];
+    const uint8_t b1 = str.str[offset + 1];
+
+    if (b1 < range.lo || b1 > range.hi)
+        goto invalid;
+
+    /*
+     * Decode paths are intentionally unrolled.
+     *
+     * Besides avoiding a loop in this hot function, each case combines
+     * continuation-byte validation with scalar construction. The compiler
+     * therefore sees fixed-width operations for every UTF-8 length.
+     */
+    switch (width) {
+    case 2: {
+        /*
+         * 110xxxxx 10xxxxxx
+         *
+         * C2..DF in the lead table already guarantees:
+         *
+         *     - exactly a legal two-byte leader
+         *     - no C0/C1 overlong encoding
+         *
+         * b1 has already been checked against 80..BF.
+         */
+        const Rune rune =
+            ((Rune)(b0 & 0x1F) << 6) |
+             (Rune)(b1 & 0x3F);
+
+        return (RuneDecode) {
+            .rune  = rune,
+            .width = 2,
+            .valid = true,
+        };
+    }
+
+    case 3: {
+        /*
+         * 1110xxxx 10xxxxxx 10xxxxxx
+         *
+         * b1 has already been validated using the descriptor-specific
+         * range. Only b2 still needs a continuation-byte check.
+         */
+        const uint8_t b2 = str.str[offset + 2];
+
+        if ((b2 & 0xC0) != 0x80)
+            goto invalid;
+
+        const Rune rune =
+            ((Rune)(b0 & 0x0F) << 12) |
+            ((Rune)(b1 & 0x3F) <<  6) |
+             (Rune)(b2 & 0x3F);
+
+        return (RuneDecode) {
+            .rune  = rune,
+            .width = 3,
+            .valid = true,
+        };
+    }
+
+    case 4: {
+        /*
+         * 11110xxx 10xxxxxx 10xxxxxx 10xxxxxx
+         *
+         * b1 has already been validated using the descriptor-specific
+         * range. b2 and b3 use the ordinary continuation-byte form.
+         */
+        const uint8_t b2 = str.str[offset + 2];
+        const uint8_t b3 = str.str[offset + 3];
+
+        /*
+         * Validate both remaining continuation bytes with one expression.
+         *
+         * For a valid continuation byte:
+         *
+         *     byte & 0xC0 == 0x80
+         *
+         * XOR with 0x80 therefore gives zero in the high two bits.
+         * ORing the two transformed bytes lets us test both at once.
+         */
+        if ((((b2 ^ 0x80) | (b3 ^ 0x80)) & 0xC0) != 0)
+            goto invalid;
+
+        const Rune rune =
+            ((Rune)(b0 & 0x07) << 18) |
+            ((Rune)(b1 & 0x3F) << 12) |
+            ((Rune)(b2 & 0x3F) <<  6) |
+             (Rune)(b3 & 0x3F);
+
+        return (RuneDecode) {
+            .rune  = rune,
+            .width = 4,
+            .valid = true,
+        };
+    }
+
+    default:
+        /*
+         * The descriptor table can only produce widths 0..4.
+         * Width 1 was handled by the ASCII fast path.
+         */
+        goto invalid;
+    }
+
+
+invalid:
+    return (RuneDecode) {
+        .rune  = RUNE_ERROR,
+        .width = 1,
+        .valid = false,
+    };
+}
+
+

src/string.h

@@ -111,3 +111,255 @@ struct StrVec {
 
 void str_split(Str str, Str separators, StrVec *vec);
 Str str_join(Allocator *allocator, StrVec strings, Str separator);
+
+
+
+static INLINE bool char_is_ascii_whitespace(U8 byte)
+{
+    return byte == ' ' || byte == '\t' || byte == '\v' || byte == '\f' || byte == '\r' || byte == '\n';
+
+}
+
+static INLINE bool char_is_digit(U8 byte)
+{
+    return byte >= '0' && byte <= '9';
+}
+
+
+
+// UTF-8/Unicode
+
+typedef S32 Rune;
+
+
+/*
+ * Unicode replacement character.
+ *
+ * Returned for malformed UTF-8
+ */
+#define RUNE_ERROR ((Rune)0xFFFD)
+
+#define RUNE_EOF ((Rune)-1)
+#define RUNE_INVALID (-1)
+
+// Maximum valid Unicode code point.
+#define RUNE_MAX  0x10FFFF
+#define RUNE_SURROGATE_MIN 0xD800
+#define RUNE_SURROGATE_MAX 0xDFFF
+#define RUNE_1_MAX ((1 <<  7)- 1)
+#define RUNE_2_MAX ((1 << 11)- 1)
+#define RUNE_3_MAX ((1 << 16)- 1)
+
+typedef struct {
+    Rune rune;
+
+    /*
+     * Number of input bytes consumed.
+     *
+     *   valid UTF-8 : 1..4
+     *   invalid     : 1
+     *   end of input: 0
+     */
+    U8 width;
+
+    /*
+     * True only when rune was decoded from valid UTF-8.
+     *
+     * This is required because U+FFFD itself may legitimately occur in
+     * valid input, so rune == RUNE_ERROR does not imply an error.
+     */
+    bool valid;
+} RuneDecode;
+
+
+/*
+ * Each entry in utf8_lead[] is an 8-bit descriptor:
+ *
+ *     7 6 5 4 3 2 1 0
+ *    +---------+-----+
+ *    |  range  | len |
+ *    +---------+-----+
+ *
+ * len:
+ *     Number of bytes in the UTF-8 sequence.
+ *
+ *     0 = byte cannot start a valid UTF-8 sequence
+ *     1 = ASCII
+ *     2 = two-byte UTF-8 sequence
+ *     3 = three-byte UTF-8 sequence
+ *     4 = four-byte UTF-8 sequence
+ *
+ * range:
+ *     Selects the legal range for byte 1, the first continuation byte.
+ *
+ * This range is enough to encode all of UTF-8's special boundary rules:
+ *
+ *     E0 A0..BF    prevents overlong 3-byte encodings
+ *     ED 80..9F    excludes UTF-16 surrogate code points
+ *     F0 90..BF    prevents overlong 4-byte encodings
+ *     F4 80..8F    limits Unicode to U+10FFFF
+ *
+ * All continuation bytes after byte 1 simply use the normal 80..BF range.
+ */
+
+#define UTF8_LENGTH_BITS 3u
+#define UTF8_LENGTH_MASK ((1u << UTF8_LENGTH_BITS) - 1u)
+
+#define UTF8_DESC(length, range) \
+    ((U8)((length) | ((U8)(range) << UTF8_LENGTH_BITS)))
+
+#define UTF8_DESC_LENGTH(desc) \
+    ((U8)((desc) & UTF8_LENGTH_MASK))
+
+#define UTF8_DESC_RANGE(desc) \
+    ((U8)((desc) >> UTF8_LENGTH_BITS))
+
+
+/*
+ * Classes of allowed values for the first continuation byte.
+ */
+enum Utf8RangeClass {
+    /*
+     * No continuation byte exists.
+     *
+     * Used only by ASCII descriptors.
+     */
+    UTF8_RANGE_NONE = 0,
+
+    /*
+     * Normal continuation-byte range.
+     */
+    UTF8_RANGE_80_BF,
+
+    /*
+     * E0 xx xx:
+     *
+     * E0 80..9F would encode values below U+0800 and therefore be
+     * non-canonical overlong UTF-8.
+     */
+    UTF8_RANGE_A0_BF,
+
+    /*
+     * ED xx xx:
+     *
+     * ED A0..BF would encode U+D800..U+DFFF, which are UTF-16 surrogate
+     * code points and are not Unicode scalar values.
+     */
+    UTF8_RANGE_80_9F,
+
+    /*
+     * F0 xx xx xx:
+     *
+     * F0 80..8F would encode values below U+10000 and therefore be
+     * non-canonical overlong UTF-8.
+     */
+    UTF8_RANGE_90_BF,
+
+    /*
+     * F4 xx xx xx:
+     *
+     * F4 90..BF would encode values above Unicode's maximum scalar value,
+     * U+10FFFF.
+     */
+    UTF8_RANGE_80_8F,
+};
+
+
+typedef struct {
+    U8 lo;
+    U8 hi;
+} Utf8Range;
+
+// Helper:
+RuneDecode _str_utf8_decode_full(Str str, U32 offset);
+/*
+ * Decode one Unicode scalar value from UTF-8.
+ *
+ * Valid input:
+ *
+ *     returns the decoded Rune
+ *     valid = true
+ *     width = number of UTF-8 bytes consumed
+ *
+ * Invalid input:
+ *
+ *     returns U+FFFD
+ *     valid = false
+ *     width = 1
+ *
+ * Consuming one byte on malformed input guarantees forward progress and
+ * allows decoding to resume immediately after the offending byte.
+ *
+ * End of input:
+ *
+ *     returns RUNE_EOF
+ *     valid = false
+ *     width = 0
+ */
+static INLINE RuneDecode str_utf8_decode(Str str, U32 offset)
+{
+    if (str.len == 0 || str.len <= offset) {
+        return (RuneDecode) {
+            .rune  = RUNE_EOF,
+            .width = 0,
+            .valid = false,
+        };
+    }
+
+    const uint8_t b0 = str.str[offset];
+
+
+    /*
+     * Handle ascii, helps inlining for ascii cases
+     *
+     * Testing b0 directly also avoids consulting utf8_ranges[] for the
+     * overwhelmingly common single-byte case.
+     */
+    if (b0 < 0x80) {
+        return (RuneDecode) {
+            .rune  = (Rune)b0,
+            .width = 1,
+            .valid = true,
+        };
+    }
+
+    return _str_utf8_decode_full(str, offset);
+}
+
+// Length of bytes to encode the Rune in utf-8.
+// returns -1 if the Rune is not a valid value to encode in UTF-8.
+static INLINE S32 rune_utf8_len(Rune rune) {
+
+	if (rune < 0) {
+		return RUNE_INVALID;
+    } else if (rune <= RUNE_1_MAX) {
+		return 1;
+	} else if (rune <= RUNE_2_MAX) {
+		return 2;
+	} else if (rune > RUNE_SURROGATE_MIN && rune <= RUNE_SURROGATE_MAX) {
+		return RUNE_INVALID;
+	} else if (rune <= RUNE_3_MAX) {
+		return 3;
+	} else if (rune <= RUNE_MAX) {
+		return 4;
+	}
+
+	return RUNE_INVALID;
+}
+
+// Number of unicode Runes in str. Invalid UTF-8 is counted as 1 Rune.
+// (1 invalud UTF-8 byte will be decoded to 1 Rune of RUNE_ERROR
+static INLINE U32 str_rune_count(Str str)
+{
+    U32 offset = 0;
+    U32 runes = 0;
+
+    RuneDecode d = str_utf8_decode(str, offset);
+    while (d.rune != RUNE_EOF) {
+        offset += d.width;
+        runes++;
+        d = str_utf8_decode(str, offset);
+    }
+
+    return runes;
+}