utf8 fixes

src/main.c

@@ -527,6 +527,12 @@ void string_test_utf8(void)
     Str str3 = S("1Å 2\xc6" "3\xc3");
     TRACEF("str3:\n");
     print_runes(str3);
+
+    const char rune_error[] = {0xEF, 0xBF, 0xBD, 0x00} ; // RUNE_ERROR encoded in utf-8
+    Str str4 = S(rune_error);
+    TRACEF("str4:\n");
+    print_runes(str4);  // this should be a valid code point
+
 }
 
 int main(int argc, char *argv[])

src/string.c

@@ -239,9 +239,16 @@ static const U8 utf8_lead[256] = {
  */
 RuneDecode _str_utf8_decode_full(Str str, U32 offset)
 {
-    const uint8_t b0 = str.str[offset];
-    const uint8_t desc = utf8_lead[b0];
-    const uint8_t width = UTF8_DESC_LENGTH(desc);
+    if (str.len == 0 || str.len <= offset) {
+        return (RuneDecode) {
+            .rune  = RUNE_EOF,
+            .width = 0,
+            .valid = false,
+        };
+    }
+    U8 b0 = str.str[offset];
+    U8 desc = utf8_lead[b0];
+    U8 width = UTF8_DESC_LENGTH(desc);
     /*
      * width == 0 covers:
      *
@@ -265,8 +272,8 @@ RuneDecode _str_utf8_decode_full(Str str, U32 offset)
      * Its descriptor-selected range handles both the normal continuation
      * range and the E0 / ED / F0 / F4 boundary cases.
      */
-    const Utf8Range range = utf8_ranges[UTF8_DESC_RANGE(desc)];
-    const uint8_t b1 = str.str[offset + 1];
+    Utf8Range range = utf8_ranges[UTF8_DESC_RANGE(desc)];
+    U8 b1 = str.str[offset + 1];
 
     if (b1 < range.lo || b1 > range.hi)
         goto invalid;
@@ -290,7 +297,7 @@ RuneDecode _str_utf8_decode_full(Str str, U32 offset)
          *
          * b1 has already been checked against 80..BF.
          */
-        const Rune rune =
+        Rune rune =
             ((Rune)(b0 & 0x1F) << 6) |
              (Rune)(b1 & 0x3F);
 
@@ -308,12 +315,12 @@ RuneDecode _str_utf8_decode_full(Str str, U32 offset)
          * b1 has already been validated using the descriptor-specific
          * range. Only b2 still needs a continuation-byte check.
          */
-        const uint8_t b2 = str.str[offset + 2];
+        U8 b2 = str.str[offset + 2];
 
         if ((b2 & 0xC0) != 0x80)
             goto invalid;
 
-        const Rune rune =
+        Rune rune =
             ((Rune)(b0 & 0x0F) << 12) |
             ((Rune)(b1 & 0x3F) <<  6) |
              (Rune)(b2 & 0x3F);
@@ -332,8 +339,8 @@ RuneDecode _str_utf8_decode_full(Str str, U32 offset)
          * b1 has already been validated using the descriptor-specific
          * range. b2 and b3 use the ordinary continuation-byte form.
          */
-        const uint8_t b2 = str.str[offset + 2];
-        const uint8_t b3 = str.str[offset + 3];
+        U8 b2 = str.str[offset + 2];
+        U8 b3 = str.str[offset + 3];
 
         /*
          * Validate both remaining continuation bytes with one expression.
@@ -348,7 +355,7 @@ RuneDecode _str_utf8_decode_full(Str str, U32 offset)
         if ((((b2 ^ 0x80) | (b3 ^ 0x80)) & 0xC0) != 0)
             goto invalid;
 
-        const Rune rune =
+        Rune rune =
             ((Rune)(b0 & 0x07) << 18) |
             ((Rune)(b1 & 0x3F) << 12) |
             ((Rune)(b2 & 0x3F) <<  6) |

src/string.h

@@ -306,16 +306,13 @@ static INLINE RuneDecode str_utf8_decode(Str str, U32 offset)
         };
     }
 
-    const uint8_t b0 = str.str[offset];
-
+    U8 b0 = str.str[offset];
 
     /*
-     * Handle ascii, helps inlining for ascii cases
-     *
      * Testing b0 directly also avoids consulting utf8_ranges[] for the
-     * overwhelmingly common single-byte case.
+     * overwhelmingly common single-byte/ascii case.
      */
-    if (b0 < 0x80) {
+    if (b0 <= RUNE_1_MAX) {
         return (RuneDecode) {
             .rune  = (Rune)b0,
             .width = 1,
@@ -336,7 +333,7 @@ static INLINE S32 rune_utf8_len(Rune rune) {
 		return 1;
 	} else if (rune <= RUNE_2_MAX) {
 		return 2;
-	} else if (rune > RUNE_SURROGATE_MIN && rune <= RUNE_SURROGATE_MAX) {
+	} else if (rune >= RUNE_SURROGATE_MIN && rune <= RUNE_SURROGATE_MAX) {
 		return RUNE_INVALID;
 	} else if (rune <= RUNE_3_MAX) {
 		return 3;
@@ -347,8 +344,7 @@ static INLINE S32 rune_utf8_len(Rune rune) {
 	return RUNE_INVALID;
 }
 
-// Number of unicode Runes in str. Invalid UTF-8 is counted as 1 Rune.
-// (1 invalud UTF-8 byte will be decoded to 1 Rune of RUNE_ERROR
+// Number of unicode Runes in str. 1 invalid UTF-8 byte is counted as 1 Rune.
 static INLINE U32 str_rune_count(Str str)
 {
     U32 offset = 0;