Move utf stuff from header file

src/string.c

@@ -224,6 +224,105 @@ Str str_join(Allocator *allocator, StrVec strings, Str separator)
 }
 
 
+
+/*
+ * Each entry in utf8_lead[] is an 8-bit descriptor:
+ *
+ *     7 6 5 4 3 2 1 0
+ *    +---------+-----+
+ *    |  range  | len |
+ *    +---------+-----+
+ *
+ * len:
+ *     Number of bytes in the UTF-8 sequence.
+ *
+ *     0 = byte cannot start a valid UTF-8 sequence
+ *     1 = ASCII
+ *     2 = two-byte UTF-8 sequence
+ *     3 = three-byte UTF-8 sequence
+ *     4 = four-byte UTF-8 sequence
+ *
+ * range:
+ *     Selects the legal range for byte 1, the first continuation byte.
+ *
+ * This range is enough to encode all of UTF-8's special boundary rules:
+ *
+ *     E0 A0..BF    prevents overlong 3-byte encodings
+ *     ED 80..9F    excludes UTF-16 surrogate code points
+ *     F0 90..BF    prevents overlong 4-byte encodings
+ *     F4 80..8F    limits Unicode to U+10FFFF
+ *
+ * All continuation bytes after byte 1 simply use the normal 80..BF range.
+ */
+
+#define UTF8_LENGTH_BITS 3u
+#define UTF8_LENGTH_MASK ((1u << UTF8_LENGTH_BITS) - 1u)
+
+#define UTF8_DESC(length, range) \
+    ((U8)((length) | ((U8)(range) << UTF8_LENGTH_BITS)))
+
+#define UTF8_DESC_LENGTH(desc) \
+    ((U8)((desc) & UTF8_LENGTH_MASK))
+
+#define UTF8_DESC_RANGE(desc) \
+    ((U8)((desc) >> UTF8_LENGTH_BITS))
+
+
+/*
+ * Classes of allowed values for the first continuation byte.
+ */
+enum Utf8RangeClass {
+    /*
+     * No continuation byte exists.
+     *
+     * Used only by ASCII descriptors.
+     */
+    UTF8_RANGE_NONE = 0,
+
+    /*
+     * Normal continuation-byte range.
+     */
+    UTF8_RANGE_80_BF,
+
+    /*
+     * E0 xx xx:
+     *
+     * E0 80..9F would encode values below U+0800 and therefore be
+     * non-canonical overlong UTF-8.
+     */
+    UTF8_RANGE_A0_BF,
+
+    /*
+     * ED xx xx:
+     *
+     * ED A0..BF would encode U+D800..U+DFFF, which are UTF-16 surrogate
+     * code points and are not Unicode scalar values.
+     */
+    UTF8_RANGE_80_9F,
+
+    /*
+     * F0 xx xx xx:
+     *
+     * F0 80..8F would encode values below U+10000 and therefore be
+     * non-canonical overlong UTF-8.
+     */
+    UTF8_RANGE_90_BF,
+
+    /*
+     * F4 xx xx xx:
+     *
+     * F4 90..BF would encode values above Unicode's maximum scalar value,
+     * U+10FFFF.
+     */
+    UTF8_RANGE_80_8F,
+};
+
+
+typedef struct {
+    U8 lo;
+    U8 hi;
+} Utf8Range;
+
 /*
  * Allowed range of the first continuation byte for each range class.
  *
@@ -239,6 +338,36 @@ static const Utf8Range utf8_ranges[] = {
 };
 
 
+
+/*
+ * Each entry in utf8_lead[] is an 8-bit descriptor:
+ *
+ *     7 6 5 4 3 2 1 0
+ *    +---------+-----+
+ *    |  range  | len |
+ *    +---------+-----+
+ *
+ * len:
+ *     Number of bytes in the UTF-8 sequence.
+ *
+ *     0 = byte cannot start a valid UTF-8 sequence
+ *     1 = ASCII
+ *     2 = two-byte UTF-8 sequence
+ *     3 = three-byte UTF-8 sequence
+ *     4 = four-byte UTF-8 sequence
+ *
+ * range:
+ *     Selects the legal range for byte 1, the first continuation byte.
+ *
+ * This range is enough to encode all of UTF-8's special boundary rules:
+ *
+ *     E0 A0..BF    prevents overlong 3-byte encodings
+ *     ED 80..9F    excludes UTF-16 surrogate code points
+ *     F0 90..BF    prevents overlong 4-byte encodings
+ *     F4 80..8F    limits Unicode to U+10FFFF
+ *
+ * All continuation bytes after byte 1 simply use the normal 80..BF range.
+ */
 /*
  * Classification of every possible first byte.
  *

src/string.h

@@ -198,111 +198,12 @@ typedef struct {
     /*
      * True only when rune was decoded from valid UTF-8.
      *
-     * This is required because U+FFFD itself may legitimately occur in
+     *  U+FFFD itself may legitimately occur in  valid
-     * valid input, so rune == RUNE_ERROR does not imply an error.
+     *  input, so rune == RUNE_ERROR does not imply an error.
      */
     bool valid;
 } RuneDecode;
 
-
-/*
- * Each entry in utf8_lead[] is an 8-bit descriptor:
- *
- *     7 6 5 4 3 2 1 0
- *    +---------+-----+
- *    |  range  | len |
- *    +---------+-----+
- *
- * len:
- *     Number of bytes in the UTF-8 sequence.
- *
- *     0 = byte cannot start a valid UTF-8 sequence
- *     1 = ASCII
- *     2 = two-byte UTF-8 sequence
- *     3 = three-byte UTF-8 sequence
- *     4 = four-byte UTF-8 sequence
- *
- * range:
- *     Selects the legal range for byte 1, the first continuation byte.
- *
- * This range is enough to encode all of UTF-8's special boundary rules:
- *
- *     E0 A0..BF    prevents overlong 3-byte encodings
- *     ED 80..9F    excludes UTF-16 surrogate code points
- *     F0 90..BF    prevents overlong 4-byte encodings
- *     F4 80..8F    limits Unicode to U+10FFFF
- *
- * All continuation bytes after byte 1 simply use the normal 80..BF range.
- */
-
-#define UTF8_LENGTH_BITS 3u
-#define UTF8_LENGTH_MASK ((1u << UTF8_LENGTH_BITS) - 1u)
-
-#define UTF8_DESC(length, range) \
-    ((U8)((length) | ((U8)(range) << UTF8_LENGTH_BITS)))
-
-#define UTF8_DESC_LENGTH(desc) \
-    ((U8)((desc) & UTF8_LENGTH_MASK))
-
-#define UTF8_DESC_RANGE(desc) \
-    ((U8)((desc) >> UTF8_LENGTH_BITS))
-
-
-/*
- * Classes of allowed values for the first continuation byte.
- */
-enum Utf8RangeClass {
-    /*
-     * No continuation byte exists.
-     *
-     * Used only by ASCII descriptors.
-     */
-    UTF8_RANGE_NONE = 0,
-
-    /*
-     * Normal continuation-byte range.
-     */
-    UTF8_RANGE_80_BF,
-
-    /*
-     * E0 xx xx:
-     *
-     * E0 80..9F would encode values below U+0800 and therefore be
-     * non-canonical overlong UTF-8.
-     */
-    UTF8_RANGE_A0_BF,
-
-    /*
-     * ED xx xx:
-     *
-     * ED A0..BF would encode U+D800..U+DFFF, which are UTF-16 surrogate
-     * code points and are not Unicode scalar values.
-     */
-    UTF8_RANGE_80_9F,
-
-    /*
-     * F0 xx xx xx:
-     *
-     * F0 80..8F would encode values below U+10000 and therefore be
-     * non-canonical overlong UTF-8.
-     */
-    UTF8_RANGE_90_BF,
-
-    /*
-     * F4 xx xx xx:
-     *
-     * F4 90..BF would encode values above Unicode's maximum scalar value,
-     * U+10FFFF.
-     */
-    UTF8_RANGE_80_8F,
-};
-
-
-typedef struct {
-    U8 lo;
-    U8 hi;
-} Utf8Range;
-
 // Helper:
 RuneDecode _str_utf8_decode_full(Str str, U32 offset);
 /*