584 lines
14 KiB
C
584 lines
14 KiB
C
#include "string.h"
|
|
|
|
Str str_empty(void)
|
|
{
|
|
static const U8 empty_;
|
|
static const Str str = {(U8 *)&empty_, 0};
|
|
return str;
|
|
}
|
|
|
|
Str str_slice(Str str, U32 start, U32 len)
|
|
{
|
|
Assert(start <= str.len);
|
|
Assert(len <= str.len - start);
|
|
|
|
Str new_str = str_create(&str.str[start], len);
|
|
return new_str;
|
|
}
|
|
|
|
Str str_concat(Allocator *alloc, Str first, Str second)
|
|
{
|
|
Str str;
|
|
Assert(first.len <= U32_Max - second.len);
|
|
str.len = first.len + second.len;
|
|
str.str = allocator_push(alloc, str.len, 1);;
|
|
|
|
MemCopy(str.str, first.str, first.len);
|
|
MemCopy(&str.str[first.len], second.str, second.len);
|
|
|
|
return str;
|
|
}
|
|
|
|
Str str_push(Allocator *alloc, Str str)
|
|
{
|
|
U8 *mem = (U8 *)allocator_push(alloc, str.len, 1);
|
|
MemCopy(mem, str.str, str.len);
|
|
|
|
return str_create(mem, str.len);
|
|
}
|
|
|
|
Str str_ascii_trim_left(Str str)
|
|
{
|
|
U8 *start = str.str;
|
|
U8 *end = str.str + str.len;
|
|
while (start != end) {
|
|
if (!char_is_ascii_whitespace(*start)) {
|
|
break;
|
|
}
|
|
start++;
|
|
}
|
|
|
|
return str_create(start, (U32)(end - start));
|
|
}
|
|
|
|
Str str_ascii_trim_right(Str str)
|
|
{
|
|
U8 *start = str.str;
|
|
U8 *end = str.str + str.len;
|
|
while (start != end) {
|
|
if (!char_is_ascii_whitespace(*(end - 1))) {
|
|
break;
|
|
}
|
|
end--;
|
|
}
|
|
|
|
return str_create(start, (U32)(end - start));
|
|
}
|
|
|
|
Str str_ascii_trim(Str str)
|
|
{
|
|
Str new_str = str_ascii_trim_left(str);
|
|
new_str = str_ascii_trim_right(new_str);
|
|
return new_str;
|
|
}
|
|
|
|
S64 str_index(Str haystack, Str needle)
|
|
{
|
|
if (needle.len == 0) {
|
|
return 0;
|
|
}
|
|
|
|
if (needle.len == 1) {
|
|
return str_find_byte(haystack, needle.str[0]);
|
|
}
|
|
if (needle.len > haystack.len) {
|
|
return -1;
|
|
}
|
|
|
|
U8 *found = MemFind(haystack.str, haystack.len, needle.str, needle.len);
|
|
if (found != NULL) {
|
|
return (S64)(found - haystack.str);
|
|
}
|
|
|
|
return -1;
|
|
}
|
|
|
|
|
|
|
|
static INLINE S16 char_hex_to_binary(U8 c)
|
|
{
|
|
if (c >='0' && c <='9')
|
|
return c - '0';
|
|
else if (c >= 'A' && c <= 'F')
|
|
return c - 'A' + 10;
|
|
else if (c >= 'a' && c <= 'f')
|
|
return c - 'a' + 10;
|
|
|
|
return -1;
|
|
}
|
|
|
|
static const char hex_chars[] = "0123456789ABCDEF";
|
|
|
|
Str str_hex_encode(Str binary_data, Str hex_result)
|
|
{
|
|
Assert(hex_result.len >= binary_data.len * 2);
|
|
U32 hex_len = binary_data.len * 2;
|
|
for (U32 i = 0, t = 0; i < binary_data.len; ++i,t+=2) {
|
|
U8 high_nibble;
|
|
U8 low_nibble;
|
|
|
|
high_nibble = (binary_data.str[i] & 0xF0) >> 4 ;
|
|
low_nibble = binary_data.str[i] & 0x0F ;
|
|
|
|
hex_result.str[t] = hex_chars[high_nibble];
|
|
hex_result.str[t+1] = hex_chars[low_nibble];
|
|
}
|
|
|
|
hex_result.len = hex_len;
|
|
return hex_result;
|
|
}
|
|
|
|
Str str_hex_decode(Str hex_str, Str result)
|
|
{
|
|
Assert(result.len >= hex_str.len/2);
|
|
U32 dest_pos = 0;
|
|
U32 source_pos = 0;
|
|
|
|
while (source_pos + 1 < hex_str.len) {
|
|
|
|
if (char_is_ascii_whitespace(hex_str.str[source_pos])) {
|
|
source_pos++;
|
|
continue;
|
|
}
|
|
|
|
S16 high_nibble;
|
|
S16 low_nibble;
|
|
|
|
high_nibble = char_hex_to_binary(hex_str.str[source_pos]);
|
|
low_nibble = char_hex_to_binary(hex_str.str[source_pos + 1]);
|
|
if (high_nibble == -1 || low_nibble == -1) {
|
|
result.len = 0;
|
|
return result;
|
|
}
|
|
|
|
result.str[dest_pos] = (high_nibble << 4 ) | low_nibble;
|
|
source_pos += 2;
|
|
dest_pos++;
|
|
}
|
|
|
|
result.len = dest_pos;
|
|
return result;
|
|
}
|
|
// split @str on any byte in @separators, place split strings in @vec
|
|
void str_split(Str str, Str separators, StrVec *vec)
|
|
{
|
|
if (vec->max_len == 0 || str.len == 0) {
|
|
return;
|
|
}
|
|
|
|
U32 start_pos = 0;
|
|
for (U32 i = 0; i < str.len; i++) {
|
|
U8 byte = str.str[i];
|
|
if (str_find_byte(separators, byte) != -1) {
|
|
if (vec->len >= vec->max_len) {
|
|
break;
|
|
}
|
|
|
|
Str sub_str = str_slice(str, start_pos, i - start_pos);
|
|
vec->strings[vec->len] = sub_str;
|
|
vec->len++;
|
|
start_pos = i + 1;
|
|
}
|
|
}
|
|
if (start_pos <= str.len && vec->len < vec->max_len) {
|
|
Str sub_str = str_slice(str, start_pos, str.len - start_pos);
|
|
vec->strings[vec->len] = sub_str;
|
|
vec->len++;
|
|
}
|
|
}
|
|
|
|
Str str_join(Allocator *allocator, StrVec strings, Str separator)
|
|
{
|
|
U32 joined_str_len = 0;
|
|
|
|
for (U32 i = 0; i < strings.len; i++) {
|
|
U32 part_len = strings.strings[i].len;
|
|
Assert(joined_str_len <= U32_Max - part_len); // overflow
|
|
joined_str_len += part_len;
|
|
}
|
|
if (strings.len > 1) {
|
|
U32 separator_count = strings.len - 1;
|
|
Assert(separator.len <= U32_Max / separator_count); // overflow
|
|
Assert(joined_str_len <= U32_Max - separator.len * separator_count); // overflow
|
|
joined_str_len += separator.len * separator_count;
|
|
}
|
|
|
|
U8 *joined_str = allocator_push(allocator, joined_str_len, 1);
|
|
if (strings.len > 0) {
|
|
U32 pos = 0;
|
|
Str *str = &strings.strings[0];
|
|
|
|
MemCopy(&joined_str[pos], str->str, str->len);
|
|
pos += str->len;
|
|
|
|
for (U32 i = 1; i < strings.len; i++) {
|
|
str = &strings.strings[i];
|
|
MemCopy(&joined_str[pos], separator.str, separator.len);
|
|
pos += separator.len;
|
|
MemCopy(&joined_str[pos], str->str, str->len);
|
|
pos += str->len;
|
|
}
|
|
}
|
|
|
|
return str_create(joined_str, joined_str_len);
|
|
}
|
|
|
|
|
|
|
|
/*
|
|
* Each entry in utf8_lead[] is an 8-bit descriptor:
|
|
*
|
|
* 7 6 5 4 3 2 1 0
|
|
* +---------+-----+
|
|
* | range | len |
|
|
* +---------+-----+
|
|
*
|
|
* len:
|
|
* Number of bytes in the UTF-8 sequence.
|
|
*
|
|
* 0 = byte cannot start a valid UTF-8 sequence
|
|
* 1 = ASCII
|
|
* 2 = two-byte UTF-8 sequence
|
|
* 3 = three-byte UTF-8 sequence
|
|
* 4 = four-byte UTF-8 sequence
|
|
*
|
|
* range:
|
|
* Selects the legal range for byte 1, the first continuation byte.
|
|
*
|
|
* This range is enough to encode all of UTF-8's special boundary rules:
|
|
*
|
|
* E0 A0..BF prevents overlong 3-byte encodings
|
|
* ED 80..9F excludes UTF-16 surrogate code points
|
|
* F0 90..BF prevents overlong 4-byte encodings
|
|
* F4 80..8F limits Unicode to U+10FFFF
|
|
*
|
|
* All continuation bytes after byte 1 simply use the normal 80..BF range.
|
|
*/
|
|
|
|
#define UTF8_LENGTH_BITS 3u
|
|
#define UTF8_LENGTH_MASK ((1u << UTF8_LENGTH_BITS) - 1u)
|
|
|
|
#define UTF8_DESC(length, range) \
|
|
((U8)((length) | ((U8)(range) << UTF8_LENGTH_BITS)))
|
|
|
|
#define UTF8_DESC_LENGTH(desc) \
|
|
((U8)((desc) & UTF8_LENGTH_MASK))
|
|
|
|
#define UTF8_DESC_RANGE(desc) \
|
|
((U8)((desc) >> UTF8_LENGTH_BITS))
|
|
|
|
|
|
/*
|
|
* Classes of allowed values for the first continuation byte.
|
|
*/
|
|
enum Utf8RangeClass {
|
|
/*
|
|
* No continuation byte exists.
|
|
*
|
|
* Used only by ASCII descriptors.
|
|
*/
|
|
UTF8_RANGE_NONE = 0,
|
|
|
|
/*
|
|
* Normal continuation-byte range.
|
|
*/
|
|
UTF8_RANGE_80_BF,
|
|
|
|
/*
|
|
* E0 xx xx:
|
|
*
|
|
* E0 80..9F would encode values below U+0800 and therefore be
|
|
* non-canonical overlong UTF-8.
|
|
*/
|
|
UTF8_RANGE_A0_BF,
|
|
|
|
/*
|
|
* ED xx xx:
|
|
*
|
|
* ED A0..BF would encode U+D800..U+DFFF, which are UTF-16 surrogate
|
|
* code points and are not Unicode scalar values.
|
|
*/
|
|
UTF8_RANGE_80_9F,
|
|
|
|
/*
|
|
* F0 xx xx xx:
|
|
*
|
|
* F0 80..8F would encode values below U+10000 and therefore be
|
|
* non-canonical overlong UTF-8.
|
|
*/
|
|
UTF8_RANGE_90_BF,
|
|
|
|
/*
|
|
* F4 xx xx xx:
|
|
*
|
|
* F4 90..BF would encode values above Unicode's maximum scalar value,
|
|
* U+10FFFF.
|
|
*/
|
|
UTF8_RANGE_80_8F,
|
|
};
|
|
|
|
|
|
typedef struct {
|
|
U8 lo;
|
|
U8 hi;
|
|
} Utf8Range;
|
|
|
|
/*
|
|
* Allowed range of the first continuation byte for each range class.
|
|
*
|
|
* The normal continuation-byte interval is 0x80..0xBF.
|
|
*/
|
|
static const Utf8Range utf8_ranges[] = {
|
|
[UTF8_RANGE_NONE] = { 0x00, 0x00 },
|
|
[UTF8_RANGE_80_BF] = { 0x80, 0xBF },
|
|
[UTF8_RANGE_A0_BF] = { 0xA0, 0xBF },
|
|
[UTF8_RANGE_80_9F] = { 0x80, 0x9F },
|
|
[UTF8_RANGE_90_BF] = { 0x90, 0xBF },
|
|
[UTF8_RANGE_80_8F] = { 0x80, 0x8F },
|
|
};
|
|
|
|
|
|
|
|
/*
|
|
* Each entry in utf8_lead[] is an 8-bit descriptor:
|
|
*
|
|
* 7 6 5 4 3 2 1 0
|
|
* +---------+-----+
|
|
* | range | len |
|
|
* +---------+-----+
|
|
*
|
|
* len:
|
|
* Number of bytes in the UTF-8 sequence.
|
|
*
|
|
* 0 = byte cannot start a valid UTF-8 sequence
|
|
* 1 = ASCII
|
|
* 2 = two-byte UTF-8 sequence
|
|
* 3 = three-byte UTF-8 sequence
|
|
* 4 = four-byte UTF-8 sequence
|
|
*
|
|
* range:
|
|
* Selects the legal range for byte 1, the first continuation byte.
|
|
*
|
|
* This range is enough to encode all of UTF-8's special boundary rules:
|
|
*
|
|
* E0 A0..BF prevents overlong 3-byte encodings
|
|
* ED 80..9F excludes UTF-16 surrogate code points
|
|
* F0 90..BF prevents overlong 4-byte encodings
|
|
* F4 80..8F limits Unicode to U+10FFFF
|
|
*
|
|
* All continuation bytes after byte 1 simply use the normal 80..BF range.
|
|
*/
|
|
/*
|
|
* Classification of every possible first byte.
|
|
*
|
|
* The zero-initialized entries are invalid UTF-8 leading bytes:
|
|
*
|
|
* 80..BF continuation bytes cannot start a sequence
|
|
* C0..C1 would produce overlong two-byte encodings
|
|
* F5..FF would encode values above U+10FFFF or are otherwise invalid
|
|
*
|
|
* Valid leading-byte ranges:
|
|
*
|
|
* 00..7F ASCII
|
|
* C2..DF 2 bytes
|
|
* E0..EF 3 bytes
|
|
* F0..F4 4 bytes
|
|
*
|
|
* E0, ED, F0, and F4 receive special range classes because their second
|
|
* byte is constrained beyond the normal 80..BF continuation-byte range.
|
|
*
|
|
*/
|
|
static const U8 utf8_lead[256] = {
|
|
[0x00 ... 0x7F] = UTF8_DESC(1, UTF8_RANGE_NONE),
|
|
|
|
[0xC2 ... 0xDF] = UTF8_DESC(2, UTF8_RANGE_80_BF),
|
|
|
|
[0xE0] = UTF8_DESC(3, UTF8_RANGE_A0_BF),
|
|
|
|
[0xE1 ... 0xEC] = UTF8_DESC(3, UTF8_RANGE_80_BF),
|
|
|
|
[0xED] = UTF8_DESC(3, UTF8_RANGE_80_9F),
|
|
|
|
[0xEE ... 0xEF] = UTF8_DESC(3, UTF8_RANGE_80_BF),
|
|
|
|
[0xF0] = UTF8_DESC(4, UTF8_RANGE_90_BF),
|
|
|
|
[0xF1 ... 0xF3] = UTF8_DESC(4, UTF8_RANGE_80_BF),
|
|
|
|
[0xF4] = UTF8_DESC(4, UTF8_RANGE_80_8F),
|
|
};
|
|
|
|
/*
|
|
* Decode one Unicode scalar value from UTF-8.
|
|
*
|
|
* Valid input:
|
|
*
|
|
* returns the decoded Rune
|
|
* valid = true
|
|
* width = number of UTF-8 bytes consumed
|
|
*
|
|
* Invalid input:
|
|
*
|
|
* returns U+FFFD
|
|
* valid = false
|
|
* width = 1
|
|
*
|
|
* Consuming one byte on malformed input guarantees forward progress and
|
|
* allows decoding to resume immediately after the offending byte.
|
|
*
|
|
* End of input:
|
|
*
|
|
* returns RUNE_EOF
|
|
* valid = false
|
|
* width = 0
|
|
*/
|
|
RuneDecode _str_utf8_decode_full(Str str, U32 offset)
|
|
{
|
|
if (str.len == 0 || str.len <= offset) {
|
|
return (RuneDecode) {
|
|
.rune = RUNE_EOF,
|
|
.width = 0,
|
|
.valid = false,
|
|
};
|
|
}
|
|
U8 b0 = str.str[offset];
|
|
U8 desc = utf8_lead[b0];
|
|
U8 width = UTF8_DESC_LENGTH(desc);
|
|
/*
|
|
* width == 0 covers:
|
|
*
|
|
* 80..BF
|
|
* C0..C1
|
|
* F5..FF
|
|
*/
|
|
if (width == 0)
|
|
goto invalid;
|
|
|
|
/*
|
|
* We must have the complete candidate sequence before accessing any
|
|
* continuation bytes below.
|
|
*/
|
|
if (str.len - offset < width)
|
|
goto invalid;
|
|
|
|
/*
|
|
* Validate the first continuation byte.
|
|
*
|
|
* Its descriptor-selected range handles both the normal continuation
|
|
* range and the E0 / ED / F0 / F4 boundary cases.
|
|
*/
|
|
Utf8Range range = utf8_ranges[UTF8_DESC_RANGE(desc)];
|
|
U8 b1 = str.str[offset + 1];
|
|
|
|
if (b1 < range.lo || b1 > range.hi)
|
|
goto invalid;
|
|
|
|
/*
|
|
* Decode paths are intentionally unrolled.
|
|
*
|
|
* Besides avoiding a loop in this hot function, each case combines
|
|
* continuation-byte validation with scalar construction. The compiler
|
|
* therefore sees fixed-width operations for every UTF-8 length.
|
|
*/
|
|
switch (width) {
|
|
case 2: {
|
|
/*
|
|
* 110xxxxx 10xxxxxx
|
|
*
|
|
* C2..DF in the lead table already guarantees:
|
|
*
|
|
* - exactly a legal two-byte leader
|
|
* - no C0/C1 overlong encoding
|
|
*
|
|
* b1 has already been checked against 80..BF.
|
|
*/
|
|
Rune rune =
|
|
((Rune)(b0 & 0x1F) << 6) |
|
|
(Rune)(b1 & 0x3F);
|
|
|
|
return (RuneDecode) {
|
|
.rune = rune,
|
|
.width = 2,
|
|
.valid = true,
|
|
};
|
|
}
|
|
|
|
case 3: {
|
|
/*
|
|
* 1110xxxx 10xxxxxx 10xxxxxx
|
|
*
|
|
* b1 has already been validated using the descriptor-specific
|
|
* range. Only b2 still needs a continuation-byte check.
|
|
*/
|
|
U8 b2 = str.str[offset + 2];
|
|
|
|
if ((b2 & 0xC0) != 0x80)
|
|
goto invalid;
|
|
|
|
Rune rune =
|
|
((Rune)(b0 & 0x0F) << 12) |
|
|
((Rune)(b1 & 0x3F) << 6) |
|
|
(Rune)(b2 & 0x3F);
|
|
|
|
return (RuneDecode) {
|
|
.rune = rune,
|
|
.width = 3,
|
|
.valid = true,
|
|
};
|
|
}
|
|
|
|
case 4: {
|
|
/*
|
|
* 11110xxx 10xxxxxx 10xxxxxx 10xxxxxx
|
|
*
|
|
* b1 has already been validated using the descriptor-specific
|
|
* range. b2 and b3 use the ordinary continuation-byte form.
|
|
*/
|
|
U8 b2 = str.str[offset + 2];
|
|
U8 b3 = str.str[offset + 3];
|
|
|
|
/*
|
|
* Validate both remaining continuation bytes with one expression.
|
|
*
|
|
* For a valid continuation byte:
|
|
*
|
|
* byte & 0xC0 == 0x80
|
|
*
|
|
* XOR with 0x80 therefore gives zero in the high two bits.
|
|
* ORing the two transformed bytes lets us test both at once.
|
|
*/
|
|
if ((((b2 ^ 0x80) | (b3 ^ 0x80)) & 0xC0) != 0)
|
|
goto invalid;
|
|
|
|
Rune rune =
|
|
((Rune)(b0 & 0x07) << 18) |
|
|
((Rune)(b1 & 0x3F) << 12) |
|
|
((Rune)(b2 & 0x3F) << 6) |
|
|
(Rune)(b3 & 0x3F);
|
|
|
|
return (RuneDecode) {
|
|
.rune = rune,
|
|
.width = 4,
|
|
.valid = true,
|
|
};
|
|
}
|
|
|
|
default:
|
|
/*
|
|
* The descriptor table can only produce widths 0..4.
|
|
* Width 1 was handled by the ASCII fast path.
|
|
*/
|
|
goto invalid;
|
|
}
|
|
|
|
|
|
invalid:
|
|
return (RuneDecode) {
|
|
.rune = RUNE_ERROR,
|
|
.width = 1,
|
|
.valid = false,
|
|
};
|
|
}
|
|
|
|
|