Reorganize files to base/ folder (#1)
This commit was merged in pull request #1.
This commit is contained in:
@@ -0,0 +1,583 @@
|
||||
#include "string.h"
|
||||
|
||||
Str str_empty(void)
|
||||
{
|
||||
static const U8 empty_;
|
||||
static const Str str = {(U8 *)&empty_, 0};
|
||||
return str;
|
||||
}
|
||||
|
||||
Str str_slice(Str str, U32 start, U32 len)
|
||||
{
|
||||
Assert(start <= str.len);
|
||||
Assert(len <= str.len - start);
|
||||
|
||||
Str new_str = str_create(&str.str[start], len);
|
||||
return new_str;
|
||||
}
|
||||
|
||||
Str str_concat(Allocator *alloc, Str first, Str second)
|
||||
{
|
||||
Str str;
|
||||
Assert(first.len <= U32_Max - second.len);
|
||||
str.len = first.len + second.len;
|
||||
str.str = allocator_push(alloc, str.len, 1);;
|
||||
|
||||
MemCopy(str.str, first.str, first.len);
|
||||
MemCopy(&str.str[first.len], second.str, second.len);
|
||||
|
||||
return str;
|
||||
}
|
||||
|
||||
Str str_push(Allocator *alloc, Str str)
|
||||
{
|
||||
U8 *mem = (U8 *)allocator_push(alloc, str.len, 1);
|
||||
MemCopy(mem, str.str, str.len);
|
||||
|
||||
return str_create(mem, str.len);
|
||||
}
|
||||
|
||||
Str str_ascii_trim_left(Str str)
|
||||
{
|
||||
U8 *start = str.str;
|
||||
U8 *end = str.str + str.len;
|
||||
while (start != end) {
|
||||
if (!char_is_ascii_whitespace(*start)) {
|
||||
break;
|
||||
}
|
||||
start++;
|
||||
}
|
||||
|
||||
return str_create(start, (U32)(end - start));
|
||||
}
|
||||
|
||||
Str str_ascii_trim_right(Str str)
|
||||
{
|
||||
U8 *start = str.str;
|
||||
U8 *end = str.str + str.len;
|
||||
while (start != end) {
|
||||
if (!char_is_ascii_whitespace(*(end - 1))) {
|
||||
break;
|
||||
}
|
||||
end--;
|
||||
}
|
||||
|
||||
return str_create(start, (U32)(end - start));
|
||||
}
|
||||
|
||||
Str str_ascii_trim(Str str)
|
||||
{
|
||||
Str new_str = str_ascii_trim_left(str);
|
||||
new_str = str_ascii_trim_right(new_str);
|
||||
return new_str;
|
||||
}
|
||||
|
||||
S64 str_index(Str haystack, Str needle)
|
||||
{
|
||||
if (needle.len == 0) {
|
||||
return 0;
|
||||
}
|
||||
|
||||
if (needle.len == 1) {
|
||||
return str_find_byte(haystack, needle.str[0]);
|
||||
}
|
||||
if (needle.len > haystack.len) {
|
||||
return -1;
|
||||
}
|
||||
|
||||
U8 *found = MemFind(haystack.str, haystack.len, needle.str, needle.len);
|
||||
if (found != NULL) {
|
||||
return (S64)(found - haystack.str);
|
||||
}
|
||||
|
||||
return -1;
|
||||
}
|
||||
|
||||
|
||||
|
||||
static INLINE S16 char_hex_to_binary(U8 c)
|
||||
{
|
||||
if (c >='0' && c <='9')
|
||||
return c - '0';
|
||||
else if (c >= 'A' && c <= 'F')
|
||||
return c - 'A' + 10;
|
||||
else if (c >= 'a' && c <= 'f')
|
||||
return c - 'a' + 10;
|
||||
|
||||
return -1;
|
||||
}
|
||||
|
||||
static const char hex_chars[] = "0123456789ABCDEF";
|
||||
|
||||
Str str_hex_encode(Str binary_data, Str hex_result)
|
||||
{
|
||||
Assert(hex_result.len >= binary_data.len * 2);
|
||||
U32 hex_len = binary_data.len * 2;
|
||||
for (U32 i = 0, t = 0; i < binary_data.len; ++i,t+=2) {
|
||||
U8 high_nibble;
|
||||
U8 low_nibble;
|
||||
|
||||
high_nibble = (binary_data.str[i] & 0xF0) >> 4 ;
|
||||
low_nibble = binary_data.str[i] & 0x0F ;
|
||||
|
||||
hex_result.str[t] = hex_chars[high_nibble];
|
||||
hex_result.str[t+1] = hex_chars[low_nibble];
|
||||
}
|
||||
|
||||
hex_result.len = hex_len;
|
||||
return hex_result;
|
||||
}
|
||||
|
||||
Str str_hex_decode(Str hex_str, Str result)
|
||||
{
|
||||
Assert(result.len >= hex_str.len/2);
|
||||
U32 dest_pos = 0;
|
||||
U32 source_pos = 0;
|
||||
|
||||
while (source_pos + 1 < hex_str.len) {
|
||||
|
||||
if (char_is_ascii_whitespace(hex_str.str[source_pos])) {
|
||||
source_pos++;
|
||||
continue;
|
||||
}
|
||||
|
||||
S16 high_nibble;
|
||||
S16 low_nibble;
|
||||
|
||||
high_nibble = char_hex_to_binary(hex_str.str[source_pos]);
|
||||
low_nibble = char_hex_to_binary(hex_str.str[source_pos + 1]);
|
||||
if (high_nibble == -1 || low_nibble == -1) {
|
||||
result.len = 0;
|
||||
return result;
|
||||
}
|
||||
|
||||
result.str[dest_pos] = (high_nibble << 4 ) | low_nibble;
|
||||
source_pos += 2;
|
||||
dest_pos++;
|
||||
}
|
||||
|
||||
result.len = dest_pos;
|
||||
return result;
|
||||
}
|
||||
// split @str on any byte in @separators, place split strings in @vec
|
||||
void str_split(Str str, Str separators, StrVec *vec)
|
||||
{
|
||||
if (vec->max_len == 0 || str.len == 0) {
|
||||
return;
|
||||
}
|
||||
|
||||
U32 start_pos = 0;
|
||||
for (U32 i = 0; i < str.len; i++) {
|
||||
U8 byte = str.str[i];
|
||||
if (str_find_byte(separators, byte) != -1) {
|
||||
if (vec->len >= vec->max_len) {
|
||||
break;
|
||||
}
|
||||
|
||||
Str sub_str = str_slice(str, start_pos, i - start_pos);
|
||||
vec->strings[vec->len] = sub_str;
|
||||
vec->len++;
|
||||
start_pos = i + 1;
|
||||
}
|
||||
}
|
||||
if (start_pos <= str.len && vec->len < vec->max_len) {
|
||||
Str sub_str = str_slice(str, start_pos, str.len - start_pos);
|
||||
vec->strings[vec->len] = sub_str;
|
||||
vec->len++;
|
||||
}
|
||||
}
|
||||
|
||||
Str str_join(Allocator *allocator, StrVec strings, Str separator)
|
||||
{
|
||||
U32 joined_str_len = 0;
|
||||
|
||||
for (U32 i = 0; i < strings.len; i++) {
|
||||
U32 part_len = strings.strings[i].len;
|
||||
Assert(joined_str_len <= U32_Max - part_len); // overflow
|
||||
joined_str_len += part_len;
|
||||
}
|
||||
if (strings.len > 1) {
|
||||
U32 separator_count = strings.len - 1;
|
||||
Assert(separator.len <= U32_Max / separator_count); // overflow
|
||||
Assert(joined_str_len <= U32_Max - separator.len * separator_count); // overflow
|
||||
joined_str_len += separator.len * separator_count;
|
||||
}
|
||||
|
||||
U8 *joined_str = allocator_push(allocator, joined_str_len, 1);
|
||||
if (strings.len > 0) {
|
||||
U32 pos = 0;
|
||||
Str *str = &strings.strings[0];
|
||||
|
||||
MemCopy(&joined_str[pos], str->str, str->len);
|
||||
pos += str->len;
|
||||
|
||||
for (U32 i = 1; i < strings.len; i++) {
|
||||
str = &strings.strings[i];
|
||||
MemCopy(&joined_str[pos], separator.str, separator.len);
|
||||
pos += separator.len;
|
||||
MemCopy(&joined_str[pos], str->str, str->len);
|
||||
pos += str->len;
|
||||
}
|
||||
}
|
||||
|
||||
return str_create(joined_str, joined_str_len);
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*
|
||||
* Each entry in utf8_lead[] is an 8-bit descriptor:
|
||||
*
|
||||
* 7 6 5 4 3 2 1 0
|
||||
* +---------+-----+
|
||||
* | range | len |
|
||||
* +---------+-----+
|
||||
*
|
||||
* len:
|
||||
* Number of bytes in the UTF-8 sequence.
|
||||
*
|
||||
* 0 = byte cannot start a valid UTF-8 sequence
|
||||
* 1 = ASCII
|
||||
* 2 = two-byte UTF-8 sequence
|
||||
* 3 = three-byte UTF-8 sequence
|
||||
* 4 = four-byte UTF-8 sequence
|
||||
*
|
||||
* range:
|
||||
* Selects the legal range for byte 1, the first continuation byte.
|
||||
*
|
||||
* This range is enough to encode all of UTF-8's special boundary rules:
|
||||
*
|
||||
* E0 A0..BF prevents overlong 3-byte encodings
|
||||
* ED 80..9F excludes UTF-16 surrogate code points
|
||||
* F0 90..BF prevents overlong 4-byte encodings
|
||||
* F4 80..8F limits Unicode to U+10FFFF
|
||||
*
|
||||
* All continuation bytes after byte 1 simply use the normal 80..BF range.
|
||||
*/
|
||||
|
||||
#define UTF8_LENGTH_BITS 3u
|
||||
#define UTF8_LENGTH_MASK ((1u << UTF8_LENGTH_BITS) - 1u)
|
||||
|
||||
#define UTF8_DESC(length, range) \
|
||||
((U8)((length) | ((U8)(range) << UTF8_LENGTH_BITS)))
|
||||
|
||||
#define UTF8_DESC_LENGTH(desc) \
|
||||
((U8)((desc) & UTF8_LENGTH_MASK))
|
||||
|
||||
#define UTF8_DESC_RANGE(desc) \
|
||||
((U8)((desc) >> UTF8_LENGTH_BITS))
|
||||
|
||||
|
||||
/*
|
||||
* Classes of allowed values for the first continuation byte.
|
||||
*/
|
||||
enum Utf8RangeClass {
|
||||
/*
|
||||
* No continuation byte exists.
|
||||
*
|
||||
* Used only by ASCII descriptors.
|
||||
*/
|
||||
UTF8_RANGE_NONE = 0,
|
||||
|
||||
/*
|
||||
* Normal continuation-byte range.
|
||||
*/
|
||||
UTF8_RANGE_80_BF,
|
||||
|
||||
/*
|
||||
* E0 xx xx:
|
||||
*
|
||||
* E0 80..9F would encode values below U+0800 and therefore be
|
||||
* non-canonical overlong UTF-8.
|
||||
*/
|
||||
UTF8_RANGE_A0_BF,
|
||||
|
||||
/*
|
||||
* ED xx xx:
|
||||
*
|
||||
* ED A0..BF would encode U+D800..U+DFFF, which are UTF-16 surrogate
|
||||
* code points and are not Unicode scalar values.
|
||||
*/
|
||||
UTF8_RANGE_80_9F,
|
||||
|
||||
/*
|
||||
* F0 xx xx xx:
|
||||
*
|
||||
* F0 80..8F would encode values below U+10000 and therefore be
|
||||
* non-canonical overlong UTF-8.
|
||||
*/
|
||||
UTF8_RANGE_90_BF,
|
||||
|
||||
/*
|
||||
* F4 xx xx xx:
|
||||
*
|
||||
* F4 90..BF would encode values above Unicode's maximum scalar value,
|
||||
* U+10FFFF.
|
||||
*/
|
||||
UTF8_RANGE_80_8F,
|
||||
};
|
||||
|
||||
|
||||
typedef struct {
|
||||
U8 lo;
|
||||
U8 hi;
|
||||
} Utf8Range;
|
||||
|
||||
/*
|
||||
* Allowed range of the first continuation byte for each range class.
|
||||
*
|
||||
* The normal continuation-byte interval is 0x80..0xBF.
|
||||
*/
|
||||
static const Utf8Range utf8_ranges[] = {
|
||||
[UTF8_RANGE_NONE] = { 0x00, 0x00 },
|
||||
[UTF8_RANGE_80_BF] = { 0x80, 0xBF },
|
||||
[UTF8_RANGE_A0_BF] = { 0xA0, 0xBF },
|
||||
[UTF8_RANGE_80_9F] = { 0x80, 0x9F },
|
||||
[UTF8_RANGE_90_BF] = { 0x90, 0xBF },
|
||||
[UTF8_RANGE_80_8F] = { 0x80, 0x8F },
|
||||
};
|
||||
|
||||
|
||||
|
||||
/*
|
||||
* Each entry in utf8_lead[] is an 8-bit descriptor:
|
||||
*
|
||||
* 7 6 5 4 3 2 1 0
|
||||
* +---------+-----+
|
||||
* | range | len |
|
||||
* +---------+-----+
|
||||
*
|
||||
* len:
|
||||
* Number of bytes in the UTF-8 sequence.
|
||||
*
|
||||
* 0 = byte cannot start a valid UTF-8 sequence
|
||||
* 1 = ASCII
|
||||
* 2 = two-byte UTF-8 sequence
|
||||
* 3 = three-byte UTF-8 sequence
|
||||
* 4 = four-byte UTF-8 sequence
|
||||
*
|
||||
* range:
|
||||
* Selects the legal range for byte 1, the first continuation byte.
|
||||
*
|
||||
* This range is enough to encode all of UTF-8's special boundary rules:
|
||||
*
|
||||
* E0 A0..BF prevents overlong 3-byte encodings
|
||||
* ED 80..9F excludes UTF-16 surrogate code points
|
||||
* F0 90..BF prevents overlong 4-byte encodings
|
||||
* F4 80..8F limits Unicode to U+10FFFF
|
||||
*
|
||||
* All continuation bytes after byte 1 simply use the normal 80..BF range.
|
||||
*/
|
||||
/*
|
||||
* Classification of every possible first byte.
|
||||
*
|
||||
* The zero-initialized entries are invalid UTF-8 leading bytes:
|
||||
*
|
||||
* 80..BF continuation bytes cannot start a sequence
|
||||
* C0..C1 would produce overlong two-byte encodings
|
||||
* F5..FF would encode values above U+10FFFF or are otherwise invalid
|
||||
*
|
||||
* Valid leading-byte ranges:
|
||||
*
|
||||
* 00..7F ASCII
|
||||
* C2..DF 2 bytes
|
||||
* E0..EF 3 bytes
|
||||
* F0..F4 4 bytes
|
||||
*
|
||||
* E0, ED, F0, and F4 receive special range classes because their second
|
||||
* byte is constrained beyond the normal 80..BF continuation-byte range.
|
||||
*
|
||||
*/
|
||||
static const U8 utf8_lead[256] = {
|
||||
[0x00 ... 0x7F] = UTF8_DESC(1, UTF8_RANGE_NONE),
|
||||
|
||||
[0xC2 ... 0xDF] = UTF8_DESC(2, UTF8_RANGE_80_BF),
|
||||
|
||||
[0xE0] = UTF8_DESC(3, UTF8_RANGE_A0_BF),
|
||||
|
||||
[0xE1 ... 0xEC] = UTF8_DESC(3, UTF8_RANGE_80_BF),
|
||||
|
||||
[0xED] = UTF8_DESC(3, UTF8_RANGE_80_9F),
|
||||
|
||||
[0xEE ... 0xEF] = UTF8_DESC(3, UTF8_RANGE_80_BF),
|
||||
|
||||
[0xF0] = UTF8_DESC(4, UTF8_RANGE_90_BF),
|
||||
|
||||
[0xF1 ... 0xF3] = UTF8_DESC(4, UTF8_RANGE_80_BF),
|
||||
|
||||
[0xF4] = UTF8_DESC(4, UTF8_RANGE_80_8F),
|
||||
};
|
||||
|
||||
/*
|
||||
* Decode one Unicode scalar value from UTF-8.
|
||||
*
|
||||
* Valid input:
|
||||
*
|
||||
* returns the decoded Rune
|
||||
* valid = true
|
||||
* width = number of UTF-8 bytes consumed
|
||||
*
|
||||
* Invalid input:
|
||||
*
|
||||
* returns U+FFFD
|
||||
* valid = false
|
||||
* width = 1
|
||||
*
|
||||
* Consuming one byte on malformed input guarantees forward progress and
|
||||
* allows decoding to resume immediately after the offending byte.
|
||||
*
|
||||
* End of input:
|
||||
*
|
||||
* returns RUNE_EOF
|
||||
* valid = false
|
||||
* width = 0
|
||||
*/
|
||||
RuneDecode _str_utf8_decode_full(Str str, U32 offset)
|
||||
{
|
||||
if (str.len == 0 || str.len <= offset) {
|
||||
return (RuneDecode) {
|
||||
.rune = RUNE_EOF,
|
||||
.width = 0,
|
||||
.valid = false,
|
||||
};
|
||||
}
|
||||
U8 b0 = str.str[offset];
|
||||
U8 desc = utf8_lead[b0];
|
||||
U8 width = UTF8_DESC_LENGTH(desc);
|
||||
/*
|
||||
* width == 0 covers:
|
||||
*
|
||||
* 80..BF
|
||||
* C0..C1
|
||||
* F5..FF
|
||||
*/
|
||||
if (width == 0)
|
||||
goto invalid;
|
||||
|
||||
/*
|
||||
* We must have the complete candidate sequence before accessing any
|
||||
* continuation bytes below.
|
||||
*/
|
||||
if (str.len - offset < width)
|
||||
goto invalid;
|
||||
|
||||
/*
|
||||
* Validate the first continuation byte.
|
||||
*
|
||||
* Its descriptor-selected range handles both the normal continuation
|
||||
* range and the E0 / ED / F0 / F4 boundary cases.
|
||||
*/
|
||||
Utf8Range range = utf8_ranges[UTF8_DESC_RANGE(desc)];
|
||||
U8 b1 = str.str[offset + 1];
|
||||
|
||||
if (b1 < range.lo || b1 > range.hi)
|
||||
goto invalid;
|
||||
|
||||
/*
|
||||
* Decode paths are intentionally unrolled.
|
||||
*
|
||||
* Besides avoiding a loop in this hot function, each case combines
|
||||
* continuation-byte validation with scalar construction. The compiler
|
||||
* therefore sees fixed-width operations for every UTF-8 length.
|
||||
*/
|
||||
switch (width) {
|
||||
case 2: {
|
||||
/*
|
||||
* 110xxxxx 10xxxxxx
|
||||
*
|
||||
* C2..DF in the lead table already guarantees:
|
||||
*
|
||||
* - exactly a legal two-byte leader
|
||||
* - no C0/C1 overlong encoding
|
||||
*
|
||||
* b1 has already been checked against 80..BF.
|
||||
*/
|
||||
Rune rune =
|
||||
((Rune)(b0 & 0x1F) << 6) |
|
||||
(Rune)(b1 & 0x3F);
|
||||
|
||||
return (RuneDecode) {
|
||||
.rune = rune,
|
||||
.width = 2,
|
||||
.valid = true,
|
||||
};
|
||||
}
|
||||
|
||||
case 3: {
|
||||
/*
|
||||
* 1110xxxx 10xxxxxx 10xxxxxx
|
||||
*
|
||||
* b1 has already been validated using the descriptor-specific
|
||||
* range. Only b2 still needs a continuation-byte check.
|
||||
*/
|
||||
U8 b2 = str.str[offset + 2];
|
||||
|
||||
if ((b2 & 0xC0) != 0x80)
|
||||
goto invalid;
|
||||
|
||||
Rune rune =
|
||||
((Rune)(b0 & 0x0F) << 12) |
|
||||
((Rune)(b1 & 0x3F) << 6) |
|
||||
(Rune)(b2 & 0x3F);
|
||||
|
||||
return (RuneDecode) {
|
||||
.rune = rune,
|
||||
.width = 3,
|
||||
.valid = true,
|
||||
};
|
||||
}
|
||||
|
||||
case 4: {
|
||||
/*
|
||||
* 11110xxx 10xxxxxx 10xxxxxx 10xxxxxx
|
||||
*
|
||||
* b1 has already been validated using the descriptor-specific
|
||||
* range. b2 and b3 use the ordinary continuation-byte form.
|
||||
*/
|
||||
U8 b2 = str.str[offset + 2];
|
||||
U8 b3 = str.str[offset + 3];
|
||||
|
||||
/*
|
||||
* Validate both remaining continuation bytes with one expression.
|
||||
*
|
||||
* For a valid continuation byte:
|
||||
*
|
||||
* byte & 0xC0 == 0x80
|
||||
*
|
||||
* XOR with 0x80 therefore gives zero in the high two bits.
|
||||
* ORing the two transformed bytes lets us test both at once.
|
||||
*/
|
||||
if ((((b2 ^ 0x80) | (b3 ^ 0x80)) & 0xC0) != 0)
|
||||
goto invalid;
|
||||
|
||||
Rune rune =
|
||||
((Rune)(b0 & 0x07) << 18) |
|
||||
((Rune)(b1 & 0x3F) << 12) |
|
||||
((Rune)(b2 & 0x3F) << 6) |
|
||||
(Rune)(b3 & 0x3F);
|
||||
|
||||
return (RuneDecode) {
|
||||
.rune = rune,
|
||||
.width = 4,
|
||||
.valid = true,
|
||||
};
|
||||
}
|
||||
|
||||
default:
|
||||
/*
|
||||
* The descriptor table can only produce widths 0..4.
|
||||
* Width 1 was handled by the ASCII fast path.
|
||||
*/
|
||||
goto invalid;
|
||||
}
|
||||
|
||||
|
||||
invalid:
|
||||
return (RuneDecode) {
|
||||
.rune = RUNE_ERROR,
|
||||
.width = 1,
|
||||
.valid = false,
|
||||
};
|
||||
}
|
||||
|
||||
|
||||
Reference in New Issue
Block a user