Lab
/
cosmopolitan
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity
cosmopolitan/third_party/duktape/duk_bi_encoding.c

534 lines
17 KiB
C
Raw Blame History

/*
* WHATWG Encoding API built-ins
*
* API specification: https://encoding.spec.whatwg.org/#api
* Web IDL: https://www.w3.org/TR/WebIDL/
*/
#include "third_party/duktape/duk_internal.h"
/*
* Data structures for encoding/decoding
*/
typedef struct {
duk_uint8_t *out; /* where to write next byte(s) */
duk_codepoint_t lead; /* lead surrogate */
} duk__encode_context;
typedef struct {
/* UTF-8 decoding state */
duk_codepoint_t codepoint; /* built up incrementally */
duk_uint8_t upper; /* max value of next byte (decode error otherwise) */
duk_uint8_t lower; /* min value of next byte (ditto) */
duk_uint8_t needed; /* how many more bytes we need */
duk_uint8_t bom_handled; /* BOM seen or no longer expected */
/* Decoder configuration */
duk_uint8_t fatal;
duk_uint8_t ignore_bom;
} duk__decode_context;
/* The signed duk_codepoint_t type is used to signal a decoded codepoint
* (>= 0) or various other states using negative values.
*/
#define DUK__CP_CONTINUE (-1) /* continue to next byte, no completed codepoint */
#define DUK__CP_ERROR (-2) /* decoding error */
#define DUK__CP_RETRY (-3) /* decoding error; retry last byte */
/*
* Raw helpers for encoding/decoding
*/
/* Emit UTF-8 (= CESU-8) encoded U+FFFD (replacement char), i.e. ef bf bd. */
DUK_LOCAL duk_uint8_t *duk__utf8_emit_repl(duk_uint8_t *ptr) {
*ptr++ = 0xef;
*ptr++ = 0xbf;
*ptr++ = 0xbd;
return ptr;
}
DUK_LOCAL void duk__utf8_decode_init(duk__decode_context *dec_ctx) {
/* (Re)init the decoding state of 'dec_ctx' but leave decoder
* configuration fields untouched.
*/
dec_ctx->codepoint = 0x0000L;
dec_ctx->upper = 0xbf;
dec_ctx->lower = 0x80;
dec_ctx->needed = 0;
dec_ctx->bom_handled = 0;
}
DUK_LOCAL duk_codepoint_t duk__utf8_decode_next(duk__decode_context *dec_ctx, duk_uint8_t x) {
/*
* UTF-8 algorithm based on the Encoding specification:
* https://encoding.spec.whatwg.org/#utf-8-decoder
*
* Two main states: decoding initial byte vs. decoding continuation
* bytes. Shortest length encoding is validated by restricting the
* allowed range of first continuation byte using 'lower' and 'upper'.
*/
if (dec_ctx->needed == 0) {
/* process initial byte */
if (x <= 0x7f) {
/* U+0000-U+007F, 1 byte (ASCII) */
return (duk_codepoint_t) x;
} else if (x >= 0xc2 && x <= 0xdf) {
/* U+0080-U+07FF, 2 bytes */
dec_ctx->needed = 1;
dec_ctx->codepoint = x & 0x1f;
DUK_ASSERT(dec_ctx->lower == 0x80);
DUK_ASSERT(dec_ctx->upper == 0xbf);
return DUK__CP_CONTINUE;
} else if (x >= 0xe0 && x <= 0xef) {
/* U+0800-U+FFFF, 3 bytes */
if (x == 0xe0) {
dec_ctx->lower = 0xa0;
DUK_ASSERT(dec_ctx->upper == 0xbf);
} else if (x == 0xed) {
DUK_ASSERT(dec_ctx->lower == 0x80);
dec_ctx->upper = 0x9f;
}
dec_ctx->needed = 2;
dec_ctx->codepoint = x & 0x0f;
return DUK__CP_CONTINUE;
} else if (x >= 0xf0 && x <= 0xf4) {
/* U+010000-U+10FFFF, 4 bytes */
if (x == 0xf0) {
dec_ctx->lower = 0x90;
DUK_ASSERT(dec_ctx->upper == 0xbf);
} else if (x == 0xf4) {
DUK_ASSERT(dec_ctx->lower == 0x80);
dec_ctx->upper = 0x8f;
}
dec_ctx->needed = 3;
dec_ctx->codepoint = x & 0x07;
return DUK__CP_CONTINUE;
} else {
/* not a legal initial byte */
return DUK__CP_ERROR;
}
} else {
/* process continuation byte */
if (x >= dec_ctx->lower && x <= dec_ctx->upper) {
dec_ctx->lower = 0x80;
dec_ctx->upper = 0xbf;
dec_ctx->codepoint = (dec_ctx->codepoint << 6) | (x & 0x3f);
if (--dec_ctx->needed > 0) {
/* need more bytes */
return DUK__CP_CONTINUE;
} else {
/* got a codepoint */
duk_codepoint_t ret;
DUK_ASSERT(dec_ctx->codepoint <= 0x10ffffL); /* Decoding rules guarantee. */
ret = dec_ctx->codepoint;
dec_ctx->codepoint = 0x0000L;
dec_ctx->needed = 0;
return ret;
}
} else {
/* We just encountered an illegal UTF-8 continuation byte. This might
* be the initial byte of the next character; if we return a plain
* error status and the decoder is in replacement mode, the character
* will be masked. We still need to alert the caller to the error
* though.
*/
dec_ctx->codepoint = 0x0000L;
dec_ctx->needed = 0;
dec_ctx->lower = 0x80;
dec_ctx->upper = 0xbf;
return DUK__CP_RETRY;
}
}
}
#if defined(DUK_USE_ENCODING_BUILTINS)
DUK_LOCAL void duk__utf8_encode_char(void *udata, duk_codepoint_t codepoint) {
duk__encode_context *enc_ctx;
DUK_ASSERT(codepoint >= 0);
enc_ctx = (duk__encode_context *) udata;
DUK_ASSERT(enc_ctx != NULL);
#if !defined(DUK_USE_PREFER_SIZE)
if (codepoint <= 0x7f && enc_ctx->lead == 0x0000L) {
/* Fast path for ASCII. */
*enc_ctx->out++ = (duk_uint8_t) codepoint;
return;
}
#endif
if (DUK_UNLIKELY(codepoint > 0x10ffffL)) {
/* cannot legally encode in UTF-8 */
codepoint = DUK_UNICODE_CP_REPLACEMENT_CHARACTER;
} else if (codepoint >= 0xd800L && codepoint <= 0xdfffL) {
if (codepoint <= 0xdbffL) {
/* high surrogate */
duk_codepoint_t prev_lead = enc_ctx->lead;
enc_ctx->lead = codepoint;
if (prev_lead == 0x0000L) {
/* high surrogate, no output */
return;
} else {
/* consecutive high surrogates, consider first one unpaired */
codepoint = DUK_UNICODE_CP_REPLACEMENT_CHARACTER;
}
} else {
/* low surrogate */
if (enc_ctx->lead != 0x0000L) {
codepoint = (duk_codepoint_t) (0x010000L + ((enc_ctx->lead - 0xd800L) << 10) + (codepoint - 0xdc00L));
enc_ctx->lead = 0x0000L;
} else {
/* unpaired low surrogate */
DUK_ASSERT(enc_ctx->lead == 0x0000L);
codepoint = DUK_UNICODE_CP_REPLACEMENT_CHARACTER;
}
}
} else {
if (enc_ctx->lead != 0x0000L) {
/* unpaired high surrogate: emit replacement character and the input codepoint */
enc_ctx->lead = 0x0000L;
enc_ctx->out = duk__utf8_emit_repl(enc_ctx->out);
}
}
/* Codepoint may be original input, a decoded surrogate pair, or may
* have been replaced with U+FFFD.
*/
enc_ctx->out += duk_unicode_encode_xutf8((duk_ucodepoint_t) codepoint, enc_ctx->out);
}
#endif /* DUK_USE_ENCODING_BUILTINS */
/* Shared helper for buffer-to-string using a TextDecoder() compatible UTF-8
* decoder.
*/
DUK_LOCAL duk_ret_t duk__decode_helper(duk_hthread *thr, duk__decode_context *dec_ctx) {
const duk_uint8_t *input;
duk_size_t len = 0;
duk_size_t len_tmp;
duk_bool_t stream = 0;
duk_codepoint_t codepoint;
duk_uint8_t *output;
const duk_uint8_t *in;
duk_uint8_t *out;
DUK_ASSERT(dec_ctx != NULL);
/* Careful with input buffer pointer: any side effects involving
* code execution (e.g. getters, coercion calls, and finalizers)
* may cause a resize and invalidate a pointer we've read. This
* is why the pointer is actually looked up at the last minute.
* Argument validation must still happen first to match WHATWG
* required side effect order.
*/
if (duk_is_undefined(thr, 0)) {
duk_push_fixed_buffer_nozero(thr, 0);
duk_replace(thr, 0);
}
(void) duk_require_buffer_data(thr, 0, &len); /* Need 'len', avoid pointer. */
if (duk_check_type_mask(thr, 1, DUK_TYPE_MASK_UNDEFINED |
DUK_TYPE_MASK_NULL |
DUK_TYPE_MASK_NONE)) {
/* Use defaults, treat missing value like undefined. */
} else {
duk_require_type_mask(thr, 1, DUK_TYPE_MASK_UNDEFINED |
DUK_TYPE_MASK_NULL |
DUK_TYPE_MASK_LIGHTFUNC |
DUK_TYPE_MASK_BUFFER |
DUK_TYPE_MASK_OBJECT);
if (duk_get_prop_literal(thr, 1, "stream")) {
stream = duk_to_boolean(thr, -1);
}
}
/* Allowance is 3*len in the general case because all bytes may potentially
* become U+FFFD. If the first byte completes a non-BMP codepoint it will
* decode to a CESU-8 surrogate pair (6 bytes) so we allow 3 extra bytes to
* compensate: (1*3)+3 = 6. Non-BMP codepoints are safe otherwise because
* the 4->6 expansion is well under the 3x allowance.
*
* XXX: As with TextEncoder, need a better buffer allocation strategy here.
*/
if (len >= (DUK_HBUFFER_MAX_BYTELEN / 3) - 3) {
DUK_ERROR_TYPE(thr, DUK_STR_RESULT_TOO_LONG);
DUK_WO_NORETURN(return 0;);
}
output = (duk_uint8_t *) duk_push_fixed_buffer_nozero(thr, 3 + (3 * len)); /* used parts will be always manually written over */
input = (const duk_uint8_t *) duk_get_buffer_data(thr, 0, &len_tmp);
DUK_ASSERT(input != NULL || len == 0);
if (DUK_UNLIKELY(len != len_tmp)) {
/* Very unlikely but possible: source buffer was resized by
* a side effect when fixed buffer was pushed. Output buffer
* may not be large enough to hold output, so just fail if
* length has changed.
*/
DUK_D(DUK_DPRINT("input buffer resized by side effect, fail"));
goto fail_type;
}
/* From this point onwards it's critical that no side effect occur
* which may disturb 'input': finalizer execution, property accesses,
* active coercions, etc. Even an allocation related mark-and-sweep
* may affect the pointer because it may trigger a pending finalizer.
*/
in = input;
out = output;
while (in < input + len) {
codepoint = duk__utf8_decode_next(dec_ctx, *in++);
if (codepoint < 0) {
if (codepoint == DUK__CP_CONTINUE) {
continue;
}
/* Decoding error with or without retry. */
DUK_ASSERT(codepoint == DUK__CP_ERROR || codepoint == DUK__CP_RETRY);
if (codepoint == DUK__CP_RETRY) {
--in; /* retry last byte */
}
/* replacement mode: replace with U+FFFD */
codepoint = DUK_UNICODE_CP_REPLACEMENT_CHARACTER;
if (dec_ctx->fatal) {
/* fatal mode: throw a TypeError */
goto fail_type;
}
/* Continue with 'codepoint', Unicode replacement. */
}
DUK_ASSERT(codepoint >= 0x0000L && codepoint <= 0x10ffffL);
if (!dec_ctx->bom_handled) {
dec_ctx->bom_handled = 1;
if (codepoint == 0xfeffL && !dec_ctx->ignore_bom) {
continue;
}
}
out += duk_unicode_encode_cesu8((duk_ucodepoint_t) codepoint, out);
DUK_ASSERT(out <= output + (3 + (3 * len)));
}
if (!stream) {
if (dec_ctx->needed != 0) {
/* truncated sequence at end of buffer */
if (dec_ctx->fatal) {
goto fail_type;
} else {
out += duk_unicode_encode_cesu8(DUK_UNICODE_CP_REPLACEMENT_CHARACTER, out);
DUK_ASSERT(out <= output + (3 + (3 * len)));
}
}
duk__utf8_decode_init(dec_ctx); /* Initialize decoding state for potential reuse. */
}
/* Output buffer is fixed and thus stable even if there had been
* side effects (which there shouldn't be).
*/
duk_push_lstring(thr, (const char *) output, (duk_size_t) (out - output));
return 1;
fail_type:
DUK_ERROR_TYPE(thr, DUK_STR_UTF8_DECODE_FAILED);
DUK_WO_NORETURN(return 0;);
}
/*
* Built-in bindings
*/
#if defined(DUK_USE_ENCODING_BUILTINS)
DUK_INTERNAL duk_ret_t duk_bi_textencoder_constructor(duk_hthread *thr) {
/* TextEncoder currently requires no persistent state, so the constructor
* does nothing on purpose.
*/
duk_require_constructor_call(thr);
return 0;
}
DUK_INTERNAL duk_ret_t duk_bi_textencoder_prototype_encoding_getter(duk_hthread *thr) {
duk_push_literal(thr, "utf-8");
return 1;
}
DUK_INTERNAL duk_ret_t duk_bi_textencoder_prototype_encode(duk_hthread *thr) {
duk__encode_context enc_ctx;
duk_size_t len;
duk_size_t final_len;
duk_uint8_t *output;
DUK_ASSERT_TOP(thr, 1);
if (duk_is_undefined(thr, 0)) {
len = 0;
} else {
duk_hstring *h_input;
h_input = duk_to_hstring(thr, 0);
DUK_ASSERT(h_input != NULL);
len = (duk_size_t) DUK_HSTRING_GET_CHARLEN(h_input);
if (len >= DUK_HBUFFER_MAX_BYTELEN / 3) {
DUK_ERROR_TYPE(thr, DUK_STR_RESULT_TOO_LONG);
DUK_WO_NORETURN(return 0;);
}
}
/* Allowance is 3*len because all bytes can potentially be replaced with
* U+FFFD -- which rather inconveniently encodes to 3 bytes in UTF-8.
* Rely on dynamic buffer data pointer stability: no other code has
* access to the data pointer.
*
* XXX: The buffer allocation strategy used here is rather inefficient.
* Maybe switch to a chunk-based strategy, or preprocess the string to
* figure out the space needed ahead of time?
*/
DUK_ASSERT(3 * len >= len);
output = (duk_uint8_t *) duk_push_dynamic_buffer(thr, 3 * len);
if (len > 0) {
DUK_ASSERT(duk_is_string(thr, 0)); /* True if len > 0. */
/* XXX: duk_decode_string() is used to process the input
* string. For standard ECMAScript strings, represented
* internally as CESU-8, this is fine. However, behavior
* beyond CESU-8 is not very strict: codepoints using an
* extended form of UTF-8 are also accepted, and invalid
* codepoint sequences (which are allowed in Duktape strings)
* are not handled as well as they could (e.g. invalid
* continuation bytes may mask following codepoints).
* This is how ECMAScript code would also see such strings.
* Maybe replace duk_decode_string() with an explicit strict
* CESU-8 decoder here?
*/
enc_ctx.lead = 0x0000L;
enc_ctx.out = output;
duk_decode_string(thr, 0, duk__utf8_encode_char, (void *) &enc_ctx);
if (enc_ctx.lead != 0x0000L) {
/* unpaired high surrogate at end of string */
enc_ctx.out = duk__utf8_emit_repl(enc_ctx.out);
DUK_ASSERT(enc_ctx.out <= output + (3 * len));
}
/* The output buffer is usually very much oversized, so shrink it to
* actually needed size. Pointer stability assumed up to this point.
*/
DUK_ASSERT_TOP(thr, 2);
DUK_ASSERT(output == (duk_uint8_t *) duk_get_buffer_data(thr, -1, NULL));
final_len = (duk_size_t) (enc_ctx.out - output);
duk_resize_buffer(thr, -1, final_len);
/* 'output' and 'enc_ctx.out' are potentially invalidated by the resize. */
} else {
final_len = 0;
}
/* Standard WHATWG output is a Uint8Array. Here the Uint8Array will
* be backed by a dynamic buffer which differs from e.g. Uint8Arrays
* created as 'new Uint8Array(N)'. ECMAScript code won't see the
* difference but C code will. When bufferobjects are not supported,
* returns a plain dynamic buffer.
*/
#if defined(DUK_USE_BUFFEROBJECT_SUPPORT)
duk_push_buffer_object(thr, -1, 0, final_len, DUK_BUFOBJ_UINT8ARRAY);
#endif
return 1;
}
DUK_INTERNAL duk_ret_t duk_bi_textdecoder_constructor(duk_hthread *thr) {
duk__decode_context *dec_ctx;
duk_bool_t fatal = 0;
duk_bool_t ignore_bom = 0;
DUK_ASSERT_TOP(thr, 2);
duk_require_constructor_call(thr);
if (!duk_is_undefined(thr, 0)) {
/* XXX: For now ignore 'label' (encoding identifier). */
duk_to_string(thr, 0);
}
if (!duk_is_null_or_undefined(thr, 1)) {
if (duk_get_prop_literal(thr, 1, "fatal")) {
fatal = duk_to_boolean(thr, -1);
}
if (duk_get_prop_literal(thr, 1, "ignoreBOM")) {
ignore_bom = duk_to_boolean(thr, -1);
}
}
duk_push_this(thr);
/* The decode context is not assumed to be zeroed; all fields are
* initialized explicitly.
*/
dec_ctx = (duk__decode_context *) duk_push_fixed_buffer(thr, sizeof(duk__decode_context));
dec_ctx->fatal = (duk_uint8_t) fatal;
dec_ctx->ignore_bom = (duk_uint8_t) ignore_bom;
duk__utf8_decode_init(dec_ctx); /* Initializes remaining fields. */
duk_put_prop_literal(thr, -2, DUK_INTERNAL_SYMBOL("Context"));
return 0;
}
/* Get TextDecoder context from 'this'; leaves garbage on stack. */
DUK_LOCAL duk__decode_context *duk__get_textdecoder_context(duk_hthread *thr) {
duk__decode_context *dec_ctx;
duk_push_this(thr);
duk_get_prop_literal(thr, -1, DUK_INTERNAL_SYMBOL("Context"));
dec_ctx = (duk__decode_context *) duk_require_buffer(thr, -1, NULL);
DUK_ASSERT(dec_ctx != NULL);
return dec_ctx;
}
DUK_INTERNAL duk_ret_t duk_bi_textdecoder_prototype_shared_getter(duk_hthread *thr) {
duk__decode_context *dec_ctx;
duk_int_t magic;
dec_ctx = duk__get_textdecoder_context(thr);
magic = duk_get_current_magic(thr);
switch (magic) {
case 0:
/* Encoding is now fixed, so _Context lookup is only needed to
* validate the 'this' binding (TypeError if not TextDecoder-like).
*/
duk_push_literal(thr, "utf-8");
break;
case 1:
duk_push_boolean(thr, dec_ctx->fatal);
break;
default:
duk_push_boolean(thr, dec_ctx->ignore_bom);
break;
}
return 1;
}
DUK_INTERNAL duk_ret_t duk_bi_textdecoder_prototype_decode(duk_hthread *thr) {
duk__decode_context *dec_ctx;
dec_ctx = duk__get_textdecoder_context(thr);
return duk__decode_helper(thr, dec_ctx);
}
#endif /* DUK_USE_ENCODING_BUILTINS */
/*
* Internal helper for Node.js Buffer
*/
/* Internal helper used for Node.js Buffer .toString(). Value stack convention
* is currently odd: it mimics TextDecoder .decode() so that argument must be at
* index 0, and decode options (not present for Buffer) at index 1. Return value
* is a Duktape/C function return value.
*/
DUK_INTERNAL duk_ret_t duk_textdecoder_decode_utf8_nodejs(duk_hthread *thr) {
duk__decode_context dec_ctx;
dec_ctx.fatal = 0; /* use replacement chars */
dec_ctx.ignore_bom = 1; /* ignore BOMs (matches Node.js Buffer .toString()) */
duk__utf8_decode_init(&dec_ctx);
return duk__decode_helper(thr, &dec_ctx);
}
Reference in New Issue View Git Blame Copy Permalink