From 12bc13eeb82592f768fb0e304bb9b6eaf1c2ce17 Mon Sep 17 00:00:00 2001 From: Paul Sokolovsky Date: Fri, 13 Jun 2014 01:05:19 +0300 Subject: [PATCH 01/32] mpconfig.h: Add MICROPY_PY_BUILTINS_STR_UNICODE. --- py/mpconfig.h | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/py/mpconfig.h b/py/mpconfig.h index d7504c140..5194ba028 100644 --- a/py/mpconfig.h +++ b/py/mpconfig.h @@ -249,6 +249,11 @@ typedef double mp_float_t; /*****************************************************************************/ /* Fine control over Python builtins, classes, modules, etc */ +// Whether str object is proper unicode +#ifndef MICROPY_PY_BUILTINS_STR_UNICODE +#define MICROPY_PY_BUILTINS_STR_UNICODE (0) +#endif + // Whether to support set object #ifndef MICROPY_PY_BUILTINS_SET #define MICROPY_PY_BUILTINS_SET (1) From c88987c1af10e19ab2703231d0702202127eb046 Mon Sep 17 00:00:00 2001 From: Chris Angelico Date: Wed, 4 Jun 2014 05:28:12 +1000 Subject: [PATCH 02/32] py: Implement basic unicode functions. --- py/misc.h | 4 +++- py/unicode.c | 33 +++++++++++++++++++++++++++++---- 2 files changed, 32 insertions(+), 5 deletions(-) diff --git a/py/misc.h b/py/misc.h index 3f62e3198..97e9b30ed 100644 --- a/py/misc.h +++ b/py/misc.h @@ -100,7 +100,9 @@ bool unichar_isupper(unichar c); bool unichar_islower(unichar c); unichar unichar_tolower(unichar c); unichar unichar_toupper(unichar c); -#define unichar_charlen(s, bytelen) (bytelen) +uint unichar_charlen(const char *str, uint len); +#define UTF8_IS_NONASCII(ch) ((ch) & 0x80) +#define UTF8_IS_CONT(ch) (((ch) & 0xC0) == 0x80) /** variable string *********************************************/ diff --git a/py/unicode.c b/py/unicode.c index 88f835131..0da247889 100644 --- a/py/unicode.c +++ b/py/unicode.c @@ -65,14 +65,39 @@ STATIC const uint8_t attr[] = { AT_LO, AT_LO, AT_LO, AT_PR, AT_PR, AT_PR, AT_PR, 0 }; -unichar utf8_get_char(const byte *s) { - return *s; +unichar utf8_get_char(const char *s) { + unichar ord = *s++; + if (!UTF8_IS_NONASCII(ord)) return ord; + ord &= 0x7F; + for (unichar mask = 0x40; ord & mask; mask >>= 1) { + ord &= ~mask; + } + while (UTF8_IS_CONT(*s)) { + ord = (ord << 6) | (*s++ & 0x3F); + } + return ord; } -const byte *utf8_next_char(const byte *s) { - return s + 1; +char *utf8_next_char(const char *s) { + ++s; + while (UTF8_IS_CONT(*s)) { + ++s; + } + return (char *)s; } +uint unichar_charlen(const char *str, uint len) +{ + uint charlen = 0; + for (const char *top = str + len; str < top; ++str) { + if (!UTF8_IS_CONT(*str)) { + ++charlen; + } + } + return charlen; +} + +// Be aware: These unichar_is* functions are actually ASCII-only! bool unichar_isspace(unichar c) { return c < 128 && (attr[c] & FL_SPACE) != 0; } From 83865347db6eab7c168a21bdf17ba68beda59d06 Mon Sep 17 00:00:00 2001 From: Paul Sokolovsky Date: Fri, 13 Jun 2014 00:51:34 +0300 Subject: [PATCH 03/32] objstrunicode: Complete copy of objstr, to be patched for unicode support. --- py/objstrunicode.c | 1906 ++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 1906 insertions(+) create mode 100644 py/objstrunicode.c diff --git a/py/objstrunicode.c b/py/objstrunicode.c new file mode 100644 index 000000000..6656090c8 --- /dev/null +++ b/py/objstrunicode.c @@ -0,0 +1,1906 @@ +/* + * This file is part of the Micro Python project, http://micropython.org/ + * + * The MIT License (MIT) + * + * Copyright (c) 2013, 2014 Damien P. George + * Copyright (c) 2014 Paul Sokolovsky + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + * THE SOFTWARE. + */ + +#include +#include +#include + +#include "mpconfig.h" +#include "nlr.h" +#include "misc.h" +#include "qstr.h" +#include "obj.h" +#include "runtime0.h" +#include "runtime.h" +#include "pfenv.h" +#include "objstr.h" +#include "objlist.h" + +STATIC mp_obj_t str_modulo_format(mp_obj_t pattern, uint n_args, const mp_obj_t *args, mp_obj_t dict); +const mp_obj_t mp_const_empty_bytes; + +// use this macro to extract the string hash +#define GET_STR_HASH(str_obj_in, str_hash) uint str_hash; if (MP_OBJ_IS_QSTR(str_obj_in)) { str_hash = qstr_hash(MP_OBJ_QSTR_VALUE(str_obj_in)); } else { str_hash = ((mp_obj_str_t*)str_obj_in)->hash; } + +// use this macro to extract the string length +#define GET_STR_LEN(str_obj_in, str_len) uint str_len; if (MP_OBJ_IS_QSTR(str_obj_in)) { str_len = qstr_len(MP_OBJ_QSTR_VALUE(str_obj_in)); } else { str_len = ((mp_obj_str_t*)str_obj_in)->len; } + +// use this macro to extract the string data and length +#define GET_STR_DATA_LEN(str_obj_in, str_data, str_len) const byte *str_data; uint str_len; if (MP_OBJ_IS_QSTR(str_obj_in)) { str_data = qstr_data(MP_OBJ_QSTR_VALUE(str_obj_in), &str_len); } else { str_len = ((mp_obj_str_t*)str_obj_in)->len; str_data = ((mp_obj_str_t*)str_obj_in)->data; } + +STATIC mp_obj_t mp_obj_new_str_iterator(mp_obj_t str); +STATIC mp_obj_t mp_obj_new_bytes_iterator(mp_obj_t str); +STATIC NORETURN void bad_implicit_conversion(mp_obj_t self_in); +STATIC NORETURN void arg_type_mixup(); + +STATIC bool is_str_or_bytes(mp_obj_t o) { + return MP_OBJ_IS_STR(o) || MP_OBJ_IS_TYPE(o, &mp_type_bytes); +} + +/******************************************************************************/ +/* str */ + +void mp_str_print_quoted(void (*print)(void *env, const char *fmt, ...), void *env, const byte *str_data, uint str_len) { + // this escapes characters, but it will be very slow to print (calling print many times) + bool has_single_quote = false; + bool has_double_quote = false; + for (const byte *s = str_data, *top = str_data + str_len; !has_double_quote && s < top; s++) { + if (*s == '\'') { + has_single_quote = true; + } else if (*s == '"') { + has_double_quote = true; + } + } + int quote_char = '\''; + if (has_single_quote && !has_double_quote) { + quote_char = '"'; + } + print(env, "%c", quote_char); + for (const byte *s = str_data, *top = str_data + str_len; s < top; s++) { + if (*s == quote_char) { + print(env, "\\%c", quote_char); + } else if (*s == '\\') { + print(env, "\\\\"); + } else if (32 <= *s && *s <= 126) { + print(env, "%c", *s); + } else if (*s == '\n') { + print(env, "\\n"); + } else if (*s == '\r') { + print(env, "\\r"); + } else if (*s == '\t') { + print(env, "\\t"); + } else { + print(env, "\\x%02x", *s); + } + } + print(env, "%c", quote_char); +} + +STATIC void str_print(void (*print)(void *env, const char *fmt, ...), void *env, mp_obj_t self_in, mp_print_kind_t kind) { + GET_STR_DATA_LEN(self_in, str_data, str_len); + bool is_bytes = MP_OBJ_IS_TYPE(self_in, &mp_type_bytes); + if (kind == PRINT_STR && !is_bytes) { + print(env, "%.*s", str_len, str_data); + } else { + if (is_bytes) { + print(env, "b"); + } + mp_str_print_quoted(print, env, str_data, str_len); + } +} + +STATIC mp_obj_t str_make_new(mp_obj_t type_in, uint n_args, uint n_kw, const mp_obj_t *args) { +#if MICROPY_CPYTHON_COMPAT + if (n_kw != 0) { + mp_arg_error_unimpl_kw(); + } +#endif + + switch (n_args) { + case 0: + return MP_OBJ_NEW_QSTR(MP_QSTR_); + + case 1: + { + vstr_t *vstr = vstr_new(); + mp_obj_print_helper((void (*)(void*, const char*, ...))vstr_printf, vstr, args[0], PRINT_STR); + mp_obj_t s = mp_obj_new_str(vstr->buf, vstr->len, false); + vstr_free(vstr); + return s; + } + + case 2: + case 3: + { + // TODO: validate 2nd/3rd args + if (!MP_OBJ_IS_TYPE(args[0], &mp_type_bytes)) { + nlr_raise(mp_obj_new_exception_msg(&mp_type_TypeError, "bytes expected")); + } + GET_STR_DATA_LEN(args[0], str_data, str_len); + GET_STR_HASH(args[0], str_hash); + mp_obj_str_t *o = mp_obj_new_str_of_type(&mp_type_str, NULL, str_len); + o->data = str_data; + o->hash = str_hash; + return o; + } + + default: + nlr_raise(mp_obj_new_exception_msg(&mp_type_TypeError, "str takes at most 3 arguments")); + } +} + +STATIC mp_obj_t bytes_make_new(mp_obj_t type_in, uint n_args, uint n_kw, const mp_obj_t *args) { + if (n_args == 0) { + return mp_const_empty_bytes; + } + +#if MICROPY_CPYTHON_COMPAT + if (n_kw != 0) { + mp_arg_error_unimpl_kw(); + } +#endif + + if (MP_OBJ_IS_STR(args[0])) { + if (n_args < 2 || n_args > 3) { + goto wrong_args; + } + GET_STR_DATA_LEN(args[0], str_data, str_len); + GET_STR_HASH(args[0], str_hash); + mp_obj_str_t *o = mp_obj_new_str_of_type(&mp_type_bytes, NULL, str_len); + o->data = str_data; + o->hash = str_hash; + return o; + } + + if (n_args > 1) { + goto wrong_args; + } + + if (MP_OBJ_IS_SMALL_INT(args[0])) { + uint len = MP_OBJ_SMALL_INT_VALUE(args[0]); + byte *data; + + mp_obj_t o = mp_obj_str_builder_start(&mp_type_bytes, len, &data); + memset(data, 0, len); + return mp_obj_str_builder_end(o); + } + + int len; + byte *data; + vstr_t *vstr = NULL; + mp_obj_t o = NULL; + // Try to create array of exact len if initializer len is known + mp_obj_t len_in = mp_obj_len_maybe(args[0]); + if (len_in == MP_OBJ_NULL) { + len = -1; + vstr = vstr_new(); + } else { + len = MP_OBJ_SMALL_INT_VALUE(len_in); + o = mp_obj_str_builder_start(&mp_type_bytes, len, &data); + } + + mp_obj_t iterable = mp_getiter(args[0]); + mp_obj_t item; + while ((item = mp_iternext(iterable)) != MP_OBJ_STOP_ITERATION) { + if (len == -1) { + vstr_add_char(vstr, MP_OBJ_SMALL_INT_VALUE(item)); + } else { + *data++ = MP_OBJ_SMALL_INT_VALUE(item); + } + } + + if (len == -1) { + vstr_shrink(vstr); + // TODO: Optimize, borrow buffer from vstr + len = vstr_len(vstr); + o = mp_obj_str_builder_start(&mp_type_bytes, len, &data); + memcpy(data, vstr_str(vstr), len); + vstr_free(vstr); + } + + return mp_obj_str_builder_end(o); + +wrong_args: + nlr_raise(mp_obj_new_exception_msg(&mp_type_TypeError, "wrong number of arguments")); +} + +// like strstr but with specified length and allows \0 bytes +// TODO replace with something more efficient/standard +STATIC const byte *find_subbytes(const byte *haystack, machine_uint_t hlen, const byte *needle, machine_uint_t nlen, machine_int_t direction) { + if (hlen >= nlen) { + machine_uint_t str_index, str_index_end; + if (direction > 0) { + str_index = 0; + str_index_end = hlen - nlen; + } else { + str_index = hlen - nlen; + str_index_end = 0; + } + for (;;) { + if (memcmp(&haystack[str_index], needle, nlen) == 0) { + //found + return haystack + str_index; + } + if (str_index == str_index_end) { + //not found + break; + } + str_index += direction; + } + } + return NULL; +} + +STATIC mp_obj_t str_binary_op(int op, mp_obj_t lhs_in, mp_obj_t rhs_in) { + GET_STR_DATA_LEN(lhs_in, lhs_data, lhs_len); + mp_obj_type_t *lhs_type = mp_obj_get_type(lhs_in); + mp_obj_type_t *rhs_type = mp_obj_get_type(rhs_in); + switch (op) { + case MP_BINARY_OP_ADD: + case MP_BINARY_OP_INPLACE_ADD: + if (lhs_type == rhs_type) { + // add 2 strings or bytes + + GET_STR_DATA_LEN(rhs_in, rhs_data, rhs_len); + int alloc_len = lhs_len + rhs_len; + + /* code for making qstr + byte *q_ptr; + byte *val = qstr_build_start(alloc_len, &q_ptr); + memcpy(val, lhs_data, lhs_len); + memcpy(val + lhs_len, rhs_data, rhs_len); + return MP_OBJ_NEW_QSTR(qstr_build_end(q_ptr)); + */ + + // code for non-qstr + byte *data; + mp_obj_t s = mp_obj_str_builder_start(lhs_type, alloc_len, &data); + memcpy(data, lhs_data, lhs_len); + memcpy(data + lhs_len, rhs_data, rhs_len); + return mp_obj_str_builder_end(s); + } + break; + + case MP_BINARY_OP_IN: + /* NOTE `a in b` is `b.__contains__(a)` */ + if (lhs_type == rhs_type) { + GET_STR_DATA_LEN(rhs_in, rhs_data, rhs_len); + return MP_BOOL(find_subbytes(lhs_data, lhs_len, rhs_data, rhs_len, 1) != NULL); + } + break; + + case MP_BINARY_OP_MULTIPLY: { + if (!MP_OBJ_IS_SMALL_INT(rhs_in)) { + return MP_OBJ_NULL; // op not supported + } + int n = MP_OBJ_SMALL_INT_VALUE(rhs_in); + byte *data; + mp_obj_t s = mp_obj_str_builder_start(lhs_type, lhs_len * n, &data); + mp_seq_multiply(lhs_data, sizeof(*lhs_data), lhs_len, n, data); + return mp_obj_str_builder_end(s); + } + + case MP_BINARY_OP_MODULO: { + mp_obj_t *args; + uint n_args; + mp_obj_t dict = MP_OBJ_NULL; + if (MP_OBJ_IS_TYPE(rhs_in, &mp_type_tuple)) { + // TODO: Support tuple subclasses? + mp_obj_tuple_get(rhs_in, &n_args, &args); + } else if (MP_OBJ_IS_TYPE(rhs_in, &mp_type_dict)) { + args = NULL; + n_args = 0; + dict = rhs_in; + } else { + args = &rhs_in; + n_args = 1; + } + return str_modulo_format(lhs_in, n_args, args, dict); + } + + //case MP_BINARY_OP_NOT_EQUAL: // This is never passed here + case MP_BINARY_OP_EQUAL: // This will be passed only for bytes, str is dealt with in mp_obj_equal() + case MP_BINARY_OP_LESS: + case MP_BINARY_OP_LESS_EQUAL: + case MP_BINARY_OP_MORE: + case MP_BINARY_OP_MORE_EQUAL: + if (lhs_type == rhs_type) { + GET_STR_DATA_LEN(rhs_in, rhs_data, rhs_len); + return MP_BOOL(mp_seq_cmp_bytes(op, lhs_data, lhs_len, rhs_data, rhs_len)); + } + if (lhs_type == &mp_type_bytes) { + mp_buffer_info_t bufinfo; + if (!mp_get_buffer(rhs_in, &bufinfo, MP_BUFFER_READ)) { + goto uncomparable; + } + return MP_BOOL(mp_seq_cmp_bytes(op, lhs_data, lhs_len, bufinfo.buf, bufinfo.len)); + } +uncomparable: + if (op == MP_BINARY_OP_EQUAL) { + return mp_const_false; + } + } + + return MP_OBJ_NULL; // op not supported +} + +STATIC mp_obj_t str_subscr(mp_obj_t self_in, mp_obj_t index, mp_obj_t value) { + mp_obj_type_t *type = mp_obj_get_type(self_in); + GET_STR_DATA_LEN(self_in, self_data, self_len); + if (value == MP_OBJ_SENTINEL) { + // load +#if MICROPY_PY_BUILTINS_SLICE + if (MP_OBJ_IS_TYPE(index, &mp_type_slice)) { + mp_bound_slice_t slice; + if (!mp_seq_get_fast_slice_indexes(self_len, index, &slice)) { + nlr_raise(mp_obj_new_exception_msg(&mp_type_NotImplementedError, + "only slices with step=1 (aka None) are supported")); + } + return mp_obj_new_str_of_type(type, self_data + slice.start, slice.stop - slice.start); + } +#endif + uint index_val = mp_get_index(type, self_len, index, false); + if (type == &mp_type_bytes) { + return MP_OBJ_NEW_SMALL_INT((mp_small_int_t)self_data[index_val]); + } else { + return mp_obj_new_str((char*)self_data + index_val, 1, true); + } + } else { + return MP_OBJ_NULL; // op not supported + } +} + +STATIC mp_obj_t str_join(mp_obj_t self_in, mp_obj_t arg) { + assert(is_str_or_bytes(self_in)); + const mp_obj_type_t *self_type = mp_obj_get_type(self_in); + + // get separation string + GET_STR_DATA_LEN(self_in, sep_str, sep_len); + + // process args + uint seq_len; + mp_obj_t *seq_items; + if (MP_OBJ_IS_TYPE(arg, &mp_type_tuple)) { + mp_obj_tuple_get(arg, &seq_len, &seq_items); + } else { + if (!MP_OBJ_IS_TYPE(arg, &mp_type_list)) { + // arg is not a list, try to convert it to one + // TODO: Try to optimize? + arg = mp_type_list.make_new((mp_obj_t)&mp_type_list, 1, 0, &arg); + } + mp_obj_list_get(arg, &seq_len, &seq_items); + } + + // count required length + int required_len = 0; + for (int i = 0; i < seq_len; i++) { + if (mp_obj_get_type(seq_items[i]) != self_type) { + nlr_raise(mp_obj_new_exception_msg(&mp_type_TypeError, + "join expects a list of str/bytes objects consistent with self object")); + } + if (i > 0) { + required_len += sep_len; + } + GET_STR_LEN(seq_items[i], l); + required_len += l; + } + + // make joined string + byte *data; + mp_obj_t joined_str = mp_obj_str_builder_start(self_type, required_len, &data); + for (int i = 0; i < seq_len; i++) { + if (i > 0) { + memcpy(data, sep_str, sep_len); + data += sep_len; + } + GET_STR_DATA_LEN(seq_items[i], s, l); + memcpy(data, s, l); + data += l; + } + + // return joined string + return mp_obj_str_builder_end(joined_str); +} + +#define is_ws(c) ((c) == ' ' || (c) == '\t') + +STATIC mp_obj_t str_split(uint n_args, const mp_obj_t *args) { + const mp_obj_type_t *self_type = mp_obj_get_type(args[0]); + machine_int_t splits = -1; + mp_obj_t sep = mp_const_none; + if (n_args > 1) { + sep = args[1]; + if (n_args > 2) { + splits = mp_obj_get_int(args[2]); + } + } + + mp_obj_t res = mp_obj_new_list(0, NULL); + GET_STR_DATA_LEN(args[0], s, len); + const byte *top = s + len; + + if (sep == mp_const_none) { + // sep not given, so separate on whitespace + + // Initial whitespace is not counted as split, so we pre-do it + while (s < top && is_ws(*s)) s++; + while (s < top && splits != 0) { + const byte *start = s; + while (s < top && !is_ws(*s)) s++; + mp_obj_list_append(res, mp_obj_new_str_of_type(self_type, start, s - start)); + if (s >= top) { + break; + } + while (s < top && is_ws(*s)) s++; + if (splits > 0) { + splits--; + } + } + + if (s < top) { + mp_obj_list_append(res, mp_obj_new_str_of_type(self_type, s, top - s)); + } + + } else { + // sep given + + uint sep_len; + const char *sep_str = mp_obj_str_get_data(sep, &sep_len); + + if (sep_len == 0) { + nlr_raise(mp_obj_new_exception_msg(&mp_type_ValueError, "empty separator")); + } + + for (;;) { + const byte *start = s; + for (;;) { + if (splits == 0 || s + sep_len > top) { + s = top; + break; + } else if (memcmp(s, sep_str, sep_len) == 0) { + break; + } + s++; + } + mp_obj_list_append(res, mp_obj_new_str_of_type(self_type, start, s - start)); + if (s >= top) { + break; + } + s += sep_len; + if (splits > 0) { + splits--; + } + } + } + + return res; +} + +STATIC mp_obj_t str_rsplit(uint n_args, const mp_obj_t *args) { + if (n_args < 3) { + // If we don't have split limit, it doesn't matter from which side + // we split. + return str_split(n_args, args); + } + const mp_obj_type_t *self_type = mp_obj_get_type(args[0]); + mp_obj_t sep = args[1]; + GET_STR_DATA_LEN(args[0], s, len); + + machine_int_t splits = mp_obj_get_int(args[2]); + machine_int_t org_splits = splits; + // Preallocate list to the max expected # of elements, as we + // will fill it from the end. + mp_obj_list_t *res = mp_obj_new_list(splits + 1, NULL); + int idx = splits; + + if (sep == mp_const_none) { + assert(!"TODO: rsplit(None,n) not implemented"); + } else { + uint sep_len; + const char *sep_str = mp_obj_str_get_data(sep, &sep_len); + + if (sep_len == 0) { + nlr_raise(mp_obj_new_exception_msg(&mp_type_ValueError, "empty separator")); + } + + const byte *beg = s; + const byte *last = s + len; + for (;;) { + s = last - sep_len; + for (;;) { + if (splits == 0 || s < beg) { + break; + } else if (memcmp(s, sep_str, sep_len) == 0) { + break; + } + s--; + } + if (s < beg || splits == 0) { + res->items[idx] = mp_obj_new_str_of_type(self_type, beg, last - beg); + break; + } + res->items[idx--] = mp_obj_new_str_of_type(self_type, s + sep_len, last - s - sep_len); + last = s; + if (splits > 0) { + splits--; + } + } + if (idx != 0) { + // We split less parts than split limit, now go cleanup surplus + int used = org_splits + 1 - idx; + memcpy(res->items, &res->items[idx], used * sizeof(mp_obj_t)); + mp_seq_clear(res->items, used, res->alloc, sizeof(*res->items)); + res->len = used; + } + } + + return res; +} + + +STATIC mp_obj_t str_finder(uint n_args, const mp_obj_t *args, machine_int_t direction, bool is_index) { + assert(2 <= n_args && n_args <= 4); + assert(MP_OBJ_IS_STR(args[0])); + assert(MP_OBJ_IS_STR(args[1])); + + GET_STR_DATA_LEN(args[0], haystack, haystack_len); + GET_STR_DATA_LEN(args[1], needle, needle_len); + + machine_uint_t start = 0; + machine_uint_t end = haystack_len; + if (n_args >= 3 && args[2] != mp_const_none) { + start = mp_get_index(&mp_type_str, haystack_len, args[2], true); + } + if (n_args >= 4 && args[3] != mp_const_none) { + end = mp_get_index(&mp_type_str, haystack_len, args[3], true); + } + + const byte *p = find_subbytes(haystack + start, end - start, needle, needle_len, direction); + if (p == NULL) { + // not found + if (is_index) { + nlr_raise(mp_obj_new_exception_msg(&mp_type_ValueError, "substring not found")); + } else { + return MP_OBJ_NEW_SMALL_INT(-1); + } + } else { + // found + return MP_OBJ_NEW_SMALL_INT(p - haystack); + } +} + +STATIC mp_obj_t str_find(uint n_args, const mp_obj_t *args) { + return str_finder(n_args, args, 1, false); +} + +STATIC mp_obj_t str_rfind(uint n_args, const mp_obj_t *args) { + return str_finder(n_args, args, -1, false); +} + +STATIC mp_obj_t str_index(uint n_args, const mp_obj_t *args) { + return str_finder(n_args, args, 1, true); +} + +STATIC mp_obj_t str_rindex(uint n_args, const mp_obj_t *args) { + return str_finder(n_args, args, -1, true); +} + +// TODO: (Much) more variety in args +STATIC mp_obj_t str_startswith(uint n_args, const mp_obj_t *args) { + GET_STR_DATA_LEN(args[0], str, str_len); + GET_STR_DATA_LEN(args[1], prefix, prefix_len); + uint index_val = 0; + if (n_args > 2) { + index_val = mp_get_index(&mp_type_str, str_len, args[2], true); + } + if (prefix_len + index_val > str_len) { + return mp_const_false; + } + return MP_BOOL(memcmp(str + index_val, prefix, prefix_len) == 0); +} + +STATIC mp_obj_t str_endswith(uint n_args, const mp_obj_t *args) { + GET_STR_DATA_LEN(args[0], str, str_len); + GET_STR_DATA_LEN(args[1], suffix, suffix_len); + assert(n_args == 2); + + if (suffix_len > str_len) { + return mp_const_false; + } + return MP_BOOL(memcmp(str + (str_len - suffix_len), suffix, suffix_len) == 0); +} + +enum { LSTRIP, RSTRIP, STRIP }; + +STATIC mp_obj_t str_uni_strip(int type, uint n_args, const mp_obj_t *args) { + assert(1 <= n_args && n_args <= 2); + assert(is_str_or_bytes(args[0])); + const mp_obj_type_t *self_type = mp_obj_get_type(args[0]); + + const byte *chars_to_del; + uint chars_to_del_len; + static const byte whitespace[] = " \t\n\r\v\f"; + + if (n_args == 1) { + chars_to_del = whitespace; + chars_to_del_len = sizeof(whitespace); + } else { + if (mp_obj_get_type(args[1]) != self_type) { + arg_type_mixup(); + } + GET_STR_DATA_LEN(args[1], s, l); + chars_to_del = s; + chars_to_del_len = l; + } + + GET_STR_DATA_LEN(args[0], orig_str, orig_str_len); + + machine_uint_t first_good_char_pos = 0; + bool first_good_char_pos_set = false; + machine_uint_t last_good_char_pos = 0; + machine_uint_t i = 0; + machine_int_t delta = 1; + if (type == RSTRIP) { + i = orig_str_len - 1; + delta = -1; + } + for (machine_uint_t len = orig_str_len; len > 0; len--) { + if (find_subbytes(chars_to_del, chars_to_del_len, &orig_str[i], 1, 1) == NULL) { + if (!first_good_char_pos_set) { + first_good_char_pos_set = true; + first_good_char_pos = i; + if (type == LSTRIP) { + last_good_char_pos = orig_str_len - 1; + break; + } else if (type == RSTRIP) { + first_good_char_pos = 0; + last_good_char_pos = i; + break; + } + } + last_good_char_pos = i; + } + i += delta; + } + + if (!first_good_char_pos_set) { + // string is all whitespace, return '' + return MP_OBJ_NEW_QSTR(MP_QSTR_); + } + + assert(last_good_char_pos >= first_good_char_pos); + //+1 to accomodate the last character + machine_uint_t stripped_len = last_good_char_pos - first_good_char_pos + 1; + if (stripped_len == orig_str_len) { + // If nothing was stripped, don't bother to dup original string + // TODO: watch out for this case when we'll get to bytearray.strip() + assert(first_good_char_pos == 0); + return args[0]; + } + return mp_obj_new_str_of_type(self_type, orig_str + first_good_char_pos, stripped_len); +} + +STATIC mp_obj_t str_strip(uint n_args, const mp_obj_t *args) { + return str_uni_strip(STRIP, n_args, args); +} + +STATIC mp_obj_t str_lstrip(uint n_args, const mp_obj_t *args) { + return str_uni_strip(LSTRIP, n_args, args); +} + +STATIC mp_obj_t str_rstrip(uint n_args, const mp_obj_t *args) { + return str_uni_strip(RSTRIP, n_args, args); +} + +// Takes an int arg, but only parses unsigned numbers, and only changes +// *num if at least one digit was parsed. +static int str_to_int(const char *str, int *num) { + const char *s = str; + if (unichar_isdigit(*s)) { + *num = 0; + do { + *num = *num * 10 + (*s - '0'); + s++; + } + while (unichar_isdigit(*s)); + } + return s - str; +} + +static bool isalignment(char ch) { + return ch && strchr("<>=^", ch) != NULL; +} + +static bool istype(char ch) { + return ch && strchr("bcdeEfFgGnosxX%", ch) != NULL; +} + +static bool arg_looks_integer(mp_obj_t arg) { + return MP_OBJ_IS_TYPE(arg, &mp_type_bool) || MP_OBJ_IS_INT(arg); +} + +static bool arg_looks_numeric(mp_obj_t arg) { + return arg_looks_integer(arg) +#if MICROPY_PY_BUILTINS_FLOAT + || MP_OBJ_IS_TYPE(arg, &mp_type_float) +#endif + ; +} + +static mp_obj_t arg_as_int(mp_obj_t arg) { +#if MICROPY_PY_BUILTINS_FLOAT + if (MP_OBJ_IS_TYPE(arg, &mp_type_float)) { + + // TODO: Needs a way to construct an mpz integer from a float + + mp_small_int_t num = mp_obj_get_float(arg); + return MP_OBJ_NEW_SMALL_INT(num); + } +#endif + return arg; +} + +mp_obj_t mp_obj_str_format(uint n_args, const mp_obj_t *args) { + assert(MP_OBJ_IS_STR(args[0])); + + GET_STR_DATA_LEN(args[0], str, len); + int arg_i = 0; + vstr_t *vstr = vstr_new(); + pfenv_t pfenv_vstr; + pfenv_vstr.data = vstr; + pfenv_vstr.print_strn = pfenv_vstr_add_strn; + + for (const byte *top = str + len; str < top; str++) { + if (*str == '}') { + str++; + if (str < top && *str == '}') { + vstr_add_char(vstr, '}'); + continue; + } + nlr_raise(mp_obj_new_exception_msg(&mp_type_ValueError, "single '}' encountered in format string")); + } + if (*str != '{') { + vstr_add_char(vstr, *str); + continue; + } + + str++; + if (str < top && *str == '{') { + vstr_add_char(vstr, '{'); + continue; + } + + // replacement_field ::= "{" [field_name] ["!" conversion] [":" format_spec] "}" + + vstr_t *field_name = NULL; + char conversion = '\0'; + vstr_t *format_spec = NULL; + + if (str < top && *str != '}' && *str != '!' && *str != ':') { + field_name = vstr_new(); + while (str < top && *str != '}' && *str != '!' && *str != ':') { + vstr_add_char(field_name, *str++); + } + vstr_add_char(field_name, '\0'); + } + + // conversion ::= "r" | "s" + + if (str < top && *str == '!') { + str++; + if (str < top && (*str == 'r' || *str == 's')) { + conversion = *str++; + } else { + nlr_raise(mp_obj_new_exception_msg(&mp_type_ValueError, "end of format while looking for conversion specifier")); + } + } + + if (str < top && *str == ':') { + str++; + // {:} is the same as {}, which is the same as {!s} + // This makes a difference when passing in a True or False + // '{}'.format(True) returns 'True' + // '{:d}'.format(True) returns '1' + // So we treat {:} as {} and this later gets treated to be {!s} + if (*str != '}') { + format_spec = vstr_new(); + while (str < top && *str != '}') { + vstr_add_char(format_spec, *str++); + } + vstr_add_char(format_spec, '\0'); + } + } + if (str >= top) { + nlr_raise(mp_obj_new_exception_msg(&mp_type_ValueError, "unmatched '{' in format")); + } + if (*str != '}') { + nlr_raise(mp_obj_new_exception_msg(&mp_type_ValueError, "expected ':' after format specifier")); + } + + mp_obj_t arg = mp_const_none; + + if (field_name) { + if (arg_i > 0) { + nlr_raise(mp_obj_new_exception_msg(&mp_type_ValueError, "can't switch from automatic field numbering to manual field specification")); + } + int index = 0; + if (str_to_int(vstr_str(field_name), &index) != vstr_len(field_name) - 1) { + nlr_raise(mp_obj_new_exception_msg(&mp_type_KeyError, "attributes not supported yet")); + } + if (index >= n_args - 1) { + nlr_raise(mp_obj_new_exception_msg(&mp_type_IndexError, "tuple index out of range")); + } + arg = args[index + 1]; + arg_i = -1; + vstr_free(field_name); + field_name = NULL; + } else { + if (arg_i < 0) { + nlr_raise(mp_obj_new_exception_msg(&mp_type_ValueError, "can't switch from manual field specification to automatic field numbering")); + } + if (arg_i >= n_args - 1) { + nlr_raise(mp_obj_new_exception_msg(&mp_type_IndexError, "tuple index out of range")); + } + arg = args[arg_i + 1]; + arg_i++; + } + if (!format_spec && !conversion) { + conversion = 's'; + } + if (conversion) { + mp_print_kind_t print_kind; + if (conversion == 's') { + print_kind = PRINT_STR; + } else if (conversion == 'r') { + print_kind = PRINT_REPR; + } else { + nlr_raise(mp_obj_new_exception_msg_varg(&mp_type_ValueError, "unknown conversion specifier %c", conversion)); + } + vstr_t *arg_vstr = vstr_new(); + mp_obj_print_helper((void (*)(void*, const char*, ...))vstr_printf, arg_vstr, arg, print_kind); + arg = mp_obj_new_str(vstr_str(arg_vstr), vstr_len(arg_vstr), false); + vstr_free(arg_vstr); + } + + char sign = '\0'; + char fill = '\0'; + char align = '\0'; + int width = -1; + int precision = -1; + char type = '\0'; + int flags = 0; + + if (format_spec) { + // The format specifier (from http://docs.python.org/2/library/string.html#formatspec) + // + // [[fill]align][sign][#][0][width][,][.precision][type] + // fill ::= + // align ::= "<" | ">" | "=" | "^" + // sign ::= "+" | "-" | " " + // width ::= integer + // precision ::= integer + // type ::= "b" | "c" | "d" | "e" | "E" | "f" | "F" | "g" | "G" | "n" | "o" | "s" | "x" | "X" | "%" + + const char *s = vstr_str(format_spec); + if (isalignment(*s)) { + align = *s++; + } else if (*s && isalignment(s[1])) { + fill = *s++; + align = *s++; + } + if (*s == '+' || *s == '-' || *s == ' ') { + if (*s == '+') { + flags |= PF_FLAG_SHOW_SIGN; + } else if (*s == ' ') { + flags |= PF_FLAG_SPACE_SIGN; + } + sign = *s++; + } + if (*s == '#') { + flags |= PF_FLAG_SHOW_PREFIX; + s++; + } + if (*s == '0') { + if (!align) { + align = '='; + } + if (!fill) { + fill = '0'; + } + } + s += str_to_int(s, &width); + if (*s == ',') { + flags |= PF_FLAG_SHOW_COMMA; + s++; + } + if (*s == '.') { + s++; + s += str_to_int(s, &precision); + } + if (istype(*s)) { + type = *s++; + } + if (*s) { + nlr_raise(mp_obj_new_exception_msg(&mp_type_KeyError, "Invalid conversion specification")); + } + vstr_free(format_spec); + format_spec = NULL; + } + if (!align) { + if (arg_looks_numeric(arg)) { + align = '>'; + } else { + align = '<'; + } + } + if (!fill) { + fill = ' '; + } + + if (sign) { + if (type == 's') { + nlr_raise(mp_obj_new_exception_msg(&mp_type_ValueError, "Sign not allowed in string format specifier")); + } + if (type == 'c') { + nlr_raise(mp_obj_new_exception_msg(&mp_type_ValueError, "Sign not allowed with integer format specifier 'c'")); + } + } else { + sign = '-'; + } + + switch (align) { + case '<': flags |= PF_FLAG_LEFT_ADJUST; break; + case '=': flags |= PF_FLAG_PAD_AFTER_SIGN; break; + case '^': flags |= PF_FLAG_CENTER_ADJUST; break; + } + + if (arg_looks_integer(arg)) { + switch (type) { + case 'b': + pfenv_print_mp_int(&pfenv_vstr, arg, 1, 2, 'a', flags, fill, width, 0); + continue; + + case 'c': + { + char ch = mp_obj_get_int(arg); + pfenv_print_strn(&pfenv_vstr, &ch, 1, flags, fill, width); + continue; + } + + case '\0': // No explicit format type implies 'd' + case 'n': // I don't think we support locales in uPy so use 'd' + case 'd': + pfenv_print_mp_int(&pfenv_vstr, arg, 1, 10, 'a', flags, fill, width, 0); + continue; + + case 'o': + if (flags & PF_FLAG_SHOW_PREFIX) { + flags |= PF_FLAG_SHOW_OCTAL_LETTER; + } + + pfenv_print_mp_int(&pfenv_vstr, arg, 1, 8, 'a', flags, fill, width, 0); + continue; + + case 'X': + case 'x': + pfenv_print_mp_int(&pfenv_vstr, arg, 1, 16, type - ('X' - 'A'), flags, fill, width, 0); + continue; + + case 'e': + case 'E': + case 'f': + case 'F': + case 'g': + case 'G': + case '%': + // The floating point formatters all work with anything that + // looks like an integer + break; + + default: + nlr_raise(mp_obj_new_exception_msg_varg(&mp_type_ValueError, + "unknown format code '%c' for object of type '%s'", type, mp_obj_get_type_str(arg))); + } + } + + // NOTE: no else here. We need the e, f, g etc formats for integer + // arguments (from above if) to take this if. + if (arg_looks_numeric(arg)) { + if (!type) { + + // Even though the docs say that an unspecified type is the same + // as 'g', there is one subtle difference, when the exponent + // is one less than the precision. + // + // '{:10.1}'.format(0.0) ==> '0e+00' + // '{:10.1g}'.format(0.0) ==> '0' + // + // TODO: Figure out how to deal with this. + // + // A proper solution would involve adding a special flag + // or something to format_float, and create a format_double + // to deal with doubles. In order to fix this when using + // sprintf, we'd need to use the e format and tweak the + // returned result to strip trailing zeros like the g format + // does. + // + // {:10.3} and {:10.2e} with 1.23e2 both produce 1.23e+02 + // but with 1.e2 you get 1e+02 and 1.00e+02 + // + // Stripping the trailing 0's (like g) does would make the + // e format give us the right format. + // + // CPython sources say: + // Omitted type specifier. Behaves in the same way as repr(x) + // and str(x) if no precision is given, else like 'g', but with + // at least one digit after the decimal point. */ + + type = 'g'; + } + if (type == 'n') { + type = 'g'; + } + + flags |= PF_FLAG_PAD_NAN_INF; // '{:06e}'.format(float('-inf')) should give '-00inf' + switch (type) { +#if MICROPY_PY_BUILTINS_FLOAT + case 'e': + case 'E': + case 'f': + case 'F': + case 'g': + case 'G': + pfenv_print_float(&pfenv_vstr, mp_obj_get_float(arg), type, flags, fill, width, precision); + break; + + case '%': + flags |= PF_FLAG_ADD_PERCENT; + pfenv_print_float(&pfenv_vstr, mp_obj_get_float(arg) * 100.0F, 'f', flags, fill, width, precision); + break; +#endif + + default: + nlr_raise(mp_obj_new_exception_msg_varg(&mp_type_ValueError, + "unknown format code '%c' for object of type 'float'", + type, mp_obj_get_type_str(arg))); + } + } else { + // arg doesn't look like a number + + if (align == '=') { + nlr_raise(mp_obj_new_exception_msg(&mp_type_ValueError, "'=' alignment not allowed in string format specifier")); + } + + switch (type) { + case '\0': + mp_obj_print_helper((void (*)(void*, const char*, ...))vstr_printf, vstr, arg, PRINT_STR); + break; + + case 's': + { + uint len; + const char *s = mp_obj_str_get_data(arg, &len); + if (precision < 0) { + precision = len; + } + if (len > precision) { + len = precision; + } + pfenv_print_strn(&pfenv_vstr, s, len, flags, fill, width); + break; + } + + default: + nlr_raise(mp_obj_new_exception_msg_varg(&mp_type_ValueError, + "unknown format code '%c' for object of type 'str'", + type, mp_obj_get_type_str(arg))); + } + } + } + + mp_obj_t s = mp_obj_new_str(vstr->buf, vstr->len, false); + vstr_free(vstr); + return s; +} + +STATIC mp_obj_t str_modulo_format(mp_obj_t pattern, uint n_args, const mp_obj_t *args, mp_obj_t dict) { + assert(MP_OBJ_IS_STR(pattern)); + + GET_STR_DATA_LEN(pattern, str, len); + const byte *start_str = str; + int arg_i = 0; + vstr_t *vstr = vstr_new(); + pfenv_t pfenv_vstr; + pfenv_vstr.data = vstr; + pfenv_vstr.print_strn = pfenv_vstr_add_strn; + + for (const byte *top = str + len; str < top; str++) { + mp_obj_t arg = MP_OBJ_NULL; + if (*str != '%') { + vstr_add_char(vstr, *str); + continue; + } + if (++str >= top) { + break; + } + if (*str == '%') { + vstr_add_char(vstr, '%'); + continue; + } + + // Dictionary value lookup + if (*str == '(') { + const byte *key = ++str; + while (*str != ')') { + if (str >= top) { + nlr_raise(mp_obj_new_exception_msg(&mp_type_ValueError, "incomplete format key")); + } + ++str; + } + mp_obj_t k_obj = mp_obj_new_str((const char*)key, str - key, true); + arg = mp_obj_dict_get(dict, k_obj); + str++; + } + + int flags = 0; + char fill = ' '; + int alt = 0; + while (str < top) { + if (*str == '-') flags |= PF_FLAG_LEFT_ADJUST; + else if (*str == '+') flags |= PF_FLAG_SHOW_SIGN; + else if (*str == ' ') flags |= PF_FLAG_SPACE_SIGN; + else if (*str == '#') alt = PF_FLAG_SHOW_PREFIX; + else if (*str == '0') { + flags |= PF_FLAG_PAD_AFTER_SIGN; + fill = '0'; + } else break; + str++; + } + // parse width, if it exists + int width = 0; + if (str < top) { + if (*str == '*') { + if (arg_i >= n_args) { + goto not_enough_args; + } + width = mp_obj_get_int(args[arg_i++]); + str++; + } else { + for (; str < top && '0' <= *str && *str <= '9'; str++) { + width = width * 10 + *str - '0'; + } + } + } + int prec = -1; + if (str < top && *str == '.') { + if (++str < top) { + if (*str == '*') { + if (arg_i >= n_args) { + goto not_enough_args; + } + prec = mp_obj_get_int(args[arg_i++]); + str++; + } else { + prec = 0; + for (; str < top && '0' <= *str && *str <= '9'; str++) { + prec = prec * 10 + *str - '0'; + } + } + } + } + + if (str >= top) { + nlr_raise(mp_obj_new_exception_msg(&mp_type_ValueError, "incomplete format")); + } + + // Tuple value lookup + if (arg == MP_OBJ_NULL) { + if (arg_i >= n_args) { +not_enough_args: + nlr_raise(mp_obj_new_exception_msg(&mp_type_TypeError, "not enough arguments for format string")); + } + arg = args[arg_i++]; + } + switch (*str) { + case 'c': + if (MP_OBJ_IS_STR(arg)) { + uint len; + const char *s = mp_obj_str_get_data(arg, &len); + if (len != 1) { + nlr_raise(mp_obj_new_exception_msg(&mp_type_TypeError, "%%c requires int or char")); + break; + } + pfenv_print_strn(&pfenv_vstr, s, 1, flags, ' ', width); + break; + } + if (arg_looks_integer(arg)) { + char ch = mp_obj_get_int(arg); + pfenv_print_strn(&pfenv_vstr, &ch, 1, flags, ' ', width); + break; + } +#if MICROPY_PY_BUILTINS_FLOAT + // This is what CPython reports, so we report the same. + if (MP_OBJ_IS_TYPE(arg, &mp_type_float)) { + nlr_raise(mp_obj_new_exception_msg(&mp_type_TypeError, "integer argument expected, got float")); + + } +#endif + nlr_raise(mp_obj_new_exception_msg(&mp_type_TypeError, "an integer is required")); + break; + + case 'd': + case 'i': + case 'u': + pfenv_print_mp_int(&pfenv_vstr, arg_as_int(arg), 1, 10, 'a', flags, fill, width, prec); + break; + +#if MICROPY_PY_BUILTINS_FLOAT + case 'e': + case 'E': + case 'f': + case 'F': + case 'g': + case 'G': + pfenv_print_float(&pfenv_vstr, mp_obj_get_float(arg), *str, flags, fill, width, prec); + break; +#endif + + case 'o': + if (alt) { + flags |= (PF_FLAG_SHOW_PREFIX | PF_FLAG_SHOW_OCTAL_LETTER); + } + pfenv_print_mp_int(&pfenv_vstr, arg, 1, 8, 'a', flags, fill, width, prec); + break; + + case 'r': + case 's': + { + vstr_t *arg_vstr = vstr_new(); + mp_obj_print_helper((void (*)(void*, const char*, ...))vstr_printf, + arg_vstr, arg, *str == 'r' ? PRINT_REPR : PRINT_STR); + uint len = vstr_len(arg_vstr); + if (prec < 0) { + prec = len; + } + if (len > prec) { + len = prec; + } + pfenv_print_strn(&pfenv_vstr, vstr_str(arg_vstr), len, flags, ' ', width); + vstr_free(arg_vstr); + break; + } + + case 'X': + case 'x': + pfenv_print_mp_int(&pfenv_vstr, arg, 1, 16, *str - ('X' - 'A'), flags | alt, fill, width, prec); + break; + + default: + nlr_raise(mp_obj_new_exception_msg_varg(&mp_type_ValueError, + "unsupported format character '%c' (0x%x) at index %d", + *str, *str, str - start_str)); + } + } + + if (arg_i != n_args) { + nlr_raise(mp_obj_new_exception_msg(&mp_type_TypeError, "not all arguments converted during string formatting")); + } + + mp_obj_t s = mp_obj_new_str(vstr->buf, vstr->len, false); + vstr_free(vstr); + return s; +} + +STATIC mp_obj_t str_replace(uint n_args, const mp_obj_t *args) { + assert(MP_OBJ_IS_STR(args[0])); + + machine_int_t max_rep = -1; + if (n_args == 4) { + max_rep = mp_obj_get_int(args[3]); + if (max_rep == 0) { + return args[0]; + } else if (max_rep < 0) { + max_rep = -1; + } + } + + // if max_rep is still -1 by this point we will need to do all possible replacements + + // check argument types + + if (!MP_OBJ_IS_STR(args[1])) { + bad_implicit_conversion(args[1]); + } + + if (!MP_OBJ_IS_STR(args[2])) { + bad_implicit_conversion(args[2]); + } + + // extract string data + + GET_STR_DATA_LEN(args[0], str, str_len); + GET_STR_DATA_LEN(args[1], old, old_len); + GET_STR_DATA_LEN(args[2], new, new_len); + + // old won't exist in str if it's longer, so nothing to replace + if (old_len > str_len) { + return args[0]; + } + + // data for the replaced string + byte *data = NULL; + mp_obj_t replaced_str = MP_OBJ_NULL; + + // do 2 passes over the string: + // first pass computes the required length of the replaced string + // second pass does the replacements + for (;;) { + machine_uint_t replaced_str_index = 0; + machine_uint_t num_replacements_done = 0; + const byte *old_occurrence; + const byte *offset_ptr = str; + machine_uint_t str_len_remain = str_len; + if (old_len == 0) { + // if old_str is empty, copy new_str to start of replaced string + // copy the replacement string + if (data != NULL) { + memcpy(data, new, new_len); + } + replaced_str_index += new_len; + num_replacements_done++; + } + while (num_replacements_done != max_rep && str_len_remain > 0 && (old_occurrence = find_subbytes(offset_ptr, str_len_remain, old, old_len, 1)) != NULL) { + if (old_len == 0) { + old_occurrence += 1; + } + // copy from just after end of last occurrence of to-be-replaced string to right before start of next occurrence + if (data != NULL) { + memcpy(data + replaced_str_index, offset_ptr, old_occurrence - offset_ptr); + } + replaced_str_index += old_occurrence - offset_ptr; + // copy the replacement string + if (data != NULL) { + memcpy(data + replaced_str_index, new, new_len); + } + replaced_str_index += new_len; + offset_ptr = old_occurrence + old_len; + str_len_remain = str + str_len - offset_ptr; + num_replacements_done++; + } + + // copy from just after end of last occurrence of to-be-replaced string to end of old string + if (data != NULL) { + memcpy(data + replaced_str_index, offset_ptr, str_len_remain); + } + replaced_str_index += str_len_remain; + + if (data == NULL) { + // first pass + if (num_replacements_done == 0) { + // no substr found, return original string + return args[0]; + } else { + // substr found, allocate new string + replaced_str = mp_obj_str_builder_start(mp_obj_get_type(args[0]), replaced_str_index, &data); + assert(data != NULL); + } + } else { + // second pass, we are done + break; + } + } + + return mp_obj_str_builder_end(replaced_str); +} + +STATIC mp_obj_t str_count(uint n_args, const mp_obj_t *args) { + assert(2 <= n_args && n_args <= 4); + assert(MP_OBJ_IS_STR(args[0])); + assert(MP_OBJ_IS_STR(args[1])); + + GET_STR_DATA_LEN(args[0], haystack, haystack_len); + GET_STR_DATA_LEN(args[1], needle, needle_len); + + machine_uint_t start = 0; + machine_uint_t end = haystack_len; + if (n_args >= 3 && args[2] != mp_const_none) { + start = mp_get_index(&mp_type_str, haystack_len, args[2], true); + } + if (n_args >= 4 && args[3] != mp_const_none) { + end = mp_get_index(&mp_type_str, haystack_len, args[3], true); + } + + // if needle_len is zero then we count each gap between characters as an occurrence + if (needle_len == 0) { + return MP_OBJ_NEW_SMALL_INT(end - start + 1); + } + + // count the occurrences + machine_int_t num_occurrences = 0; + for (machine_uint_t haystack_index = start; haystack_index + needle_len <= end; haystack_index++) { + if (memcmp(&haystack[haystack_index], needle, needle_len) == 0) { + num_occurrences++; + haystack_index += needle_len - 1; + } + } + + return MP_OBJ_NEW_SMALL_INT(num_occurrences); +} + +STATIC mp_obj_t str_partitioner(mp_obj_t self_in, mp_obj_t arg, machine_int_t direction) { + if (!is_str_or_bytes(self_in)) { + assert(0); + } + mp_obj_type_t *self_type = mp_obj_get_type(self_in); + if (self_type != mp_obj_get_type(arg)) { + arg_type_mixup(); + } + + GET_STR_DATA_LEN(self_in, str, str_len); + GET_STR_DATA_LEN(arg, sep, sep_len); + + if (sep_len == 0) { + nlr_raise(mp_obj_new_exception_msg(&mp_type_ValueError, "empty separator")); + } + + mp_obj_t result[] = {MP_OBJ_NEW_QSTR(MP_QSTR_), MP_OBJ_NEW_QSTR(MP_QSTR_), MP_OBJ_NEW_QSTR(MP_QSTR_)}; + + if (direction > 0) { + result[0] = self_in; + } else { + result[2] = self_in; + } + + const byte *position_ptr = find_subbytes(str, str_len, sep, sep_len, direction); + if (position_ptr != NULL) { + machine_uint_t position = position_ptr - str; + result[0] = mp_obj_new_str_of_type(self_type, str, position); + result[1] = arg; + result[2] = mp_obj_new_str_of_type(self_type, str + position + sep_len, str_len - position - sep_len); + } + + return mp_obj_new_tuple(3, result); +} + +STATIC mp_obj_t str_partition(mp_obj_t self_in, mp_obj_t arg) { + return str_partitioner(self_in, arg, 1); +} + +STATIC mp_obj_t str_rpartition(mp_obj_t self_in, mp_obj_t arg) { + return str_partitioner(self_in, arg, -1); +} + +// Supposedly not too critical operations, so optimize for code size +STATIC mp_obj_t str_caseconv(unichar (*op)(unichar), mp_obj_t self_in) { + GET_STR_DATA_LEN(self_in, self_data, self_len); + byte *data; + mp_obj_t s = mp_obj_str_builder_start(mp_obj_get_type(self_in), self_len, &data); + for (int i = 0; i < self_len; i++) { + *data++ = op(*self_data++); + } + *data = 0; + return mp_obj_str_builder_end(s); +} + +STATIC mp_obj_t str_lower(mp_obj_t self_in) { + return str_caseconv(unichar_tolower, self_in); +} + +STATIC mp_obj_t str_upper(mp_obj_t self_in) { + return str_caseconv(unichar_toupper, self_in); +} + +STATIC mp_obj_t str_uni_istype(bool (*f)(unichar), mp_obj_t self_in) { + GET_STR_DATA_LEN(self_in, self_data, self_len); + + if (self_len == 0) { + return mp_const_false; // default to False for empty str + } + + if (f != unichar_isupper && f != unichar_islower) { + for (int i = 0; i < self_len; i++) { + if (!f(*self_data++)) { + return mp_const_false; + } + } + } else { + bool contains_alpha = false; + + for (int i = 0; i < self_len; i++) { // only check alphanumeric characters + if (unichar_isalpha(*self_data++)) { + contains_alpha = true; + if (!f(*(self_data - 1))) { // -1 because we already incremented above + return mp_const_false; + } + } + } + + if (!contains_alpha) { + return mp_const_false; + } + } + + return mp_const_true; +} + +STATIC mp_obj_t str_isspace(mp_obj_t self_in) { + return str_uni_istype(unichar_isspace, self_in); +} + +STATIC mp_obj_t str_isalpha(mp_obj_t self_in) { + return str_uni_istype(unichar_isalpha, self_in); +} + +STATIC mp_obj_t str_isdigit(mp_obj_t self_in) { + return str_uni_istype(unichar_isdigit, self_in); +} + +STATIC mp_obj_t str_isupper(mp_obj_t self_in) { + return str_uni_istype(unichar_isupper, self_in); +} + +STATIC mp_obj_t str_islower(mp_obj_t self_in) { + return str_uni_istype(unichar_islower, self_in); +} + +#if MICROPY_CPYTHON_COMPAT +// These methods are superfluous in the presense of str() and bytes() +// constructors. +// TODO: should accept kwargs too +STATIC mp_obj_t bytes_decode(uint n_args, const mp_obj_t *args) { + mp_obj_t new_args[2]; + if (n_args == 1) { + new_args[0] = args[0]; + new_args[1] = MP_OBJ_NEW_QSTR(MP_QSTR_utf_hyphen_8); + args = new_args; + n_args++; + } + return str_make_new(NULL, n_args, 0, args); +} + +// TODO: should accept kwargs too +STATIC mp_obj_t str_encode(uint n_args, const mp_obj_t *args) { + mp_obj_t new_args[2]; + if (n_args == 1) { + new_args[0] = args[0]; + new_args[1] = MP_OBJ_NEW_QSTR(MP_QSTR_utf_hyphen_8); + args = new_args; + n_args++; + } + return bytes_make_new(NULL, n_args, 0, args); +} +#endif + +STATIC machine_int_t str_get_buffer(mp_obj_t self_in, mp_buffer_info_t *bufinfo, int flags) { + if (flags == MP_BUFFER_READ) { + GET_STR_DATA_LEN(self_in, str_data, str_len); + bufinfo->buf = (void*)str_data; + bufinfo->len = str_len; + bufinfo->typecode = 'b'; + return 0; + } else { + // can't write to a string + bufinfo->buf = NULL; + bufinfo->len = 0; + bufinfo->typecode = -1; + return 1; + } +} + +#if MICROPY_CPYTHON_COMPAT +STATIC MP_DEFINE_CONST_FUN_OBJ_VAR_BETWEEN(bytes_decode_obj, 1, 3, bytes_decode); +STATIC MP_DEFINE_CONST_FUN_OBJ_VAR_BETWEEN(str_encode_obj, 1, 3, str_encode); +#endif +STATIC MP_DEFINE_CONST_FUN_OBJ_VAR_BETWEEN(str_find_obj, 2, 4, str_find); +STATIC MP_DEFINE_CONST_FUN_OBJ_VAR_BETWEEN(str_rfind_obj, 2, 4, str_rfind); +STATIC MP_DEFINE_CONST_FUN_OBJ_VAR_BETWEEN(str_index_obj, 2, 4, str_index); +STATIC MP_DEFINE_CONST_FUN_OBJ_VAR_BETWEEN(str_rindex_obj, 2, 4, str_rindex); +STATIC MP_DEFINE_CONST_FUN_OBJ_2(str_join_obj, str_join); +STATIC MP_DEFINE_CONST_FUN_OBJ_VAR_BETWEEN(str_split_obj, 1, 3, str_split); +STATIC MP_DEFINE_CONST_FUN_OBJ_VAR_BETWEEN(str_rsplit_obj, 1, 3, str_rsplit); +STATIC MP_DEFINE_CONST_FUN_OBJ_VAR_BETWEEN(str_startswith_obj, 2, 3, str_startswith); +STATIC MP_DEFINE_CONST_FUN_OBJ_VAR_BETWEEN(str_endswith_obj, 2, 3, str_endswith); +STATIC MP_DEFINE_CONST_FUN_OBJ_VAR_BETWEEN(str_strip_obj, 1, 2, str_strip); +STATIC MP_DEFINE_CONST_FUN_OBJ_VAR_BETWEEN(str_lstrip_obj, 1, 2, str_lstrip); +STATIC MP_DEFINE_CONST_FUN_OBJ_VAR_BETWEEN(str_rstrip_obj, 1, 2, str_rstrip); +STATIC MP_DEFINE_CONST_FUN_OBJ_VAR(str_format_obj, 1, mp_obj_str_format); +STATIC MP_DEFINE_CONST_FUN_OBJ_VAR_BETWEEN(str_replace_obj, 3, 4, str_replace); +STATIC MP_DEFINE_CONST_FUN_OBJ_VAR_BETWEEN(str_count_obj, 2, 4, str_count); +STATIC MP_DEFINE_CONST_FUN_OBJ_2(str_partition_obj, str_partition); +STATIC MP_DEFINE_CONST_FUN_OBJ_2(str_rpartition_obj, str_rpartition); +STATIC MP_DEFINE_CONST_FUN_OBJ_1(str_lower_obj, str_lower); +STATIC MP_DEFINE_CONST_FUN_OBJ_1(str_upper_obj, str_upper); +STATIC MP_DEFINE_CONST_FUN_OBJ_1(str_isspace_obj, str_isspace); +STATIC MP_DEFINE_CONST_FUN_OBJ_1(str_isalpha_obj, str_isalpha); +STATIC MP_DEFINE_CONST_FUN_OBJ_1(str_isdigit_obj, str_isdigit); +STATIC MP_DEFINE_CONST_FUN_OBJ_1(str_isupper_obj, str_isupper); +STATIC MP_DEFINE_CONST_FUN_OBJ_1(str_islower_obj, str_islower); + +STATIC const mp_map_elem_t str_locals_dict_table[] = { +#if MICROPY_CPYTHON_COMPAT + { MP_OBJ_NEW_QSTR(MP_QSTR_decode), (mp_obj_t)&bytes_decode_obj }, + { MP_OBJ_NEW_QSTR(MP_QSTR_encode), (mp_obj_t)&str_encode_obj }, +#endif + { MP_OBJ_NEW_QSTR(MP_QSTR_find), (mp_obj_t)&str_find_obj }, + { MP_OBJ_NEW_QSTR(MP_QSTR_rfind), (mp_obj_t)&str_rfind_obj }, + { MP_OBJ_NEW_QSTR(MP_QSTR_index), (mp_obj_t)&str_index_obj }, + { MP_OBJ_NEW_QSTR(MP_QSTR_rindex), (mp_obj_t)&str_rindex_obj }, + { MP_OBJ_NEW_QSTR(MP_QSTR_join), (mp_obj_t)&str_join_obj }, + { MP_OBJ_NEW_QSTR(MP_QSTR_split), (mp_obj_t)&str_split_obj }, + { MP_OBJ_NEW_QSTR(MP_QSTR_rsplit), (mp_obj_t)&str_rsplit_obj }, + { MP_OBJ_NEW_QSTR(MP_QSTR_startswith), (mp_obj_t)&str_startswith_obj }, + { MP_OBJ_NEW_QSTR(MP_QSTR_endswith), (mp_obj_t)&str_endswith_obj }, + { MP_OBJ_NEW_QSTR(MP_QSTR_strip), (mp_obj_t)&str_strip_obj }, + { MP_OBJ_NEW_QSTR(MP_QSTR_lstrip), (mp_obj_t)&str_lstrip_obj }, + { MP_OBJ_NEW_QSTR(MP_QSTR_rstrip), (mp_obj_t)&str_rstrip_obj }, + { MP_OBJ_NEW_QSTR(MP_QSTR_format), (mp_obj_t)&str_format_obj }, + { MP_OBJ_NEW_QSTR(MP_QSTR_replace), (mp_obj_t)&str_replace_obj }, + { MP_OBJ_NEW_QSTR(MP_QSTR_count), (mp_obj_t)&str_count_obj }, + { MP_OBJ_NEW_QSTR(MP_QSTR_partition), (mp_obj_t)&str_partition_obj }, + { MP_OBJ_NEW_QSTR(MP_QSTR_rpartition), (mp_obj_t)&str_rpartition_obj }, + { MP_OBJ_NEW_QSTR(MP_QSTR_lower), (mp_obj_t)&str_lower_obj }, + { MP_OBJ_NEW_QSTR(MP_QSTR_upper), (mp_obj_t)&str_upper_obj }, + { MP_OBJ_NEW_QSTR(MP_QSTR_isspace), (mp_obj_t)&str_isspace_obj }, + { MP_OBJ_NEW_QSTR(MP_QSTR_isalpha), (mp_obj_t)&str_isalpha_obj }, + { MP_OBJ_NEW_QSTR(MP_QSTR_isdigit), (mp_obj_t)&str_isdigit_obj }, + { MP_OBJ_NEW_QSTR(MP_QSTR_isupper), (mp_obj_t)&str_isupper_obj }, + { MP_OBJ_NEW_QSTR(MP_QSTR_islower), (mp_obj_t)&str_islower_obj }, +}; + +STATIC MP_DEFINE_CONST_DICT(str_locals_dict, str_locals_dict_table); + +const mp_obj_type_t mp_type_str = { + { &mp_type_type }, + .name = MP_QSTR_str, + .print = str_print, + .make_new = str_make_new, + .binary_op = str_binary_op, + .subscr = str_subscr, + .getiter = mp_obj_new_str_iterator, + .buffer_p = { .get_buffer = str_get_buffer }, + .locals_dict = (mp_obj_t)&str_locals_dict, +}; + +// Reuses most of methods from str +const mp_obj_type_t mp_type_bytes = { + { &mp_type_type }, + .name = MP_QSTR_bytes, + .print = str_print, + .make_new = bytes_make_new, + .binary_op = str_binary_op, + .subscr = str_subscr, + .getiter = mp_obj_new_bytes_iterator, + .buffer_p = { .get_buffer = str_get_buffer }, + .locals_dict = (mp_obj_t)&str_locals_dict, +}; + +// the zero-length bytes +STATIC const mp_obj_str_t empty_bytes_obj = {{&mp_type_bytes}, 0, 0, NULL}; +const mp_obj_t mp_const_empty_bytes = (mp_obj_t)&empty_bytes_obj; + +mp_obj_t mp_obj_str_builder_start(const mp_obj_type_t *type, uint len, byte **data) { + mp_obj_str_t *o = m_new_obj(mp_obj_str_t); + o->base.type = type; + o->len = len; + o->hash = 0; + byte *p = m_new(byte, len + 1); + o->data = p; + *data = p; + return o; +} + +mp_obj_t mp_obj_str_builder_end(mp_obj_t o_in) { + mp_obj_str_t *o = o_in; + o->hash = qstr_compute_hash(o->data, o->len); + byte *p = (byte*)o->data; + p[o->len] = '\0'; // for now we add null for compatibility with C ASCIIZ strings + return o; +} + +mp_obj_t mp_obj_new_str_of_type(const mp_obj_type_t *type, const byte* data, uint len) { + mp_obj_str_t *o = m_new_obj(mp_obj_str_t); + o->base.type = type; + o->len = len; + if (data) { + o->hash = qstr_compute_hash(data, len); + byte *p = m_new(byte, len + 1); + o->data = p; + memcpy(p, data, len * sizeof(byte)); + p[len] = '\0'; // for now we add null for compatibility with C ASCIIZ strings + } + return o; +} + +mp_obj_t mp_obj_new_str(const char* data, uint len, bool make_qstr_if_not_already) { + if (make_qstr_if_not_already) { + // use existing, or make a new qstr + return MP_OBJ_NEW_QSTR(qstr_from_strn(data, len)); + } else { + qstr q = qstr_find_strn(data, len); + if (q != MP_QSTR_NULL) { + // qstr with this data already exists + return MP_OBJ_NEW_QSTR(q); + } else { + // no existing qstr, don't make one + return mp_obj_new_str_of_type(&mp_type_str, (const byte*)data, len); + } + } +} + +mp_obj_t mp_obj_str_intern(mp_obj_t str) { + GET_STR_DATA_LEN(str, data, len); + return MP_OBJ_NEW_QSTR(qstr_from_strn((const char*)data, len)); +} + +mp_obj_t mp_obj_new_bytes(const byte* data, uint len) { + return mp_obj_new_str_of_type(&mp_type_bytes, data, len); +} + +bool mp_obj_str_equal(mp_obj_t s1, mp_obj_t s2) { + if (MP_OBJ_IS_QSTR(s1) && MP_OBJ_IS_QSTR(s2)) { + return s1 == s2; + } else { + GET_STR_HASH(s1, h1); + GET_STR_HASH(s2, h2); + // If any of hashes is 0, it means it's not valid + if (h1 != 0 && h2 != 0 && h1 != h2) { + return false; + } + GET_STR_DATA_LEN(s1, d1, l1); + GET_STR_DATA_LEN(s2, d2, l2); + if (l1 != l2) { + return false; + } + return memcmp(d1, d2, l1) == 0; + } +} + +STATIC void bad_implicit_conversion(mp_obj_t self_in) { + nlr_raise(mp_obj_new_exception_msg_varg(&mp_type_TypeError, "Can't convert '%s' object to str implicitly", mp_obj_get_type_str(self_in))); +} + +STATIC void arg_type_mixup() { + nlr_raise(mp_obj_new_exception_msg(&mp_type_TypeError, "Can't mix str and bytes arguments")); +} + +uint mp_obj_str_get_hash(mp_obj_t self_in) { + // TODO: This has too big overhead for hash accessor + if (MP_OBJ_IS_STR(self_in) || MP_OBJ_IS_TYPE(self_in, &mp_type_bytes)) { + GET_STR_HASH(self_in, h); + return h; + } else { + bad_implicit_conversion(self_in); + } +} + +uint mp_obj_str_get_len(mp_obj_t self_in) { + // TODO This has a double check for the type, one in obj.c and one here + if (MP_OBJ_IS_STR(self_in) || MP_OBJ_IS_TYPE(self_in, &mp_type_bytes)) { + GET_STR_LEN(self_in, l); + return l; + } else { + bad_implicit_conversion(self_in); + } +} + +// use this if you will anyway convert the string to a qstr +// will be more efficient for the case where it's already a qstr +qstr mp_obj_str_get_qstr(mp_obj_t self_in) { + if (MP_OBJ_IS_QSTR(self_in)) { + return MP_OBJ_QSTR_VALUE(self_in); + } else if (MP_OBJ_IS_TYPE(self_in, &mp_type_str)) { + mp_obj_str_t *self = self_in; + return qstr_from_strn((char*)self->data, self->len); + } else { + bad_implicit_conversion(self_in); + } +} + +// only use this function if you need the str data to be zero terminated +// at the moment all strings are zero terminated to help with C ASCIIZ compatibility +const char *mp_obj_str_get_str(mp_obj_t self_in) { + if (MP_OBJ_IS_STR(self_in)) { + GET_STR_DATA_LEN(self_in, s, l); + (void)l; // len unused + return (const char*)s; + } else { + bad_implicit_conversion(self_in); + } +} + +const char *mp_obj_str_get_data(mp_obj_t self_in, uint *len) { + if (is_str_or_bytes(self_in)) { + GET_STR_DATA_LEN(self_in, s, l); + *len = l; + return (const char*)s; + } else { + bad_implicit_conversion(self_in); + } +} + +/******************************************************************************/ +/* str iterator */ + +typedef struct _mp_obj_str_it_t { + mp_obj_base_t base; + mp_obj_t str; + machine_uint_t cur; +} mp_obj_str_it_t; + +STATIC mp_obj_t str_it_iternext(mp_obj_t self_in) { + mp_obj_str_it_t *self = self_in; + GET_STR_DATA_LEN(self->str, str, len); + if (self->cur < len) { + mp_obj_t o_out = mp_obj_new_str((const char*)str + self->cur, 1, true); + self->cur += 1; + return o_out; + } else { + return MP_OBJ_STOP_ITERATION; + } +} + +STATIC const mp_obj_type_t mp_type_str_it = { + { &mp_type_type }, + .name = MP_QSTR_iterator, + .getiter = mp_identity, + .iternext = str_it_iternext, +}; + +STATIC mp_obj_t bytes_it_iternext(mp_obj_t self_in) { + mp_obj_str_it_t *self = self_in; + GET_STR_DATA_LEN(self->str, str, len); + if (self->cur < len) { + mp_obj_t o_out = MP_OBJ_NEW_SMALL_INT((mp_small_int_t)str[self->cur]); + self->cur += 1; + return o_out; + } else { + return MP_OBJ_STOP_ITERATION; + } +} + +STATIC const mp_obj_type_t mp_type_bytes_it = { + { &mp_type_type }, + .name = MP_QSTR_iterator, + .getiter = mp_identity, + .iternext = bytes_it_iternext, +}; + +mp_obj_t mp_obj_new_str_iterator(mp_obj_t str) { + mp_obj_str_it_t *o = m_new_obj(mp_obj_str_it_t); + o->base.type = &mp_type_str_it; + o->str = str; + o->cur = 0; + return o; +} + +mp_obj_t mp_obj_new_bytes_iterator(mp_obj_t str) { + mp_obj_str_it_t *o = m_new_obj(mp_obj_str_it_t); + o->base.type = &mp_type_bytes_it; + o->str = str; + o->cur = 0; + return o; +} From 64b468d8733845ac9f04776ee41434cdcc1a2602 Mon Sep 17 00:00:00 2001 From: Chris Angelico Date: Wed, 4 Jun 2014 05:28:12 +1000 Subject: [PATCH 04/32] objstrunicode: Basic implementation of unicode handling. Squashed commit of the following: commit 99dc21b67a895dc10d3c846bc158d27c839cee48 Author: Chris Angelico Date: Thu Jun 12 02:18:54 2014 +1000 Optimize as per TODO (thanks Damien!) commit 5bf0153ecad8348443058d449d74504fc458fe51 Author: Chris Angelico Date: Tue Jun 10 08:42:06 2014 +1000 Test a default (= UTF-8) encode and decode commit c962057ac340832c4fde60896f656a3fe3ad78a9 Merge: e2c9782 195de32 Author: Chris Angelico Date: Tue Jun 10 05:23:03 2014 +1000 Merge branch 'master' into unicode, resolving conflict on py/obj.h commit e2c9782a65eb57f481d441d40161de427e1940ba Author: Chris Angelico Date: Tue Jun 10 05:05:57 2014 +1000 More whitespace fixups commit 086a2a0f57afbc1f731697fd5d3a0cbbb80e5418 Author: Chris Angelico Date: Tue Jun 10 05:04:20 2014 +1000 Properly implement string slicing commit 0d339a143e2b6442366145e7f3d64aada293eaa0 Author: Chris Angelico Date: Tue Jun 10 02:24:11 2014 +1000 Support slicing in str_index_to_ptr, and fix a bounds error commit 24371c7267d360e77cf5eabc2e8ce9a73d2ee0da Author: Chris Angelico Date: Tue Jun 10 02:10:22 2014 +1000 Break out index-to-pointer calculation into a function commit 616c24ac014c3ca56008428c506034dd1bfff7a8 Author: Chris Angelico Date: Tue Jun 10 02:03:11 2014 +1000 Add tests of string slicing, which currently fail commit a24d19f676fe8cc21dad512d91b826892e162a5b Author: Chris Angelico Date: Tue Jun 10 01:56:53 2014 +1000 Change string indexing to not precalculate the charlen, and add test for neg indexing commit 0bcc7ab89eafb2ae53195e94c9bea42a4e886b64 Author: Chris Angelico Date: Sun Jun 8 22:09:17 2014 +1000 Clean up constant qstr declarations now that charlen isn't needed commit 5473e1a1dba2124b7b0c207f2964293cfbe80167 Author: Chris Angelico Date: Sun Jun 8 07:18:42 2014 +1000 Remove the charlen field from strings, calculating it when required commit 5c1658ec71aefbdc88c261ce2e57dc7670cdc6ef Author: Chris Angelico Date: Sun Jun 8 07:11:27 2014 +1000 Get rid of mp_obj_str_get_data_len() which was used in only one place commit a019ba968b4e8daf7f3674f63c5cc400e304c509 Author: Chris Angelico Date: Sun Jun 8 06:58:26 2014 +1000 Add a unichar_charlen() function to calculate length-in-characters from length-in-bytes commit 44b0d5cff846ba487c526ed95be1b3d1cd3d762a Author: Chris Angelico Date: Sun Jun 8 06:32:44 2014 +1000 Use utf8_get/next_char in building up a string's repr commit 30d1bad33f7af90f1971987c39864c8fcf3f5c21 Author: Chris Angelico Date: Sun Jun 8 06:10:45 2014 +1000 Make utf8_get_char() and utf8_next_char() actually do what their names say commit bc990dad9afb8ec112f5e7f7f79d5ab415da0e72 Author: Chris Angelico Date: Sun Jun 8 02:10:59 2014 +1000 Revert "Add PEP 393-flags to strings and stub usage." This reverts commit c239f509521d1a0f9563bf9c5de0c4fb9a6a33ba. commit f9bebb28ad52467f2f2d7a752bb033296b6c2f9b Author: Chris Angelico Date: Sat Jun 7 15:41:48 2014 +1000 Whitespace fixes commit 279de0c8eb3cb186914799ccc5ee94ea97f56de4 Author: Chris Angelico Date: Sat Jun 7 15:28:35 2014 +1000 Formatting/layout improvements - introduce macros for UTF-8 byte detection, add braces. No functional changes. commit f1911f53d56da809c97b07245f5728a419e8fb30 Author: Chris Angelico Date: Sat Jun 7 11:56:02 2014 +1000 Make chr() Unicode-aware commit f51ad737b48ac04c161197a4012821d50885c4c7 Author: Chris Angelico Date: Sat Jun 7 11:44:07 2014 +1000 Make a string's repr Unicode-aware commit 01bd68684611585d437982dccdf05b33cbedc630 Author: Chris Angelico Date: Sat Jun 7 11:33:43 2014 +1000 Expand the Unicode tests commit 7bc91904f899f8012089fc14a06495680a51e590 Author: Chris Angelico Date: Sat Jun 7 11:27:30 2014 +1000 Record byte lengths for byte strings commit bb132120717cf176dcfb26f87fa309378f76ab5f Author: Chris Angelico Date: Sat Jun 7 11:25:06 2014 +1000 Make ord() Unicode-aware commit 03f0cbe9051b62192be97b59f84f63f9216668bf Author: Chris Angelico Date: Sat Jun 7 10:24:35 2014 +1000 Retain characters as UTF-8 encoded Unicode commit e924659b85c001916a5ff7f4d1d8b3ebe2bf0c2f Author: Chris Angelico Date: Sat Jun 7 08:37:27 2014 +1000 Add support for \u and \U escapes, but not \N (with explanatory comment) commit 231031ac5f0346e4ffcf9c4abec2bd33f566232c Author: Chris Angelico Date: Sat Jun 7 05:09:35 2014 +1000 Add character length to qstr commit 6df1b946fb17d8d5df3d91b21cde627c3d4556a8 Author: Chris Angelico Date: Fri Jun 6 13:48:36 2014 +1000 Add test of UTF-8 encoded source file resulting in properly formed string commit 16429b81a8483cf25865ed11afd81a7d9c253c26 Author: Chris Angelico Date: Fri Jun 6 13:44:15 2014 +1000 Make len(s) return character length (even though creation's still buggy) commit cd2cf6663cc47831dbc97819ad5c50ad33f939d3 Author: Chris Angelico Date: Fri Jun 6 13:15:36 2014 +1000 HACK - When indexing a qstr, count its charlen. Stupidly inefficient but POC. All tests pass now, though string creation is still buggy. commit 47c234584d3358dfa6b4003d5e7264105d17b8f7 Author: Chris Angelico Date: Fri Jun 6 13:15:32 2014 +1000 objstr: Record character length separately from byte length CAUTION: Buggy, may crash stuff - qstr needs equivalent functionality too commit b0f41c72af27d3b361027146025877b3d7e8785c Author: Chris Angelico Date: Fri Jun 6 05:37:36 2014 +1000 Beginnings of UTF-8 support - construct strings from that many UTF-8-encoded chars, and subscript bytes the same way commit 89452be641674601e9bfce86dc71c17c3140a6cf Author: Chris Angelico Date: Fri Jun 6 05:28:47 2014 +1000 Update comments - now aiming for UTF-8 rather than PEP 393 strings commit c239f509521d1a0f9563bf9c5de0c4fb9a6a33ba Author: Chris Angelico Date: Wed Jun 4 05:28:12 2014 +1000 Add PEP 393-flags to strings and stub usage. The test suite all passes, but nothing has actually been changed. --- py/objstrunicode.c | 154 ++++++++++++++++++++++++++++++++++++++------- 1 file changed, 130 insertions(+), 24 deletions(-) diff --git a/py/objstrunicode.c b/py/objstrunicode.c index 6656090c8..b1b61d74f 100644 --- a/py/objstrunicode.c +++ b/py/objstrunicode.c @@ -64,7 +64,7 @@ STATIC bool is_str_or_bytes(mp_obj_t o) { /******************************************************************************/ /* str */ -void mp_str_print_quoted(void (*print)(void *env, const char *fmt, ...), void *env, const byte *str_data, uint str_len) { +void mp_str_print_quoted(void (*print)(void *env, const char *fmt, ...), void *env, const byte *str_data, uint str_len, bool is_bytes) { // this escapes characters, but it will be very slow to print (calling print many times) bool has_single_quote = false; bool has_double_quote = false; @@ -80,21 +80,33 @@ void mp_str_print_quoted(void (*print)(void *env, const char *fmt, ...), void *e quote_char = '"'; } print(env, "%c", quote_char); - for (const byte *s = str_data, *top = str_data + str_len; s < top; s++) { - if (*s == quote_char) { - print(env, "\\%c", quote_char); - } else if (*s == '\\') { - print(env, "\\\\"); - } else if (32 <= *s && *s <= 126) { - print(env, "%c", *s); - } else if (*s == '\n') { - print(env, "\\n"); - } else if (*s == '\r') { - print(env, "\\r"); - } else if (*s == '\t') { - print(env, "\\t"); + const char *s = (const char *)str_data, *top = (const char *)str_data + str_len; + while (s < top) { + unichar ch; + if (is_bytes) { + ch = *(unsigned char *)s++; // Don't sign-extend bytes } else { - print(env, "\\x%02x", *s); + ch = utf8_get_char(s); + s = utf8_next_char(s); + } + if (ch == quote_char) { + print(env, "\\%c", quote_char); + } else if (ch == '\\') { + print(env, "\\\\"); + } else if (32 <= ch && ch <= 126) { + print(env, "%c", ch); + } else if (ch == '\n') { + print(env, "\\n"); + } else if (ch == '\r') { + print(env, "\\r"); + } else if (ch == '\t') { + print(env, "\\t"); + } else if (ch < 0x100) { + print(env, "\\x%02x", ch); + } else if (ch < 0x10000) { + print(env, "\\u%04x", ch); + } else { + print(env, "\\U%08x", ch); } } print(env, "%c", quote_char); @@ -109,7 +121,7 @@ STATIC void str_print(void (*print)(void *env, const char *fmt, ...), void *env, if (is_bytes) { print(env, "b"); } - mp_str_print_quoted(print, env, str_data, str_len); + mp_str_print_quoted(print, env, str_data, str_len, is_bytes); } } @@ -348,6 +360,59 @@ uncomparable: return MP_OBJ_NULL; // op not supported } +// Convert an index into a pointer to its lead byte. Out of bounds indexing will raise IndexError or +// be capped to the first/last character of the string, depending on is_slice. +STATIC const char *str_index_to_ptr(const char *self_data, uint self_len, mp_obj_t index, bool is_slice) { + machine_int_t i; + // Copied from mp_get_index; I don't want bounds checking, just give me + // the integer as-is. (I can't bounds-check without scanning the whole + // string; an out-of-bounds index will be caught in the loops below.) + if (MP_OBJ_IS_SMALL_INT(index)) { + i = MP_OBJ_SMALL_INT_VALUE(index); + } else if (!mp_obj_get_int_maybe(index, &i)) { + nlr_raise(mp_obj_new_exception_msg_varg(&mp_type_TypeError, "string indices must be integers, not %s", mp_obj_get_type_str(index))); + } + const char *s, *top = self_data + self_len; + if (i < 0) + { + // Negative indexing is performed by counting from the end of the string. + for (s = top - 1; i; --s) { + if (s < self_data) { + if (is_slice) { + return self_data; + } + nlr_raise(mp_obj_new_exception_msg_varg(&mp_type_IndexError, "string index out of range")); + } + if (!UTF8_IS_CONT(*s)) { + ++i; + } + } + ++s; + } else if (!i) { + return self_data; // Shortcut - str[0] is its base pointer + } else { + // Positive indexing, correspondingly, counts from the start of the string. + // It's assumed that negative indexing will generally be used with small + // absolute values (eg str[-1], not str[-1000000]), which means it'll be + // more efficient this way. + for (s = self_data; true; ++s) { + if (s >= top) { + if (is_slice) { + return top; + } + nlr_raise(mp_obj_new_exception_msg_varg(&mp_type_IndexError, "string index out of range")); + } + while (UTF8_IS_CONT(*s)) { + ++s; + } + if (!i--) { + return s; + } + } + } + return s; +} + STATIC mp_obj_t str_subscr(mp_obj_t self_in, mp_obj_t index, mp_obj_t value) { mp_obj_type_t *type = mp_obj_get_type(self_in); GET_STR_DATA_LEN(self_in, self_data, self_len); @@ -355,20 +420,61 @@ STATIC mp_obj_t str_subscr(mp_obj_t self_in, mp_obj_t index, mp_obj_t value) { // load #if MICROPY_PY_BUILTINS_SLICE if (MP_OBJ_IS_TYPE(index, &mp_type_slice)) { - mp_bound_slice_t slice; - if (!mp_seq_get_fast_slice_indexes(self_len, index, &slice)) { + mp_obj_t ostart, ostop, ostep; + mp_obj_slice_get(index, &ostart, &ostop, &ostep); + if (ostep != mp_const_none && ostep != MP_OBJ_NEW_SMALL_INT(1)) { nlr_raise(mp_obj_new_exception_msg(&mp_type_NotImplementedError, "only slices with step=1 (aka None) are supported")); } - return mp_obj_new_str_of_type(type, self_data + slice.start, slice.stop - slice.start); + + if (type == &mp_type_bytes) { + machine_int_t start = 0, stop = self_len; + if (ostart != mp_const_none) { + start = MP_OBJ_SMALL_INT_VALUE(ostart); + if (start < 0) { + start = self_len + start; + } + } + if (ostop != mp_const_none) { + stop = MP_OBJ_SMALL_INT_VALUE(ostop); + if (stop < 0) { + stop = self_len + stop; + } + } + return mp_obj_new_str_of_type(type, self_data + start, stop - start); + } + const char *pstart, *pstop; + if (ostart != mp_const_none) { + pstart = str_index_to_ptr((const char *)self_data, self_len, ostart, true); + } else { + pstart = (const char *)self_data; + } + if (ostop != mp_const_none) { + // pstop will point just after the stop character. This depends on + // the \0 at the end of the string. + pstop = str_index_to_ptr((const char *)self_data, self_len, ostop, true); + } else { + pstop = (const char *)self_data + self_len; + } + if (pstop < pstart) { + return MP_OBJ_NEW_QSTR(MP_QSTR_); + } + return mp_obj_new_str_of_type(type, (const byte *)pstart, pstop - pstart); } #endif - uint index_val = mp_get_index(type, self_len, index, false); if (type == &mp_type_bytes) { + uint index_val = mp_get_index(type, self_len, index, false); return MP_OBJ_NEW_SMALL_INT((mp_small_int_t)self_data[index_val]); - } else { - return mp_obj_new_str((char*)self_data + index_val, 1, true); } + const char *s = str_index_to_ptr((const char *)self_data, self_len, index, false); + int len = 1; + if (UTF8_IS_NONASCII(*s)) { + // Count the number of 1 bits (after the first) + for (char mask = 0x40; *s & mask; mask >>= 1) { + ++len; + } + } + return mp_obj_new_str(s, len, true); // This will create a one-character string } else { return MP_OBJ_NULL; // op not supported } @@ -1800,8 +1906,8 @@ uint mp_obj_str_get_hash(mp_obj_t self_in) { uint mp_obj_str_get_len(mp_obj_t self_in) { // TODO This has a double check for the type, one in obj.c and one here if (MP_OBJ_IS_STR(self_in) || MP_OBJ_IS_TYPE(self_in, &mp_type_bytes)) { - GET_STR_LEN(self_in, l); - return l; + GET_STR_DATA_LEN(self_in, self_data, self_len); + return unichar_charlen((const char *)self_data, self_len); } else { bad_implicit_conversion(self_in); } From 9a1a4beb563f8e2ae111ff062a64989774f29058 Mon Sep 17 00:00:00 2001 From: Chris Angelico Date: Wed, 4 Jun 2014 05:28:12 +1000 Subject: [PATCH 05/32] builtin: ord, chr: Unicode support. --- py/builtin.c | 45 ++++++++++++++++++++++++++++++++++++--------- 1 file changed, 36 insertions(+), 9 deletions(-) diff --git a/py/builtin.c b/py/builtin.c index d4b77d37a..0c0163d00 100644 --- a/py/builtin.c +++ b/py/builtin.c @@ -172,13 +172,30 @@ STATIC mp_obj_t mp_builtin_callable(mp_obj_t o_in) { MP_DEFINE_CONST_FUN_OBJ_1(mp_builtin_callable_obj, mp_builtin_callable); STATIC mp_obj_t mp_builtin_chr(mp_obj_t o_in) { - int ord = mp_obj_get_int(o_in); - if (0 <= ord && ord <= 0x10ffff) { - char str[1] = {ord}; - return mp_obj_new_str(str, 1, true); + int c = mp_obj_get_int(o_in); + char str[4]; + int len = 0; + if (c < 0x80) { + *str = c; len = 1; + } else if (c < 0x800) { + str[0] = (c >> 6) | 0xC0; + str[1] = (c & 0x3F) | 0x80; + len = 2; + } else if (c < 0x10000) { + str[0] = (c >> 12) | 0xE0; + str[1] = ((c >> 6) & 0x3F) | 0x80; + str[2] = (c & 0x3F) | 0x80; + len = 3; + } else if (c < 0x110000) { + str[0] = (c >> 18) | 0xF0; + str[1] = ((c >> 12) & 0x3F) | 0x80; + str[2] = ((c >> 6) & 0x3F) | 0x80; + str[3] = (c & 0x3F) | 0x80; + len = 4; } else { nlr_raise(mp_obj_new_exception_msg(&mp_type_ValueError, "chr() arg not in range(0x110000)")); } + return mp_obj_new_str(str, len, true); } MP_DEFINE_CONST_FUN_OBJ_1(mp_builtin_chr_obj, mp_builtin_chr); @@ -344,12 +361,22 @@ MP_DEFINE_CONST_FUN_OBJ_1(mp_builtin_oct_obj, mp_builtin_oct); STATIC mp_obj_t mp_builtin_ord(mp_obj_t o_in) { uint len; const char *str = mp_obj_str_get_data(o_in, &len); - if (len == 1) { - // don't sign extend when converting to ord - // TODO unicode - return mp_obj_new_int(((const byte*)str)[0]); + uint charlen = unichar_charlen(str, len); + if (charlen == 1) { + if (MP_OBJ_IS_STR(o_in) && UTF8_IS_NONASCII(*str)) { + machine_int_t ord = *str++ & 0x7F; + for (machine_int_t mask = 0x40; ord & mask; mask >>= 1) { + ord &= ~mask; + } + while (UTF8_IS_CONT(*str)) { + ord = (ord << 6) | (*str++ & 0x3F); + } + return mp_obj_new_int(ord); + } else { + return mp_obj_new_int(((const byte*)str)[0]); + } } else { - nlr_raise(mp_obj_new_exception_msg_varg(&mp_type_TypeError, "ord() expected a character, but string of length %d found", len)); + nlr_raise(mp_obj_new_exception_msg_varg(&mp_type_TypeError, "ord() expected a character, but string of length %d found", charlen)); } } From 1e3781bc3527f72053fdc4aad4f4887c567c457c Mon Sep 17 00:00:00 2001 From: Chris Angelico Date: Wed, 4 Jun 2014 05:28:12 +1000 Subject: [PATCH 06/32] tests: Add unicode test. --- tests/unicode/unicode.py | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) create mode 100644 tests/unicode/unicode.py diff --git a/tests/unicode/unicode.py b/tests/unicode/unicode.py new file mode 100644 index 000000000..c7e523f06 --- /dev/null +++ b/tests/unicode/unicode.py @@ -0,0 +1,18 @@ +# Test a UTF-8 encoded literal +s = "asdf©qwer" +for i in range(len(s)): + print("s[%d]: %s %X"%(i, s[i], ord(s[i]))) + +# Test all three forms of Unicode escape, and +# all blocks of UTF-8 byte patterns +s = "a\xA9\xFF\u0123\u0800\uFFEE\U0001F44C" +for i in range(-len(s), len(s)): + print("s[%d]: %s %X"%(i, s[i], ord(s[i]))) + print("s[:%d]: %d chars, '%s'"%(i, len(s[:i]), s[:i])) + for j in range(i, len(s)): + print("s[%d:%d]: %d chars, '%s'"%(i, j, len(s[i:j]), s[i:j])) + print("s[%d:]: %d chars, '%s'"%(i, len(s[i:]), s[i:])) + +# Test UTF-8 encode and decode +enc = s.encode() +print(enc, enc.decode() == s) From 2ba2299d28b35ec9da6636d14a7fce0d954cd66e Mon Sep 17 00:00:00 2001 From: Chris Angelico Date: Wed, 4 Jun 2014 05:28:12 +1000 Subject: [PATCH 07/32] lexer, vstr: Add unicode support. --- py/lexer.c | 29 ++++++++++++++++++++++++----- py/vstr.c | 38 +++++++++++++++++++++++++++++++++----- 2 files changed, 57 insertions(+), 10 deletions(-) diff --git a/py/lexer.c b/py/lexer.c index 5d1113fb3..8732d6436 100644 --- a/py/lexer.c +++ b/py/lexer.c @@ -502,19 +502,32 @@ STATIC void mp_lexer_next_token_into(mp_lexer_t *lex, mp_token_t *tok, bool firs case 'v': c = 0x0b; break; case 'f': c = 0x0c; break; case 'r': c = 0x0d; break; + case 'u': + case 'U': + if (is_bytes) { + // b'\u1234' == b'\\u1234' + vstr_add_char(&lex->vstr, '\\'); + break; + } + // Otherwise fall through. case 'x': { uint num = 0; - if (!get_hex(lex, 2, &num)) { + if (!get_hex(lex, (c == 'x' ? 2 : c == 'u' ? 4 : 8), &num)) { // TODO error message assert(0); } c = num; break; } - case 'N': break; // TODO \N{name} only in strings - case 'u': break; // TODO \uxxxx only in strings - case 'U': break; // TODO \Uxxxxxxxx only in strings + case 'N': + // Supporting '\N{LATIN SMALL LETTER A}' == 'a' would require keeping the + // entire Unicode name table in the core. As of Unicode 6.3.0, that's nearly + // 3MB of text; even gzip-compressed and with minimal structure, it'll take + // roughly half a meg of storage. This form of Unicode escape may be added + // later on, but it's definitely not a priority right now. -- CJA 20140607 + assert(!"Unicode name escapes not supported"); + break; default: if (c >= '0' && c <= '7') { // Octal sequence, 1-3 chars @@ -533,7 +546,13 @@ STATIC void mp_lexer_next_token_into(mp_lexer_t *lex, mp_token_t *tok, bool firs } } if (c != MP_LEXER_CHAR_EOF) { - vstr_add_char(&lex->vstr, c); + if (c < 0x110000 && !is_bytes) { + vstr_add_char(&lex->vstr, c); + } else if (c < 0x100 && is_bytes) { + vstr_add_byte(&lex->vstr, c); + } else { + assert(!"TODO: Throw an error, invalid escape code probably"); + } } } else { vstr_add_char(&lex->vstr, CUR_CHAR(lex)); diff --git a/py/vstr.c b/py/vstr.c index f8b7e4dab..2dbc6f04a 100644 --- a/py/vstr.c +++ b/py/vstr.c @@ -199,12 +199,40 @@ void vstr_add_byte(vstr_t *vstr, byte b) { } void vstr_add_char(vstr_t *vstr, unichar c) { - // TODO UNICODE - byte *buf = (byte*)vstr_add_len(vstr, 1); - if (buf == NULL) { - return; + // TODO: Can this be simplified and deduplicated? + // Is it worth just calling vstr_add_len(vstr, 4)? + if (c < 0x80) { + byte *buf = (byte*)vstr_add_len(vstr, 1); + if (buf == NULL) { + return; + } + *buf = (byte)c; + } else if (c < 0x800) { + byte *buf = (byte*)vstr_add_len(vstr, 2); + if (buf == NULL) { + return; + } + buf[0] = (c >> 6) | 0xC0; + buf[1] = (c & 0x3F) | 0x80; + } else if (c < 0x10000) { + byte *buf = (byte*)vstr_add_len(vstr, 3); + if (buf == NULL) { + return; + } + buf[0] = (c >> 12) | 0xE0; + buf[1] = ((c >> 6) & 0x3F) | 0x80; + buf[2] = (c & 0x3F) | 0x80; + } else { + assert(c < 0x110000); + byte *buf = (byte*)vstr_add_len(vstr, 4); + if (buf == NULL) { + return; + } + buf[0] = (c >> 18) | 0xF0; + buf[1] = ((c >> 12) & 0x3F) | 0x80; + buf[2] = ((c >> 6) & 0x3F) | 0x80; + buf[3] = (c & 0x3F) | 0x80; } - buf[0] = c; } void vstr_add_str(vstr_t *vstr, const char *str) { From 42a52516fe36a93979954844fcd2d8bfe61eb63c Mon Sep 17 00:00:00 2001 From: Paul Sokolovsky Date: Fri, 13 Jun 2014 02:39:37 +0300 Subject: [PATCH 08/32] builtin: Restore bytestr compatibility. --- py/builtin.c | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/py/builtin.c b/py/builtin.c index 0c0163d00..1dd364911 100644 --- a/py/builtin.c +++ b/py/builtin.c @@ -172,6 +172,7 @@ STATIC mp_obj_t mp_builtin_callable(mp_obj_t o_in) { MP_DEFINE_CONST_FUN_OBJ_1(mp_builtin_callable_obj, mp_builtin_callable); STATIC mp_obj_t mp_builtin_chr(mp_obj_t o_in) { + #if MICROPY_PY_BUILTINS_STR_UNICODE int c = mp_obj_get_int(o_in); char str[4]; int len = 0; @@ -196,6 +197,15 @@ STATIC mp_obj_t mp_builtin_chr(mp_obj_t o_in) { nlr_raise(mp_obj_new_exception_msg(&mp_type_ValueError, "chr() arg not in range(0x110000)")); } return mp_obj_new_str(str, len, true); + #else + int ord = mp_obj_get_int(o_in); + if (0 <= ord && ord <= 0x10ffff) { + char str[1] = {ord}; + return mp_obj_new_str(str, 1, true); + } else { + nlr_raise(mp_obj_new_exception_msg(&mp_type_ValueError, "chr() arg not in range(0x110000)")); + } + #endif } MP_DEFINE_CONST_FUN_OBJ_1(mp_builtin_chr_obj, mp_builtin_chr); @@ -361,6 +371,7 @@ MP_DEFINE_CONST_FUN_OBJ_1(mp_builtin_oct_obj, mp_builtin_oct); STATIC mp_obj_t mp_builtin_ord(mp_obj_t o_in) { uint len; const char *str = mp_obj_str_get_data(o_in, &len); + #if MICROPY_PY_BUILTINS_STR_UNICODE uint charlen = unichar_charlen(str, len); if (charlen == 1) { if (MP_OBJ_IS_STR(o_in) && UTF8_IS_NONASCII(*str)) { @@ -378,6 +389,14 @@ STATIC mp_obj_t mp_builtin_ord(mp_obj_t o_in) { } else { nlr_raise(mp_obj_new_exception_msg_varg(&mp_type_TypeError, "ord() expected a character, but string of length %d found", charlen)); } + #else + if (len == 1) { + // don't sign extend when converting to ord + return mp_obj_new_int(((const byte*)str)[0]); + } else { + nlr_raise(mp_obj_new_exception_msg_varg(&mp_type_TypeError, "ord() expected a character, but string of length %d found", len)); + } + #endif } MP_DEFINE_CONST_FUN_OBJ_1(mp_builtin_ord_obj, mp_builtin_ord); From 165eb69b86e7160ffd64f9cbfb8543c4d25a0fa3 Mon Sep 17 00:00:00 2001 From: Paul Sokolovsky Date: Fri, 13 Jun 2014 02:42:34 +0300 Subject: [PATCH 09/32] vstr: Restore bytestr compatibility. --- py/vstr.c | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/py/vstr.c b/py/vstr.c index 2dbc6f04a..9ccc95d49 100644 --- a/py/vstr.c +++ b/py/vstr.c @@ -199,6 +199,7 @@ void vstr_add_byte(vstr_t *vstr, byte b) { } void vstr_add_char(vstr_t *vstr, unichar c) { +#if MICROPY_PY_BUILTINS_STR_UNICODE // TODO: Can this be simplified and deduplicated? // Is it worth just calling vstr_add_len(vstr, 4)? if (c < 0x80) { @@ -233,6 +234,13 @@ void vstr_add_char(vstr_t *vstr, unichar c) { buf[2] = ((c >> 6) & 0x3F) | 0x80; buf[3] = (c & 0x3F) | 0x80; } +#else + byte *buf = (byte*)vstr_add_len(vstr, 1); + if (buf == NULL) { + return; + } + buf[0] = c; +#endif } void vstr_add_str(vstr_t *vstr, const char *str) { From 9731912ccba94adeda95a150502d3ed7e97ce99b Mon Sep 17 00:00:00 2001 From: Paul Sokolovsky Date: Fri, 13 Jun 2014 22:01:26 +0300 Subject: [PATCH 10/32] py: Prune unneeded code from objstrunicode, reuse code in objstr. --- py/objstr.c | 63 +- py/objstr.h | 44 ++ py/objstrunicode.c | 1671 +------------------------------------------- py/py.mk | 1 + 4 files changed, 77 insertions(+), 1702 deletions(-) diff --git a/py/objstr.c b/py/objstr.c index c84d7c900..c5acca325 100644 --- a/py/objstr.c +++ b/py/objstr.c @@ -43,15 +43,6 @@ STATIC mp_obj_t str_modulo_format(mp_obj_t pattern, uint n_args, const mp_obj_t *args, mp_obj_t dict); const mp_obj_t mp_const_empty_bytes; -// use this macro to extract the string hash -#define GET_STR_HASH(str_obj_in, str_hash) uint str_hash; if (MP_OBJ_IS_QSTR(str_obj_in)) { str_hash = qstr_hash(MP_OBJ_QSTR_VALUE(str_obj_in)); } else { str_hash = ((mp_obj_str_t*)str_obj_in)->hash; } - -// use this macro to extract the string length -#define GET_STR_LEN(str_obj_in, str_len) uint str_len; if (MP_OBJ_IS_QSTR(str_obj_in)) { str_len = qstr_len(MP_OBJ_QSTR_VALUE(str_obj_in)); } else { str_len = ((mp_obj_str_t*)str_obj_in)->len; } - -// use this macro to extract the string data and length -#define GET_STR_DATA_LEN(str_obj_in, str_data, str_len) const byte *str_data; uint str_len; if (MP_OBJ_IS_QSTR(str_obj_in)) { str_data = qstr_data(MP_OBJ_QSTR_VALUE(str_obj_in), &str_len); } else { str_len = ((mp_obj_str_t*)str_obj_in)->len; str_data = ((mp_obj_str_t*)str_obj_in)->data; } - STATIC mp_obj_t mp_obj_new_str_iterator(mp_obj_t str); STATIC mp_obj_t mp_obj_new_bytes_iterator(mp_obj_t str); STATIC NORETURN void bad_implicit_conversion(mp_obj_t self_in); @@ -259,7 +250,7 @@ STATIC const byte *find_subbytes(const byte *haystack, machine_uint_t hlen, cons return NULL; } -STATIC mp_obj_t str_binary_op(int op, mp_obj_t lhs_in, mp_obj_t rhs_in) { +mp_obj_t str_binary_op(int op, mp_obj_t lhs_in, mp_obj_t rhs_in) { GET_STR_DATA_LEN(lhs_in, lhs_data, lhs_len); mp_obj_type_t *lhs_type = mp_obj_get_type(lhs_in); mp_obj_type_t *rhs_type = mp_obj_get_type(rhs_in); @@ -1627,33 +1618,33 @@ STATIC machine_int_t str_get_buffer(mp_obj_t self_in, mp_buffer_info_t *bufinfo, } #if MICROPY_CPYTHON_COMPAT -STATIC MP_DEFINE_CONST_FUN_OBJ_VAR_BETWEEN(bytes_decode_obj, 1, 3, bytes_decode); -STATIC MP_DEFINE_CONST_FUN_OBJ_VAR_BETWEEN(str_encode_obj, 1, 3, str_encode); +MP_DEFINE_CONST_FUN_OBJ_VAR_BETWEEN(bytes_decode_obj, 1, 3, bytes_decode); +MP_DEFINE_CONST_FUN_OBJ_VAR_BETWEEN(str_encode_obj, 1, 3, str_encode); #endif -STATIC MP_DEFINE_CONST_FUN_OBJ_VAR_BETWEEN(str_find_obj, 2, 4, str_find); -STATIC MP_DEFINE_CONST_FUN_OBJ_VAR_BETWEEN(str_rfind_obj, 2, 4, str_rfind); -STATIC MP_DEFINE_CONST_FUN_OBJ_VAR_BETWEEN(str_index_obj, 2, 4, str_index); -STATIC MP_DEFINE_CONST_FUN_OBJ_VAR_BETWEEN(str_rindex_obj, 2, 4, str_rindex); -STATIC MP_DEFINE_CONST_FUN_OBJ_2(str_join_obj, str_join); -STATIC MP_DEFINE_CONST_FUN_OBJ_VAR_BETWEEN(str_split_obj, 1, 3, str_split); -STATIC MP_DEFINE_CONST_FUN_OBJ_VAR_BETWEEN(str_rsplit_obj, 1, 3, str_rsplit); -STATIC MP_DEFINE_CONST_FUN_OBJ_VAR_BETWEEN(str_startswith_obj, 2, 3, str_startswith); -STATIC MP_DEFINE_CONST_FUN_OBJ_VAR_BETWEEN(str_endswith_obj, 2, 3, str_endswith); -STATIC MP_DEFINE_CONST_FUN_OBJ_VAR_BETWEEN(str_strip_obj, 1, 2, str_strip); -STATIC MP_DEFINE_CONST_FUN_OBJ_VAR_BETWEEN(str_lstrip_obj, 1, 2, str_lstrip); -STATIC MP_DEFINE_CONST_FUN_OBJ_VAR_BETWEEN(str_rstrip_obj, 1, 2, str_rstrip); -STATIC MP_DEFINE_CONST_FUN_OBJ_VAR(str_format_obj, 1, mp_obj_str_format); -STATIC MP_DEFINE_CONST_FUN_OBJ_VAR_BETWEEN(str_replace_obj, 3, 4, str_replace); -STATIC MP_DEFINE_CONST_FUN_OBJ_VAR_BETWEEN(str_count_obj, 2, 4, str_count); -STATIC MP_DEFINE_CONST_FUN_OBJ_2(str_partition_obj, str_partition); -STATIC MP_DEFINE_CONST_FUN_OBJ_2(str_rpartition_obj, str_rpartition); -STATIC MP_DEFINE_CONST_FUN_OBJ_1(str_lower_obj, str_lower); -STATIC MP_DEFINE_CONST_FUN_OBJ_1(str_upper_obj, str_upper); -STATIC MP_DEFINE_CONST_FUN_OBJ_1(str_isspace_obj, str_isspace); -STATIC MP_DEFINE_CONST_FUN_OBJ_1(str_isalpha_obj, str_isalpha); -STATIC MP_DEFINE_CONST_FUN_OBJ_1(str_isdigit_obj, str_isdigit); -STATIC MP_DEFINE_CONST_FUN_OBJ_1(str_isupper_obj, str_isupper); -STATIC MP_DEFINE_CONST_FUN_OBJ_1(str_islower_obj, str_islower); +MP_DEFINE_CONST_FUN_OBJ_VAR_BETWEEN(str_find_obj, 2, 4, str_find); +MP_DEFINE_CONST_FUN_OBJ_VAR_BETWEEN(str_rfind_obj, 2, 4, str_rfind); +MP_DEFINE_CONST_FUN_OBJ_VAR_BETWEEN(str_index_obj, 2, 4, str_index); +MP_DEFINE_CONST_FUN_OBJ_VAR_BETWEEN(str_rindex_obj, 2, 4, str_rindex); +MP_DEFINE_CONST_FUN_OBJ_2(str_join_obj, str_join); +MP_DEFINE_CONST_FUN_OBJ_VAR_BETWEEN(str_split_obj, 1, 3, str_split); +MP_DEFINE_CONST_FUN_OBJ_VAR_BETWEEN(str_rsplit_obj, 1, 3, str_rsplit); +MP_DEFINE_CONST_FUN_OBJ_VAR_BETWEEN(str_startswith_obj, 2, 3, str_startswith); +MP_DEFINE_CONST_FUN_OBJ_VAR_BETWEEN(str_endswith_obj, 2, 3, str_endswith); +MP_DEFINE_CONST_FUN_OBJ_VAR_BETWEEN(str_strip_obj, 1, 2, str_strip); +MP_DEFINE_CONST_FUN_OBJ_VAR_BETWEEN(str_lstrip_obj, 1, 2, str_lstrip); +MP_DEFINE_CONST_FUN_OBJ_VAR_BETWEEN(str_rstrip_obj, 1, 2, str_rstrip); +MP_DEFINE_CONST_FUN_OBJ_VAR(str_format_obj, 1, mp_obj_str_format); +MP_DEFINE_CONST_FUN_OBJ_VAR_BETWEEN(str_replace_obj, 3, 4, str_replace); +MP_DEFINE_CONST_FUN_OBJ_VAR_BETWEEN(str_count_obj, 2, 4, str_count); +MP_DEFINE_CONST_FUN_OBJ_2(str_partition_obj, str_partition); +MP_DEFINE_CONST_FUN_OBJ_2(str_rpartition_obj, str_rpartition); +MP_DEFINE_CONST_FUN_OBJ_1(str_lower_obj, str_lower); +MP_DEFINE_CONST_FUN_OBJ_1(str_upper_obj, str_upper); +MP_DEFINE_CONST_FUN_OBJ_1(str_isspace_obj, str_isspace); +MP_DEFINE_CONST_FUN_OBJ_1(str_isalpha_obj, str_isalpha); +MP_DEFINE_CONST_FUN_OBJ_1(str_isdigit_obj, str_isdigit); +MP_DEFINE_CONST_FUN_OBJ_1(str_isupper_obj, str_isupper); +MP_DEFINE_CONST_FUN_OBJ_1(str_islower_obj, str_islower); STATIC const mp_map_elem_t str_locals_dict_table[] = { #if MICROPY_CPYTHON_COMPAT diff --git a/py/objstr.h b/py/objstr.h index 5be137d36..2e5aaeb09 100644 --- a/py/objstr.h +++ b/py/objstr.h @@ -35,5 +35,49 @@ typedef struct _mp_obj_str_t { #define MP_DEFINE_STR_OBJ(obj_name, str) mp_obj_str_t obj_name = {{&mp_type_str}, 0, sizeof(str) - 1, (const byte*)str}; +// use this macro to extract the string hash +#define GET_STR_HASH(str_obj_in, str_hash) \ + uint str_hash; if (MP_OBJ_IS_QSTR(str_obj_in)) \ + { str_hash = qstr_hash(MP_OBJ_QSTR_VALUE(str_obj_in)); } else { str_hash = ((mp_obj_str_t*)str_obj_in)->hash; } + +// use this macro to extract the string length +#define GET_STR_LEN(str_obj_in, str_len) \ + uint str_len; if (MP_OBJ_IS_QSTR(str_obj_in)) \ + { str_len = qstr_len(MP_OBJ_QSTR_VALUE(str_obj_in)); } else { str_len = ((mp_obj_str_t*)str_obj_in)->len; } + +// use this macro to extract the string data and length +#define GET_STR_DATA_LEN(str_obj_in, str_data, str_len) \ + const byte *str_data; uint str_len; if (MP_OBJ_IS_QSTR(str_obj_in)) \ + { str_data = qstr_data(MP_OBJ_QSTR_VALUE(str_obj_in), &str_len); } \ + else { str_len = ((mp_obj_str_t*)str_obj_in)->len; str_data = ((mp_obj_str_t*)str_obj_in)->data; } + mp_obj_t mp_obj_str_format(uint n_args, const mp_obj_t *args); mp_obj_t mp_obj_new_str_of_type(const mp_obj_type_t *type, const byte* data, uint len); + +mp_obj_t str_binary_op(int op, mp_obj_t lhs_in, mp_obj_t rhs_in); + +MP_DECLARE_CONST_FUN_OBJ(str_encode_obj); +MP_DECLARE_CONST_FUN_OBJ(str_find_obj); +MP_DECLARE_CONST_FUN_OBJ(str_rfind_obj); +MP_DECLARE_CONST_FUN_OBJ(str_index_obj); +MP_DECLARE_CONST_FUN_OBJ(str_rindex_obj); +MP_DECLARE_CONST_FUN_OBJ(str_join_obj); +MP_DECLARE_CONST_FUN_OBJ(str_split_obj); +MP_DECLARE_CONST_FUN_OBJ(str_rsplit_obj); +MP_DECLARE_CONST_FUN_OBJ(str_startswith_obj); +MP_DECLARE_CONST_FUN_OBJ(str_endswith_obj); +MP_DECLARE_CONST_FUN_OBJ(str_strip_obj); +MP_DECLARE_CONST_FUN_OBJ(str_lstrip_obj); +MP_DECLARE_CONST_FUN_OBJ(str_rstrip_obj); +MP_DECLARE_CONST_FUN_OBJ(str_format_obj); +MP_DECLARE_CONST_FUN_OBJ(str_replace_obj); +MP_DECLARE_CONST_FUN_OBJ(str_count_obj); +MP_DECLARE_CONST_FUN_OBJ(str_partition_obj); +MP_DECLARE_CONST_FUN_OBJ(str_rpartition_obj); +MP_DECLARE_CONST_FUN_OBJ(str_lower_obj); +MP_DECLARE_CONST_FUN_OBJ(str_upper_obj); +MP_DECLARE_CONST_FUN_OBJ(str_isspace_obj); +MP_DECLARE_CONST_FUN_OBJ(str_isalpha_obj); +MP_DECLARE_CONST_FUN_OBJ(str_isdigit_obj); +MP_DECLARE_CONST_FUN_OBJ(str_isupper_obj); +MP_DECLARE_CONST_FUN_OBJ(str_islower_obj); diff --git a/py/objstrunicode.c b/py/objstrunicode.c index b1b61d74f..741f1b76e 100644 --- a/py/objstrunicode.c +++ b/py/objstrunicode.c @@ -40,31 +40,14 @@ #include "objstr.h" #include "objlist.h" -STATIC mp_obj_t str_modulo_format(mp_obj_t pattern, uint n_args, const mp_obj_t *args, mp_obj_t dict); -const mp_obj_t mp_const_empty_bytes; - -// use this macro to extract the string hash -#define GET_STR_HASH(str_obj_in, str_hash) uint str_hash; if (MP_OBJ_IS_QSTR(str_obj_in)) { str_hash = qstr_hash(MP_OBJ_QSTR_VALUE(str_obj_in)); } else { str_hash = ((mp_obj_str_t*)str_obj_in)->hash; } - -// use this macro to extract the string length -#define GET_STR_LEN(str_obj_in, str_len) uint str_len; if (MP_OBJ_IS_QSTR(str_obj_in)) { str_len = qstr_len(MP_OBJ_QSTR_VALUE(str_obj_in)); } else { str_len = ((mp_obj_str_t*)str_obj_in)->len; } - -// use this macro to extract the string data and length -#define GET_STR_DATA_LEN(str_obj_in, str_data, str_len) const byte *str_data; uint str_len; if (MP_OBJ_IS_QSTR(str_obj_in)) { str_data = qstr_data(MP_OBJ_QSTR_VALUE(str_obj_in), &str_len); } else { str_len = ((mp_obj_str_t*)str_obj_in)->len; str_data = ((mp_obj_str_t*)str_obj_in)->data; } +#if MICROPY_PY_BUILTINS_STR_UNICODE STATIC mp_obj_t mp_obj_new_str_iterator(mp_obj_t str); -STATIC mp_obj_t mp_obj_new_bytes_iterator(mp_obj_t str); -STATIC NORETURN void bad_implicit_conversion(mp_obj_t self_in); -STATIC NORETURN void arg_type_mixup(); - -STATIC bool is_str_or_bytes(mp_obj_t o) { - return MP_OBJ_IS_STR(o) || MP_OBJ_IS_TYPE(o, &mp_type_bytes); -} /******************************************************************************/ /* str */ -void mp_str_print_quoted(void (*print)(void *env, const char *fmt, ...), void *env, const byte *str_data, uint str_len, bool is_bytes) { +STATIC void uni_print_quoted(void (*print)(void *env, const char *fmt, ...), void *env, const byte *str_data, uint str_len, bool is_bytes) { // this escapes characters, but it will be very slow to print (calling print many times) bool has_single_quote = false; bool has_double_quote = false; @@ -121,7 +104,7 @@ STATIC void str_print(void (*print)(void *env, const char *fmt, ...), void *env, if (is_bytes) { print(env, "b"); } - mp_str_print_quoted(print, env, str_data, str_len, is_bytes); + uni_print_quoted(print, env, str_data, str_len, is_bytes); } } @@ -165,201 +148,6 @@ STATIC mp_obj_t str_make_new(mp_obj_t type_in, uint n_args, uint n_kw, const mp_ } } -STATIC mp_obj_t bytes_make_new(mp_obj_t type_in, uint n_args, uint n_kw, const mp_obj_t *args) { - if (n_args == 0) { - return mp_const_empty_bytes; - } - -#if MICROPY_CPYTHON_COMPAT - if (n_kw != 0) { - mp_arg_error_unimpl_kw(); - } -#endif - - if (MP_OBJ_IS_STR(args[0])) { - if (n_args < 2 || n_args > 3) { - goto wrong_args; - } - GET_STR_DATA_LEN(args[0], str_data, str_len); - GET_STR_HASH(args[0], str_hash); - mp_obj_str_t *o = mp_obj_new_str_of_type(&mp_type_bytes, NULL, str_len); - o->data = str_data; - o->hash = str_hash; - return o; - } - - if (n_args > 1) { - goto wrong_args; - } - - if (MP_OBJ_IS_SMALL_INT(args[0])) { - uint len = MP_OBJ_SMALL_INT_VALUE(args[0]); - byte *data; - - mp_obj_t o = mp_obj_str_builder_start(&mp_type_bytes, len, &data); - memset(data, 0, len); - return mp_obj_str_builder_end(o); - } - - int len; - byte *data; - vstr_t *vstr = NULL; - mp_obj_t o = NULL; - // Try to create array of exact len if initializer len is known - mp_obj_t len_in = mp_obj_len_maybe(args[0]); - if (len_in == MP_OBJ_NULL) { - len = -1; - vstr = vstr_new(); - } else { - len = MP_OBJ_SMALL_INT_VALUE(len_in); - o = mp_obj_str_builder_start(&mp_type_bytes, len, &data); - } - - mp_obj_t iterable = mp_getiter(args[0]); - mp_obj_t item; - while ((item = mp_iternext(iterable)) != MP_OBJ_STOP_ITERATION) { - if (len == -1) { - vstr_add_char(vstr, MP_OBJ_SMALL_INT_VALUE(item)); - } else { - *data++ = MP_OBJ_SMALL_INT_VALUE(item); - } - } - - if (len == -1) { - vstr_shrink(vstr); - // TODO: Optimize, borrow buffer from vstr - len = vstr_len(vstr); - o = mp_obj_str_builder_start(&mp_type_bytes, len, &data); - memcpy(data, vstr_str(vstr), len); - vstr_free(vstr); - } - - return mp_obj_str_builder_end(o); - -wrong_args: - nlr_raise(mp_obj_new_exception_msg(&mp_type_TypeError, "wrong number of arguments")); -} - -// like strstr but with specified length and allows \0 bytes -// TODO replace with something more efficient/standard -STATIC const byte *find_subbytes(const byte *haystack, machine_uint_t hlen, const byte *needle, machine_uint_t nlen, machine_int_t direction) { - if (hlen >= nlen) { - machine_uint_t str_index, str_index_end; - if (direction > 0) { - str_index = 0; - str_index_end = hlen - nlen; - } else { - str_index = hlen - nlen; - str_index_end = 0; - } - for (;;) { - if (memcmp(&haystack[str_index], needle, nlen) == 0) { - //found - return haystack + str_index; - } - if (str_index == str_index_end) { - //not found - break; - } - str_index += direction; - } - } - return NULL; -} - -STATIC mp_obj_t str_binary_op(int op, mp_obj_t lhs_in, mp_obj_t rhs_in) { - GET_STR_DATA_LEN(lhs_in, lhs_data, lhs_len); - mp_obj_type_t *lhs_type = mp_obj_get_type(lhs_in); - mp_obj_type_t *rhs_type = mp_obj_get_type(rhs_in); - switch (op) { - case MP_BINARY_OP_ADD: - case MP_BINARY_OP_INPLACE_ADD: - if (lhs_type == rhs_type) { - // add 2 strings or bytes - - GET_STR_DATA_LEN(rhs_in, rhs_data, rhs_len); - int alloc_len = lhs_len + rhs_len; - - /* code for making qstr - byte *q_ptr; - byte *val = qstr_build_start(alloc_len, &q_ptr); - memcpy(val, lhs_data, lhs_len); - memcpy(val + lhs_len, rhs_data, rhs_len); - return MP_OBJ_NEW_QSTR(qstr_build_end(q_ptr)); - */ - - // code for non-qstr - byte *data; - mp_obj_t s = mp_obj_str_builder_start(lhs_type, alloc_len, &data); - memcpy(data, lhs_data, lhs_len); - memcpy(data + lhs_len, rhs_data, rhs_len); - return mp_obj_str_builder_end(s); - } - break; - - case MP_BINARY_OP_IN: - /* NOTE `a in b` is `b.__contains__(a)` */ - if (lhs_type == rhs_type) { - GET_STR_DATA_LEN(rhs_in, rhs_data, rhs_len); - return MP_BOOL(find_subbytes(lhs_data, lhs_len, rhs_data, rhs_len, 1) != NULL); - } - break; - - case MP_BINARY_OP_MULTIPLY: { - if (!MP_OBJ_IS_SMALL_INT(rhs_in)) { - return MP_OBJ_NULL; // op not supported - } - int n = MP_OBJ_SMALL_INT_VALUE(rhs_in); - byte *data; - mp_obj_t s = mp_obj_str_builder_start(lhs_type, lhs_len * n, &data); - mp_seq_multiply(lhs_data, sizeof(*lhs_data), lhs_len, n, data); - return mp_obj_str_builder_end(s); - } - - case MP_BINARY_OP_MODULO: { - mp_obj_t *args; - uint n_args; - mp_obj_t dict = MP_OBJ_NULL; - if (MP_OBJ_IS_TYPE(rhs_in, &mp_type_tuple)) { - // TODO: Support tuple subclasses? - mp_obj_tuple_get(rhs_in, &n_args, &args); - } else if (MP_OBJ_IS_TYPE(rhs_in, &mp_type_dict)) { - args = NULL; - n_args = 0; - dict = rhs_in; - } else { - args = &rhs_in; - n_args = 1; - } - return str_modulo_format(lhs_in, n_args, args, dict); - } - - //case MP_BINARY_OP_NOT_EQUAL: // This is never passed here - case MP_BINARY_OP_EQUAL: // This will be passed only for bytes, str is dealt with in mp_obj_equal() - case MP_BINARY_OP_LESS: - case MP_BINARY_OP_LESS_EQUAL: - case MP_BINARY_OP_MORE: - case MP_BINARY_OP_MORE_EQUAL: - if (lhs_type == rhs_type) { - GET_STR_DATA_LEN(rhs_in, rhs_data, rhs_len); - return MP_BOOL(mp_seq_cmp_bytes(op, lhs_data, lhs_len, rhs_data, rhs_len)); - } - if (lhs_type == &mp_type_bytes) { - mp_buffer_info_t bufinfo; - if (!mp_get_buffer(rhs_in, &bufinfo, MP_BUFFER_READ)) { - goto uncomparable; - } - return MP_BOOL(mp_seq_cmp_bytes(op, lhs_data, lhs_len, bufinfo.buf, bufinfo.len)); - } -uncomparable: - if (op == MP_BINARY_OP_EQUAL) { - return mp_const_false; - } - } - - return MP_OBJ_NULL; // op not supported -} - // Convert an index into a pointer to its lead byte. Out of bounds indexing will raise IndexError or // be capped to the first/last character of the string, depending on is_slice. STATIC const char *str_index_to_ptr(const char *self_data, uint self_len, mp_obj_t index, bool is_slice) { @@ -480,1275 +268,8 @@ STATIC mp_obj_t str_subscr(mp_obj_t self_in, mp_obj_t index, mp_obj_t value) { } } -STATIC mp_obj_t str_join(mp_obj_t self_in, mp_obj_t arg) { - assert(is_str_or_bytes(self_in)); - const mp_obj_type_t *self_type = mp_obj_get_type(self_in); - - // get separation string - GET_STR_DATA_LEN(self_in, sep_str, sep_len); - - // process args - uint seq_len; - mp_obj_t *seq_items; - if (MP_OBJ_IS_TYPE(arg, &mp_type_tuple)) { - mp_obj_tuple_get(arg, &seq_len, &seq_items); - } else { - if (!MP_OBJ_IS_TYPE(arg, &mp_type_list)) { - // arg is not a list, try to convert it to one - // TODO: Try to optimize? - arg = mp_type_list.make_new((mp_obj_t)&mp_type_list, 1, 0, &arg); - } - mp_obj_list_get(arg, &seq_len, &seq_items); - } - - // count required length - int required_len = 0; - for (int i = 0; i < seq_len; i++) { - if (mp_obj_get_type(seq_items[i]) != self_type) { - nlr_raise(mp_obj_new_exception_msg(&mp_type_TypeError, - "join expects a list of str/bytes objects consistent with self object")); - } - if (i > 0) { - required_len += sep_len; - } - GET_STR_LEN(seq_items[i], l); - required_len += l; - } - - // make joined string - byte *data; - mp_obj_t joined_str = mp_obj_str_builder_start(self_type, required_len, &data); - for (int i = 0; i < seq_len; i++) { - if (i > 0) { - memcpy(data, sep_str, sep_len); - data += sep_len; - } - GET_STR_DATA_LEN(seq_items[i], s, l); - memcpy(data, s, l); - data += l; - } - - // return joined string - return mp_obj_str_builder_end(joined_str); -} - -#define is_ws(c) ((c) == ' ' || (c) == '\t') - -STATIC mp_obj_t str_split(uint n_args, const mp_obj_t *args) { - const mp_obj_type_t *self_type = mp_obj_get_type(args[0]); - machine_int_t splits = -1; - mp_obj_t sep = mp_const_none; - if (n_args > 1) { - sep = args[1]; - if (n_args > 2) { - splits = mp_obj_get_int(args[2]); - } - } - - mp_obj_t res = mp_obj_new_list(0, NULL); - GET_STR_DATA_LEN(args[0], s, len); - const byte *top = s + len; - - if (sep == mp_const_none) { - // sep not given, so separate on whitespace - - // Initial whitespace is not counted as split, so we pre-do it - while (s < top && is_ws(*s)) s++; - while (s < top && splits != 0) { - const byte *start = s; - while (s < top && !is_ws(*s)) s++; - mp_obj_list_append(res, mp_obj_new_str_of_type(self_type, start, s - start)); - if (s >= top) { - break; - } - while (s < top && is_ws(*s)) s++; - if (splits > 0) { - splits--; - } - } - - if (s < top) { - mp_obj_list_append(res, mp_obj_new_str_of_type(self_type, s, top - s)); - } - - } else { - // sep given - - uint sep_len; - const char *sep_str = mp_obj_str_get_data(sep, &sep_len); - - if (sep_len == 0) { - nlr_raise(mp_obj_new_exception_msg(&mp_type_ValueError, "empty separator")); - } - - for (;;) { - const byte *start = s; - for (;;) { - if (splits == 0 || s + sep_len > top) { - s = top; - break; - } else if (memcmp(s, sep_str, sep_len) == 0) { - break; - } - s++; - } - mp_obj_list_append(res, mp_obj_new_str_of_type(self_type, start, s - start)); - if (s >= top) { - break; - } - s += sep_len; - if (splits > 0) { - splits--; - } - } - } - - return res; -} - -STATIC mp_obj_t str_rsplit(uint n_args, const mp_obj_t *args) { - if (n_args < 3) { - // If we don't have split limit, it doesn't matter from which side - // we split. - return str_split(n_args, args); - } - const mp_obj_type_t *self_type = mp_obj_get_type(args[0]); - mp_obj_t sep = args[1]; - GET_STR_DATA_LEN(args[0], s, len); - - machine_int_t splits = mp_obj_get_int(args[2]); - machine_int_t org_splits = splits; - // Preallocate list to the max expected # of elements, as we - // will fill it from the end. - mp_obj_list_t *res = mp_obj_new_list(splits + 1, NULL); - int idx = splits; - - if (sep == mp_const_none) { - assert(!"TODO: rsplit(None,n) not implemented"); - } else { - uint sep_len; - const char *sep_str = mp_obj_str_get_data(sep, &sep_len); - - if (sep_len == 0) { - nlr_raise(mp_obj_new_exception_msg(&mp_type_ValueError, "empty separator")); - } - - const byte *beg = s; - const byte *last = s + len; - for (;;) { - s = last - sep_len; - for (;;) { - if (splits == 0 || s < beg) { - break; - } else if (memcmp(s, sep_str, sep_len) == 0) { - break; - } - s--; - } - if (s < beg || splits == 0) { - res->items[idx] = mp_obj_new_str_of_type(self_type, beg, last - beg); - break; - } - res->items[idx--] = mp_obj_new_str_of_type(self_type, s + sep_len, last - s - sep_len); - last = s; - if (splits > 0) { - splits--; - } - } - if (idx != 0) { - // We split less parts than split limit, now go cleanup surplus - int used = org_splits + 1 - idx; - memcpy(res->items, &res->items[idx], used * sizeof(mp_obj_t)); - mp_seq_clear(res->items, used, res->alloc, sizeof(*res->items)); - res->len = used; - } - } - - return res; -} - - -STATIC mp_obj_t str_finder(uint n_args, const mp_obj_t *args, machine_int_t direction, bool is_index) { - assert(2 <= n_args && n_args <= 4); - assert(MP_OBJ_IS_STR(args[0])); - assert(MP_OBJ_IS_STR(args[1])); - - GET_STR_DATA_LEN(args[0], haystack, haystack_len); - GET_STR_DATA_LEN(args[1], needle, needle_len); - - machine_uint_t start = 0; - machine_uint_t end = haystack_len; - if (n_args >= 3 && args[2] != mp_const_none) { - start = mp_get_index(&mp_type_str, haystack_len, args[2], true); - } - if (n_args >= 4 && args[3] != mp_const_none) { - end = mp_get_index(&mp_type_str, haystack_len, args[3], true); - } - - const byte *p = find_subbytes(haystack + start, end - start, needle, needle_len, direction); - if (p == NULL) { - // not found - if (is_index) { - nlr_raise(mp_obj_new_exception_msg(&mp_type_ValueError, "substring not found")); - } else { - return MP_OBJ_NEW_SMALL_INT(-1); - } - } else { - // found - return MP_OBJ_NEW_SMALL_INT(p - haystack); - } -} - -STATIC mp_obj_t str_find(uint n_args, const mp_obj_t *args) { - return str_finder(n_args, args, 1, false); -} - -STATIC mp_obj_t str_rfind(uint n_args, const mp_obj_t *args) { - return str_finder(n_args, args, -1, false); -} - -STATIC mp_obj_t str_index(uint n_args, const mp_obj_t *args) { - return str_finder(n_args, args, 1, true); -} - -STATIC mp_obj_t str_rindex(uint n_args, const mp_obj_t *args) { - return str_finder(n_args, args, -1, true); -} - -// TODO: (Much) more variety in args -STATIC mp_obj_t str_startswith(uint n_args, const mp_obj_t *args) { - GET_STR_DATA_LEN(args[0], str, str_len); - GET_STR_DATA_LEN(args[1], prefix, prefix_len); - uint index_val = 0; - if (n_args > 2) { - index_val = mp_get_index(&mp_type_str, str_len, args[2], true); - } - if (prefix_len + index_val > str_len) { - return mp_const_false; - } - return MP_BOOL(memcmp(str + index_val, prefix, prefix_len) == 0); -} - -STATIC mp_obj_t str_endswith(uint n_args, const mp_obj_t *args) { - GET_STR_DATA_LEN(args[0], str, str_len); - GET_STR_DATA_LEN(args[1], suffix, suffix_len); - assert(n_args == 2); - - if (suffix_len > str_len) { - return mp_const_false; - } - return MP_BOOL(memcmp(str + (str_len - suffix_len), suffix, suffix_len) == 0); -} - -enum { LSTRIP, RSTRIP, STRIP }; - -STATIC mp_obj_t str_uni_strip(int type, uint n_args, const mp_obj_t *args) { - assert(1 <= n_args && n_args <= 2); - assert(is_str_or_bytes(args[0])); - const mp_obj_type_t *self_type = mp_obj_get_type(args[0]); - - const byte *chars_to_del; - uint chars_to_del_len; - static const byte whitespace[] = " \t\n\r\v\f"; - - if (n_args == 1) { - chars_to_del = whitespace; - chars_to_del_len = sizeof(whitespace); - } else { - if (mp_obj_get_type(args[1]) != self_type) { - arg_type_mixup(); - } - GET_STR_DATA_LEN(args[1], s, l); - chars_to_del = s; - chars_to_del_len = l; - } - - GET_STR_DATA_LEN(args[0], orig_str, orig_str_len); - - machine_uint_t first_good_char_pos = 0; - bool first_good_char_pos_set = false; - machine_uint_t last_good_char_pos = 0; - machine_uint_t i = 0; - machine_int_t delta = 1; - if (type == RSTRIP) { - i = orig_str_len - 1; - delta = -1; - } - for (machine_uint_t len = orig_str_len; len > 0; len--) { - if (find_subbytes(chars_to_del, chars_to_del_len, &orig_str[i], 1, 1) == NULL) { - if (!first_good_char_pos_set) { - first_good_char_pos_set = true; - first_good_char_pos = i; - if (type == LSTRIP) { - last_good_char_pos = orig_str_len - 1; - break; - } else if (type == RSTRIP) { - first_good_char_pos = 0; - last_good_char_pos = i; - break; - } - } - last_good_char_pos = i; - } - i += delta; - } - - if (!first_good_char_pos_set) { - // string is all whitespace, return '' - return MP_OBJ_NEW_QSTR(MP_QSTR_); - } - - assert(last_good_char_pos >= first_good_char_pos); - //+1 to accomodate the last character - machine_uint_t stripped_len = last_good_char_pos - first_good_char_pos + 1; - if (stripped_len == orig_str_len) { - // If nothing was stripped, don't bother to dup original string - // TODO: watch out for this case when we'll get to bytearray.strip() - assert(first_good_char_pos == 0); - return args[0]; - } - return mp_obj_new_str_of_type(self_type, orig_str + first_good_char_pos, stripped_len); -} - -STATIC mp_obj_t str_strip(uint n_args, const mp_obj_t *args) { - return str_uni_strip(STRIP, n_args, args); -} - -STATIC mp_obj_t str_lstrip(uint n_args, const mp_obj_t *args) { - return str_uni_strip(LSTRIP, n_args, args); -} - -STATIC mp_obj_t str_rstrip(uint n_args, const mp_obj_t *args) { - return str_uni_strip(RSTRIP, n_args, args); -} - -// Takes an int arg, but only parses unsigned numbers, and only changes -// *num if at least one digit was parsed. -static int str_to_int(const char *str, int *num) { - const char *s = str; - if (unichar_isdigit(*s)) { - *num = 0; - do { - *num = *num * 10 + (*s - '0'); - s++; - } - while (unichar_isdigit(*s)); - } - return s - str; -} - -static bool isalignment(char ch) { - return ch && strchr("<>=^", ch) != NULL; -} - -static bool istype(char ch) { - return ch && strchr("bcdeEfFgGnosxX%", ch) != NULL; -} - -static bool arg_looks_integer(mp_obj_t arg) { - return MP_OBJ_IS_TYPE(arg, &mp_type_bool) || MP_OBJ_IS_INT(arg); -} - -static bool arg_looks_numeric(mp_obj_t arg) { - return arg_looks_integer(arg) -#if MICROPY_PY_BUILTINS_FLOAT - || MP_OBJ_IS_TYPE(arg, &mp_type_float) -#endif - ; -} - -static mp_obj_t arg_as_int(mp_obj_t arg) { -#if MICROPY_PY_BUILTINS_FLOAT - if (MP_OBJ_IS_TYPE(arg, &mp_type_float)) { - - // TODO: Needs a way to construct an mpz integer from a float - - mp_small_int_t num = mp_obj_get_float(arg); - return MP_OBJ_NEW_SMALL_INT(num); - } -#endif - return arg; -} - -mp_obj_t mp_obj_str_format(uint n_args, const mp_obj_t *args) { - assert(MP_OBJ_IS_STR(args[0])); - - GET_STR_DATA_LEN(args[0], str, len); - int arg_i = 0; - vstr_t *vstr = vstr_new(); - pfenv_t pfenv_vstr; - pfenv_vstr.data = vstr; - pfenv_vstr.print_strn = pfenv_vstr_add_strn; - - for (const byte *top = str + len; str < top; str++) { - if (*str == '}') { - str++; - if (str < top && *str == '}') { - vstr_add_char(vstr, '}'); - continue; - } - nlr_raise(mp_obj_new_exception_msg(&mp_type_ValueError, "single '}' encountered in format string")); - } - if (*str != '{') { - vstr_add_char(vstr, *str); - continue; - } - - str++; - if (str < top && *str == '{') { - vstr_add_char(vstr, '{'); - continue; - } - - // replacement_field ::= "{" [field_name] ["!" conversion] [":" format_spec] "}" - - vstr_t *field_name = NULL; - char conversion = '\0'; - vstr_t *format_spec = NULL; - - if (str < top && *str != '}' && *str != '!' && *str != ':') { - field_name = vstr_new(); - while (str < top && *str != '}' && *str != '!' && *str != ':') { - vstr_add_char(field_name, *str++); - } - vstr_add_char(field_name, '\0'); - } - - // conversion ::= "r" | "s" - - if (str < top && *str == '!') { - str++; - if (str < top && (*str == 'r' || *str == 's')) { - conversion = *str++; - } else { - nlr_raise(mp_obj_new_exception_msg(&mp_type_ValueError, "end of format while looking for conversion specifier")); - } - } - - if (str < top && *str == ':') { - str++; - // {:} is the same as {}, which is the same as {!s} - // This makes a difference when passing in a True or False - // '{}'.format(True) returns 'True' - // '{:d}'.format(True) returns '1' - // So we treat {:} as {} and this later gets treated to be {!s} - if (*str != '}') { - format_spec = vstr_new(); - while (str < top && *str != '}') { - vstr_add_char(format_spec, *str++); - } - vstr_add_char(format_spec, '\0'); - } - } - if (str >= top) { - nlr_raise(mp_obj_new_exception_msg(&mp_type_ValueError, "unmatched '{' in format")); - } - if (*str != '}') { - nlr_raise(mp_obj_new_exception_msg(&mp_type_ValueError, "expected ':' after format specifier")); - } - - mp_obj_t arg = mp_const_none; - - if (field_name) { - if (arg_i > 0) { - nlr_raise(mp_obj_new_exception_msg(&mp_type_ValueError, "can't switch from automatic field numbering to manual field specification")); - } - int index = 0; - if (str_to_int(vstr_str(field_name), &index) != vstr_len(field_name) - 1) { - nlr_raise(mp_obj_new_exception_msg(&mp_type_KeyError, "attributes not supported yet")); - } - if (index >= n_args - 1) { - nlr_raise(mp_obj_new_exception_msg(&mp_type_IndexError, "tuple index out of range")); - } - arg = args[index + 1]; - arg_i = -1; - vstr_free(field_name); - field_name = NULL; - } else { - if (arg_i < 0) { - nlr_raise(mp_obj_new_exception_msg(&mp_type_ValueError, "can't switch from manual field specification to automatic field numbering")); - } - if (arg_i >= n_args - 1) { - nlr_raise(mp_obj_new_exception_msg(&mp_type_IndexError, "tuple index out of range")); - } - arg = args[arg_i + 1]; - arg_i++; - } - if (!format_spec && !conversion) { - conversion = 's'; - } - if (conversion) { - mp_print_kind_t print_kind; - if (conversion == 's') { - print_kind = PRINT_STR; - } else if (conversion == 'r') { - print_kind = PRINT_REPR; - } else { - nlr_raise(mp_obj_new_exception_msg_varg(&mp_type_ValueError, "unknown conversion specifier %c", conversion)); - } - vstr_t *arg_vstr = vstr_new(); - mp_obj_print_helper((void (*)(void*, const char*, ...))vstr_printf, arg_vstr, arg, print_kind); - arg = mp_obj_new_str(vstr_str(arg_vstr), vstr_len(arg_vstr), false); - vstr_free(arg_vstr); - } - - char sign = '\0'; - char fill = '\0'; - char align = '\0'; - int width = -1; - int precision = -1; - char type = '\0'; - int flags = 0; - - if (format_spec) { - // The format specifier (from http://docs.python.org/2/library/string.html#formatspec) - // - // [[fill]align][sign][#][0][width][,][.precision][type] - // fill ::= - // align ::= "<" | ">" | "=" | "^" - // sign ::= "+" | "-" | " " - // width ::= integer - // precision ::= integer - // type ::= "b" | "c" | "d" | "e" | "E" | "f" | "F" | "g" | "G" | "n" | "o" | "s" | "x" | "X" | "%" - - const char *s = vstr_str(format_spec); - if (isalignment(*s)) { - align = *s++; - } else if (*s && isalignment(s[1])) { - fill = *s++; - align = *s++; - } - if (*s == '+' || *s == '-' || *s == ' ') { - if (*s == '+') { - flags |= PF_FLAG_SHOW_SIGN; - } else if (*s == ' ') { - flags |= PF_FLAG_SPACE_SIGN; - } - sign = *s++; - } - if (*s == '#') { - flags |= PF_FLAG_SHOW_PREFIX; - s++; - } - if (*s == '0') { - if (!align) { - align = '='; - } - if (!fill) { - fill = '0'; - } - } - s += str_to_int(s, &width); - if (*s == ',') { - flags |= PF_FLAG_SHOW_COMMA; - s++; - } - if (*s == '.') { - s++; - s += str_to_int(s, &precision); - } - if (istype(*s)) { - type = *s++; - } - if (*s) { - nlr_raise(mp_obj_new_exception_msg(&mp_type_KeyError, "Invalid conversion specification")); - } - vstr_free(format_spec); - format_spec = NULL; - } - if (!align) { - if (arg_looks_numeric(arg)) { - align = '>'; - } else { - align = '<'; - } - } - if (!fill) { - fill = ' '; - } - - if (sign) { - if (type == 's') { - nlr_raise(mp_obj_new_exception_msg(&mp_type_ValueError, "Sign not allowed in string format specifier")); - } - if (type == 'c') { - nlr_raise(mp_obj_new_exception_msg(&mp_type_ValueError, "Sign not allowed with integer format specifier 'c'")); - } - } else { - sign = '-'; - } - - switch (align) { - case '<': flags |= PF_FLAG_LEFT_ADJUST; break; - case '=': flags |= PF_FLAG_PAD_AFTER_SIGN; break; - case '^': flags |= PF_FLAG_CENTER_ADJUST; break; - } - - if (arg_looks_integer(arg)) { - switch (type) { - case 'b': - pfenv_print_mp_int(&pfenv_vstr, arg, 1, 2, 'a', flags, fill, width, 0); - continue; - - case 'c': - { - char ch = mp_obj_get_int(arg); - pfenv_print_strn(&pfenv_vstr, &ch, 1, flags, fill, width); - continue; - } - - case '\0': // No explicit format type implies 'd' - case 'n': // I don't think we support locales in uPy so use 'd' - case 'd': - pfenv_print_mp_int(&pfenv_vstr, arg, 1, 10, 'a', flags, fill, width, 0); - continue; - - case 'o': - if (flags & PF_FLAG_SHOW_PREFIX) { - flags |= PF_FLAG_SHOW_OCTAL_LETTER; - } - - pfenv_print_mp_int(&pfenv_vstr, arg, 1, 8, 'a', flags, fill, width, 0); - continue; - - case 'X': - case 'x': - pfenv_print_mp_int(&pfenv_vstr, arg, 1, 16, type - ('X' - 'A'), flags, fill, width, 0); - continue; - - case 'e': - case 'E': - case 'f': - case 'F': - case 'g': - case 'G': - case '%': - // The floating point formatters all work with anything that - // looks like an integer - break; - - default: - nlr_raise(mp_obj_new_exception_msg_varg(&mp_type_ValueError, - "unknown format code '%c' for object of type '%s'", type, mp_obj_get_type_str(arg))); - } - } - - // NOTE: no else here. We need the e, f, g etc formats for integer - // arguments (from above if) to take this if. - if (arg_looks_numeric(arg)) { - if (!type) { - - // Even though the docs say that an unspecified type is the same - // as 'g', there is one subtle difference, when the exponent - // is one less than the precision. - // - // '{:10.1}'.format(0.0) ==> '0e+00' - // '{:10.1g}'.format(0.0) ==> '0' - // - // TODO: Figure out how to deal with this. - // - // A proper solution would involve adding a special flag - // or something to format_float, and create a format_double - // to deal with doubles. In order to fix this when using - // sprintf, we'd need to use the e format and tweak the - // returned result to strip trailing zeros like the g format - // does. - // - // {:10.3} and {:10.2e} with 1.23e2 both produce 1.23e+02 - // but with 1.e2 you get 1e+02 and 1.00e+02 - // - // Stripping the trailing 0's (like g) does would make the - // e format give us the right format. - // - // CPython sources say: - // Omitted type specifier. Behaves in the same way as repr(x) - // and str(x) if no precision is given, else like 'g', but with - // at least one digit after the decimal point. */ - - type = 'g'; - } - if (type == 'n') { - type = 'g'; - } - - flags |= PF_FLAG_PAD_NAN_INF; // '{:06e}'.format(float('-inf')) should give '-00inf' - switch (type) { -#if MICROPY_PY_BUILTINS_FLOAT - case 'e': - case 'E': - case 'f': - case 'F': - case 'g': - case 'G': - pfenv_print_float(&pfenv_vstr, mp_obj_get_float(arg), type, flags, fill, width, precision); - break; - - case '%': - flags |= PF_FLAG_ADD_PERCENT; - pfenv_print_float(&pfenv_vstr, mp_obj_get_float(arg) * 100.0F, 'f', flags, fill, width, precision); - break; -#endif - - default: - nlr_raise(mp_obj_new_exception_msg_varg(&mp_type_ValueError, - "unknown format code '%c' for object of type 'float'", - type, mp_obj_get_type_str(arg))); - } - } else { - // arg doesn't look like a number - - if (align == '=') { - nlr_raise(mp_obj_new_exception_msg(&mp_type_ValueError, "'=' alignment not allowed in string format specifier")); - } - - switch (type) { - case '\0': - mp_obj_print_helper((void (*)(void*, const char*, ...))vstr_printf, vstr, arg, PRINT_STR); - break; - - case 's': - { - uint len; - const char *s = mp_obj_str_get_data(arg, &len); - if (precision < 0) { - precision = len; - } - if (len > precision) { - len = precision; - } - pfenv_print_strn(&pfenv_vstr, s, len, flags, fill, width); - break; - } - - default: - nlr_raise(mp_obj_new_exception_msg_varg(&mp_type_ValueError, - "unknown format code '%c' for object of type 'str'", - type, mp_obj_get_type_str(arg))); - } - } - } - - mp_obj_t s = mp_obj_new_str(vstr->buf, vstr->len, false); - vstr_free(vstr); - return s; -} - -STATIC mp_obj_t str_modulo_format(mp_obj_t pattern, uint n_args, const mp_obj_t *args, mp_obj_t dict) { - assert(MP_OBJ_IS_STR(pattern)); - - GET_STR_DATA_LEN(pattern, str, len); - const byte *start_str = str; - int arg_i = 0; - vstr_t *vstr = vstr_new(); - pfenv_t pfenv_vstr; - pfenv_vstr.data = vstr; - pfenv_vstr.print_strn = pfenv_vstr_add_strn; - - for (const byte *top = str + len; str < top; str++) { - mp_obj_t arg = MP_OBJ_NULL; - if (*str != '%') { - vstr_add_char(vstr, *str); - continue; - } - if (++str >= top) { - break; - } - if (*str == '%') { - vstr_add_char(vstr, '%'); - continue; - } - - // Dictionary value lookup - if (*str == '(') { - const byte *key = ++str; - while (*str != ')') { - if (str >= top) { - nlr_raise(mp_obj_new_exception_msg(&mp_type_ValueError, "incomplete format key")); - } - ++str; - } - mp_obj_t k_obj = mp_obj_new_str((const char*)key, str - key, true); - arg = mp_obj_dict_get(dict, k_obj); - str++; - } - - int flags = 0; - char fill = ' '; - int alt = 0; - while (str < top) { - if (*str == '-') flags |= PF_FLAG_LEFT_ADJUST; - else if (*str == '+') flags |= PF_FLAG_SHOW_SIGN; - else if (*str == ' ') flags |= PF_FLAG_SPACE_SIGN; - else if (*str == '#') alt = PF_FLAG_SHOW_PREFIX; - else if (*str == '0') { - flags |= PF_FLAG_PAD_AFTER_SIGN; - fill = '0'; - } else break; - str++; - } - // parse width, if it exists - int width = 0; - if (str < top) { - if (*str == '*') { - if (arg_i >= n_args) { - goto not_enough_args; - } - width = mp_obj_get_int(args[arg_i++]); - str++; - } else { - for (; str < top && '0' <= *str && *str <= '9'; str++) { - width = width * 10 + *str - '0'; - } - } - } - int prec = -1; - if (str < top && *str == '.') { - if (++str < top) { - if (*str == '*') { - if (arg_i >= n_args) { - goto not_enough_args; - } - prec = mp_obj_get_int(args[arg_i++]); - str++; - } else { - prec = 0; - for (; str < top && '0' <= *str && *str <= '9'; str++) { - prec = prec * 10 + *str - '0'; - } - } - } - } - - if (str >= top) { - nlr_raise(mp_obj_new_exception_msg(&mp_type_ValueError, "incomplete format")); - } - - // Tuple value lookup - if (arg == MP_OBJ_NULL) { - if (arg_i >= n_args) { -not_enough_args: - nlr_raise(mp_obj_new_exception_msg(&mp_type_TypeError, "not enough arguments for format string")); - } - arg = args[arg_i++]; - } - switch (*str) { - case 'c': - if (MP_OBJ_IS_STR(arg)) { - uint len; - const char *s = mp_obj_str_get_data(arg, &len); - if (len != 1) { - nlr_raise(mp_obj_new_exception_msg(&mp_type_TypeError, "%%c requires int or char")); - break; - } - pfenv_print_strn(&pfenv_vstr, s, 1, flags, ' ', width); - break; - } - if (arg_looks_integer(arg)) { - char ch = mp_obj_get_int(arg); - pfenv_print_strn(&pfenv_vstr, &ch, 1, flags, ' ', width); - break; - } -#if MICROPY_PY_BUILTINS_FLOAT - // This is what CPython reports, so we report the same. - if (MP_OBJ_IS_TYPE(arg, &mp_type_float)) { - nlr_raise(mp_obj_new_exception_msg(&mp_type_TypeError, "integer argument expected, got float")); - - } -#endif - nlr_raise(mp_obj_new_exception_msg(&mp_type_TypeError, "an integer is required")); - break; - - case 'd': - case 'i': - case 'u': - pfenv_print_mp_int(&pfenv_vstr, arg_as_int(arg), 1, 10, 'a', flags, fill, width, prec); - break; - -#if MICROPY_PY_BUILTINS_FLOAT - case 'e': - case 'E': - case 'f': - case 'F': - case 'g': - case 'G': - pfenv_print_float(&pfenv_vstr, mp_obj_get_float(arg), *str, flags, fill, width, prec); - break; -#endif - - case 'o': - if (alt) { - flags |= (PF_FLAG_SHOW_PREFIX | PF_FLAG_SHOW_OCTAL_LETTER); - } - pfenv_print_mp_int(&pfenv_vstr, arg, 1, 8, 'a', flags, fill, width, prec); - break; - - case 'r': - case 's': - { - vstr_t *arg_vstr = vstr_new(); - mp_obj_print_helper((void (*)(void*, const char*, ...))vstr_printf, - arg_vstr, arg, *str == 'r' ? PRINT_REPR : PRINT_STR); - uint len = vstr_len(arg_vstr); - if (prec < 0) { - prec = len; - } - if (len > prec) { - len = prec; - } - pfenv_print_strn(&pfenv_vstr, vstr_str(arg_vstr), len, flags, ' ', width); - vstr_free(arg_vstr); - break; - } - - case 'X': - case 'x': - pfenv_print_mp_int(&pfenv_vstr, arg, 1, 16, *str - ('X' - 'A'), flags | alt, fill, width, prec); - break; - - default: - nlr_raise(mp_obj_new_exception_msg_varg(&mp_type_ValueError, - "unsupported format character '%c' (0x%x) at index %d", - *str, *str, str - start_str)); - } - } - - if (arg_i != n_args) { - nlr_raise(mp_obj_new_exception_msg(&mp_type_TypeError, "not all arguments converted during string formatting")); - } - - mp_obj_t s = mp_obj_new_str(vstr->buf, vstr->len, false); - vstr_free(vstr); - return s; -} - -STATIC mp_obj_t str_replace(uint n_args, const mp_obj_t *args) { - assert(MP_OBJ_IS_STR(args[0])); - - machine_int_t max_rep = -1; - if (n_args == 4) { - max_rep = mp_obj_get_int(args[3]); - if (max_rep == 0) { - return args[0]; - } else if (max_rep < 0) { - max_rep = -1; - } - } - - // if max_rep is still -1 by this point we will need to do all possible replacements - - // check argument types - - if (!MP_OBJ_IS_STR(args[1])) { - bad_implicit_conversion(args[1]); - } - - if (!MP_OBJ_IS_STR(args[2])) { - bad_implicit_conversion(args[2]); - } - - // extract string data - - GET_STR_DATA_LEN(args[0], str, str_len); - GET_STR_DATA_LEN(args[1], old, old_len); - GET_STR_DATA_LEN(args[2], new, new_len); - - // old won't exist in str if it's longer, so nothing to replace - if (old_len > str_len) { - return args[0]; - } - - // data for the replaced string - byte *data = NULL; - mp_obj_t replaced_str = MP_OBJ_NULL; - - // do 2 passes over the string: - // first pass computes the required length of the replaced string - // second pass does the replacements - for (;;) { - machine_uint_t replaced_str_index = 0; - machine_uint_t num_replacements_done = 0; - const byte *old_occurrence; - const byte *offset_ptr = str; - machine_uint_t str_len_remain = str_len; - if (old_len == 0) { - // if old_str is empty, copy new_str to start of replaced string - // copy the replacement string - if (data != NULL) { - memcpy(data, new, new_len); - } - replaced_str_index += new_len; - num_replacements_done++; - } - while (num_replacements_done != max_rep && str_len_remain > 0 && (old_occurrence = find_subbytes(offset_ptr, str_len_remain, old, old_len, 1)) != NULL) { - if (old_len == 0) { - old_occurrence += 1; - } - // copy from just after end of last occurrence of to-be-replaced string to right before start of next occurrence - if (data != NULL) { - memcpy(data + replaced_str_index, offset_ptr, old_occurrence - offset_ptr); - } - replaced_str_index += old_occurrence - offset_ptr; - // copy the replacement string - if (data != NULL) { - memcpy(data + replaced_str_index, new, new_len); - } - replaced_str_index += new_len; - offset_ptr = old_occurrence + old_len; - str_len_remain = str + str_len - offset_ptr; - num_replacements_done++; - } - - // copy from just after end of last occurrence of to-be-replaced string to end of old string - if (data != NULL) { - memcpy(data + replaced_str_index, offset_ptr, str_len_remain); - } - replaced_str_index += str_len_remain; - - if (data == NULL) { - // first pass - if (num_replacements_done == 0) { - // no substr found, return original string - return args[0]; - } else { - // substr found, allocate new string - replaced_str = mp_obj_str_builder_start(mp_obj_get_type(args[0]), replaced_str_index, &data); - assert(data != NULL); - } - } else { - // second pass, we are done - break; - } - } - - return mp_obj_str_builder_end(replaced_str); -} - -STATIC mp_obj_t str_count(uint n_args, const mp_obj_t *args) { - assert(2 <= n_args && n_args <= 4); - assert(MP_OBJ_IS_STR(args[0])); - assert(MP_OBJ_IS_STR(args[1])); - - GET_STR_DATA_LEN(args[0], haystack, haystack_len); - GET_STR_DATA_LEN(args[1], needle, needle_len); - - machine_uint_t start = 0; - machine_uint_t end = haystack_len; - if (n_args >= 3 && args[2] != mp_const_none) { - start = mp_get_index(&mp_type_str, haystack_len, args[2], true); - } - if (n_args >= 4 && args[3] != mp_const_none) { - end = mp_get_index(&mp_type_str, haystack_len, args[3], true); - } - - // if needle_len is zero then we count each gap between characters as an occurrence - if (needle_len == 0) { - return MP_OBJ_NEW_SMALL_INT(end - start + 1); - } - - // count the occurrences - machine_int_t num_occurrences = 0; - for (machine_uint_t haystack_index = start; haystack_index + needle_len <= end; haystack_index++) { - if (memcmp(&haystack[haystack_index], needle, needle_len) == 0) { - num_occurrences++; - haystack_index += needle_len - 1; - } - } - - return MP_OBJ_NEW_SMALL_INT(num_occurrences); -} - -STATIC mp_obj_t str_partitioner(mp_obj_t self_in, mp_obj_t arg, machine_int_t direction) { - if (!is_str_or_bytes(self_in)) { - assert(0); - } - mp_obj_type_t *self_type = mp_obj_get_type(self_in); - if (self_type != mp_obj_get_type(arg)) { - arg_type_mixup(); - } - - GET_STR_DATA_LEN(self_in, str, str_len); - GET_STR_DATA_LEN(arg, sep, sep_len); - - if (sep_len == 0) { - nlr_raise(mp_obj_new_exception_msg(&mp_type_ValueError, "empty separator")); - } - - mp_obj_t result[] = {MP_OBJ_NEW_QSTR(MP_QSTR_), MP_OBJ_NEW_QSTR(MP_QSTR_), MP_OBJ_NEW_QSTR(MP_QSTR_)}; - - if (direction > 0) { - result[0] = self_in; - } else { - result[2] = self_in; - } - - const byte *position_ptr = find_subbytes(str, str_len, sep, sep_len, direction); - if (position_ptr != NULL) { - machine_uint_t position = position_ptr - str; - result[0] = mp_obj_new_str_of_type(self_type, str, position); - result[1] = arg; - result[2] = mp_obj_new_str_of_type(self_type, str + position + sep_len, str_len - position - sep_len); - } - - return mp_obj_new_tuple(3, result); -} - -STATIC mp_obj_t str_partition(mp_obj_t self_in, mp_obj_t arg) { - return str_partitioner(self_in, arg, 1); -} - -STATIC mp_obj_t str_rpartition(mp_obj_t self_in, mp_obj_t arg) { - return str_partitioner(self_in, arg, -1); -} - -// Supposedly not too critical operations, so optimize for code size -STATIC mp_obj_t str_caseconv(unichar (*op)(unichar), mp_obj_t self_in) { - GET_STR_DATA_LEN(self_in, self_data, self_len); - byte *data; - mp_obj_t s = mp_obj_str_builder_start(mp_obj_get_type(self_in), self_len, &data); - for (int i = 0; i < self_len; i++) { - *data++ = op(*self_data++); - } - *data = 0; - return mp_obj_str_builder_end(s); -} - -STATIC mp_obj_t str_lower(mp_obj_t self_in) { - return str_caseconv(unichar_tolower, self_in); -} - -STATIC mp_obj_t str_upper(mp_obj_t self_in) { - return str_caseconv(unichar_toupper, self_in); -} - -STATIC mp_obj_t str_uni_istype(bool (*f)(unichar), mp_obj_t self_in) { - GET_STR_DATA_LEN(self_in, self_data, self_len); - - if (self_len == 0) { - return mp_const_false; // default to False for empty str - } - - if (f != unichar_isupper && f != unichar_islower) { - for (int i = 0; i < self_len; i++) { - if (!f(*self_data++)) { - return mp_const_false; - } - } - } else { - bool contains_alpha = false; - - for (int i = 0; i < self_len; i++) { // only check alphanumeric characters - if (unichar_isalpha(*self_data++)) { - contains_alpha = true; - if (!f(*(self_data - 1))) { // -1 because we already incremented above - return mp_const_false; - } - } - } - - if (!contains_alpha) { - return mp_const_false; - } - } - - return mp_const_true; -} - -STATIC mp_obj_t str_isspace(mp_obj_t self_in) { - return str_uni_istype(unichar_isspace, self_in); -} - -STATIC mp_obj_t str_isalpha(mp_obj_t self_in) { - return str_uni_istype(unichar_isalpha, self_in); -} - -STATIC mp_obj_t str_isdigit(mp_obj_t self_in) { - return str_uni_istype(unichar_isdigit, self_in); -} - -STATIC mp_obj_t str_isupper(mp_obj_t self_in) { - return str_uni_istype(unichar_isupper, self_in); -} - -STATIC mp_obj_t str_islower(mp_obj_t self_in) { - return str_uni_istype(unichar_islower, self_in); -} - -#if MICROPY_CPYTHON_COMPAT -// These methods are superfluous in the presense of str() and bytes() -// constructors. -// TODO: should accept kwargs too -STATIC mp_obj_t bytes_decode(uint n_args, const mp_obj_t *args) { - mp_obj_t new_args[2]; - if (n_args == 1) { - new_args[0] = args[0]; - new_args[1] = MP_OBJ_NEW_QSTR(MP_QSTR_utf_hyphen_8); - args = new_args; - n_args++; - } - return str_make_new(NULL, n_args, 0, args); -} - -// TODO: should accept kwargs too -STATIC mp_obj_t str_encode(uint n_args, const mp_obj_t *args) { - mp_obj_t new_args[2]; - if (n_args == 1) { - new_args[0] = args[0]; - new_args[1] = MP_OBJ_NEW_QSTR(MP_QSTR_utf_hyphen_8); - args = new_args; - n_args++; - } - return bytes_make_new(NULL, n_args, 0, args); -} -#endif - -STATIC machine_int_t str_get_buffer(mp_obj_t self_in, mp_buffer_info_t *bufinfo, int flags) { - if (flags == MP_BUFFER_READ) { - GET_STR_DATA_LEN(self_in, str_data, str_len); - bufinfo->buf = (void*)str_data; - bufinfo->len = str_len; - bufinfo->typecode = 'b'; - return 0; - } else { - // can't write to a string - bufinfo->buf = NULL; - bufinfo->len = 0; - bufinfo->typecode = -1; - return 1; - } -} - -#if MICROPY_CPYTHON_COMPAT -STATIC MP_DEFINE_CONST_FUN_OBJ_VAR_BETWEEN(bytes_decode_obj, 1, 3, bytes_decode); -STATIC MP_DEFINE_CONST_FUN_OBJ_VAR_BETWEEN(str_encode_obj, 1, 3, str_encode); -#endif -STATIC MP_DEFINE_CONST_FUN_OBJ_VAR_BETWEEN(str_find_obj, 2, 4, str_find); -STATIC MP_DEFINE_CONST_FUN_OBJ_VAR_BETWEEN(str_rfind_obj, 2, 4, str_rfind); -STATIC MP_DEFINE_CONST_FUN_OBJ_VAR_BETWEEN(str_index_obj, 2, 4, str_index); -STATIC MP_DEFINE_CONST_FUN_OBJ_VAR_BETWEEN(str_rindex_obj, 2, 4, str_rindex); -STATIC MP_DEFINE_CONST_FUN_OBJ_2(str_join_obj, str_join); -STATIC MP_DEFINE_CONST_FUN_OBJ_VAR_BETWEEN(str_split_obj, 1, 3, str_split); -STATIC MP_DEFINE_CONST_FUN_OBJ_VAR_BETWEEN(str_rsplit_obj, 1, 3, str_rsplit); -STATIC MP_DEFINE_CONST_FUN_OBJ_VAR_BETWEEN(str_startswith_obj, 2, 3, str_startswith); -STATIC MP_DEFINE_CONST_FUN_OBJ_VAR_BETWEEN(str_endswith_obj, 2, 3, str_endswith); -STATIC MP_DEFINE_CONST_FUN_OBJ_VAR_BETWEEN(str_strip_obj, 1, 2, str_strip); -STATIC MP_DEFINE_CONST_FUN_OBJ_VAR_BETWEEN(str_lstrip_obj, 1, 2, str_lstrip); -STATIC MP_DEFINE_CONST_FUN_OBJ_VAR_BETWEEN(str_rstrip_obj, 1, 2, str_rstrip); -STATIC MP_DEFINE_CONST_FUN_OBJ_VAR(str_format_obj, 1, mp_obj_str_format); -STATIC MP_DEFINE_CONST_FUN_OBJ_VAR_BETWEEN(str_replace_obj, 3, 4, str_replace); -STATIC MP_DEFINE_CONST_FUN_OBJ_VAR_BETWEEN(str_count_obj, 2, 4, str_count); -STATIC MP_DEFINE_CONST_FUN_OBJ_2(str_partition_obj, str_partition); -STATIC MP_DEFINE_CONST_FUN_OBJ_2(str_rpartition_obj, str_rpartition); -STATIC MP_DEFINE_CONST_FUN_OBJ_1(str_lower_obj, str_lower); -STATIC MP_DEFINE_CONST_FUN_OBJ_1(str_upper_obj, str_upper); -STATIC MP_DEFINE_CONST_FUN_OBJ_1(str_isspace_obj, str_isspace); -STATIC MP_DEFINE_CONST_FUN_OBJ_1(str_isalpha_obj, str_isalpha); -STATIC MP_DEFINE_CONST_FUN_OBJ_1(str_isdigit_obj, str_isdigit); -STATIC MP_DEFINE_CONST_FUN_OBJ_1(str_isupper_obj, str_isupper); -STATIC MP_DEFINE_CONST_FUN_OBJ_1(str_islower_obj, str_islower); - STATIC const mp_map_elem_t str_locals_dict_table[] = { #if MICROPY_CPYTHON_COMPAT - { MP_OBJ_NEW_QSTR(MP_QSTR_decode), (mp_obj_t)&bytes_decode_obj }, { MP_OBJ_NEW_QSTR(MP_QSTR_encode), (mp_obj_t)&str_encode_obj }, #endif { MP_OBJ_NEW_QSTR(MP_QSTR_find), (mp_obj_t)&str_find_obj }, @@ -1787,167 +308,10 @@ const mp_obj_type_t mp_type_str = { .binary_op = str_binary_op, .subscr = str_subscr, .getiter = mp_obj_new_str_iterator, - .buffer_p = { .get_buffer = str_get_buffer }, +// .buffer_p = { .get_buffer = str_get_buffer }, .locals_dict = (mp_obj_t)&str_locals_dict, }; -// Reuses most of methods from str -const mp_obj_type_t mp_type_bytes = { - { &mp_type_type }, - .name = MP_QSTR_bytes, - .print = str_print, - .make_new = bytes_make_new, - .binary_op = str_binary_op, - .subscr = str_subscr, - .getiter = mp_obj_new_bytes_iterator, - .buffer_p = { .get_buffer = str_get_buffer }, - .locals_dict = (mp_obj_t)&str_locals_dict, -}; - -// the zero-length bytes -STATIC const mp_obj_str_t empty_bytes_obj = {{&mp_type_bytes}, 0, 0, NULL}; -const mp_obj_t mp_const_empty_bytes = (mp_obj_t)&empty_bytes_obj; - -mp_obj_t mp_obj_str_builder_start(const mp_obj_type_t *type, uint len, byte **data) { - mp_obj_str_t *o = m_new_obj(mp_obj_str_t); - o->base.type = type; - o->len = len; - o->hash = 0; - byte *p = m_new(byte, len + 1); - o->data = p; - *data = p; - return o; -} - -mp_obj_t mp_obj_str_builder_end(mp_obj_t o_in) { - mp_obj_str_t *o = o_in; - o->hash = qstr_compute_hash(o->data, o->len); - byte *p = (byte*)o->data; - p[o->len] = '\0'; // for now we add null for compatibility with C ASCIIZ strings - return o; -} - -mp_obj_t mp_obj_new_str_of_type(const mp_obj_type_t *type, const byte* data, uint len) { - mp_obj_str_t *o = m_new_obj(mp_obj_str_t); - o->base.type = type; - o->len = len; - if (data) { - o->hash = qstr_compute_hash(data, len); - byte *p = m_new(byte, len + 1); - o->data = p; - memcpy(p, data, len * sizeof(byte)); - p[len] = '\0'; // for now we add null for compatibility with C ASCIIZ strings - } - return o; -} - -mp_obj_t mp_obj_new_str(const char* data, uint len, bool make_qstr_if_not_already) { - if (make_qstr_if_not_already) { - // use existing, or make a new qstr - return MP_OBJ_NEW_QSTR(qstr_from_strn(data, len)); - } else { - qstr q = qstr_find_strn(data, len); - if (q != MP_QSTR_NULL) { - // qstr with this data already exists - return MP_OBJ_NEW_QSTR(q); - } else { - // no existing qstr, don't make one - return mp_obj_new_str_of_type(&mp_type_str, (const byte*)data, len); - } - } -} - -mp_obj_t mp_obj_str_intern(mp_obj_t str) { - GET_STR_DATA_LEN(str, data, len); - return MP_OBJ_NEW_QSTR(qstr_from_strn((const char*)data, len)); -} - -mp_obj_t mp_obj_new_bytes(const byte* data, uint len) { - return mp_obj_new_str_of_type(&mp_type_bytes, data, len); -} - -bool mp_obj_str_equal(mp_obj_t s1, mp_obj_t s2) { - if (MP_OBJ_IS_QSTR(s1) && MP_OBJ_IS_QSTR(s2)) { - return s1 == s2; - } else { - GET_STR_HASH(s1, h1); - GET_STR_HASH(s2, h2); - // If any of hashes is 0, it means it's not valid - if (h1 != 0 && h2 != 0 && h1 != h2) { - return false; - } - GET_STR_DATA_LEN(s1, d1, l1); - GET_STR_DATA_LEN(s2, d2, l2); - if (l1 != l2) { - return false; - } - return memcmp(d1, d2, l1) == 0; - } -} - -STATIC void bad_implicit_conversion(mp_obj_t self_in) { - nlr_raise(mp_obj_new_exception_msg_varg(&mp_type_TypeError, "Can't convert '%s' object to str implicitly", mp_obj_get_type_str(self_in))); -} - -STATIC void arg_type_mixup() { - nlr_raise(mp_obj_new_exception_msg(&mp_type_TypeError, "Can't mix str and bytes arguments")); -} - -uint mp_obj_str_get_hash(mp_obj_t self_in) { - // TODO: This has too big overhead for hash accessor - if (MP_OBJ_IS_STR(self_in) || MP_OBJ_IS_TYPE(self_in, &mp_type_bytes)) { - GET_STR_HASH(self_in, h); - return h; - } else { - bad_implicit_conversion(self_in); - } -} - -uint mp_obj_str_get_len(mp_obj_t self_in) { - // TODO This has a double check for the type, one in obj.c and one here - if (MP_OBJ_IS_STR(self_in) || MP_OBJ_IS_TYPE(self_in, &mp_type_bytes)) { - GET_STR_DATA_LEN(self_in, self_data, self_len); - return unichar_charlen((const char *)self_data, self_len); - } else { - bad_implicit_conversion(self_in); - } -} - -// use this if you will anyway convert the string to a qstr -// will be more efficient for the case where it's already a qstr -qstr mp_obj_str_get_qstr(mp_obj_t self_in) { - if (MP_OBJ_IS_QSTR(self_in)) { - return MP_OBJ_QSTR_VALUE(self_in); - } else if (MP_OBJ_IS_TYPE(self_in, &mp_type_str)) { - mp_obj_str_t *self = self_in; - return qstr_from_strn((char*)self->data, self->len); - } else { - bad_implicit_conversion(self_in); - } -} - -// only use this function if you need the str data to be zero terminated -// at the moment all strings are zero terminated to help with C ASCIIZ compatibility -const char *mp_obj_str_get_str(mp_obj_t self_in) { - if (MP_OBJ_IS_STR(self_in)) { - GET_STR_DATA_LEN(self_in, s, l); - (void)l; // len unused - return (const char*)s; - } else { - bad_implicit_conversion(self_in); - } -} - -const char *mp_obj_str_get_data(mp_obj_t self_in, uint *len) { - if (is_str_or_bytes(self_in)) { - GET_STR_DATA_LEN(self_in, s, l); - *len = l; - return (const char*)s; - } else { - bad_implicit_conversion(self_in); - } -} - /******************************************************************************/ /* str iterator */ @@ -1976,25 +340,6 @@ STATIC const mp_obj_type_t mp_type_str_it = { .iternext = str_it_iternext, }; -STATIC mp_obj_t bytes_it_iternext(mp_obj_t self_in) { - mp_obj_str_it_t *self = self_in; - GET_STR_DATA_LEN(self->str, str, len); - if (self->cur < len) { - mp_obj_t o_out = MP_OBJ_NEW_SMALL_INT((mp_small_int_t)str[self->cur]); - self->cur += 1; - return o_out; - } else { - return MP_OBJ_STOP_ITERATION; - } -} - -STATIC const mp_obj_type_t mp_type_bytes_it = { - { &mp_type_type }, - .name = MP_QSTR_iterator, - .getiter = mp_identity, - .iternext = bytes_it_iternext, -}; - mp_obj_t mp_obj_new_str_iterator(mp_obj_t str) { mp_obj_str_it_t *o = m_new_obj(mp_obj_str_it_t); o->base.type = &mp_type_str_it; @@ -2003,10 +348,4 @@ mp_obj_t mp_obj_new_str_iterator(mp_obj_t str) { return o; } -mp_obj_t mp_obj_new_bytes_iterator(mp_obj_t str) { - mp_obj_str_it_t *o = m_new_obj(mp_obj_str_it_t); - o->base.type = &mp_type_bytes_it; - o->str = str; - o->cur = 0; - return o; -} +#endif // MICROPY_PY_BUILTINS_STR_UNICODE diff --git a/py/py.mk b/py/py.mk index 374bb11d6..549c35d32 100644 --- a/py/py.mk +++ b/py/py.mk @@ -75,6 +75,7 @@ PY_O_BASENAME = \ objset.o \ objslice.o \ objstr.o \ + objstrunicode.o \ objstringio.o \ objtuple.o \ objtype.o \ From d215ee1dc1e67bbeedf4bc0564de5ab3facfa6c0 Mon Sep 17 00:00:00 2001 From: Paul Sokolovsky Date: Fri, 13 Jun 2014 22:41:45 +0300 Subject: [PATCH 11/32] py: Make MICROPY_PY_BUILTINS_STR_UNICODE=1 buildable. --- py/objstr.c | 29 ++++++++++++++++++++--------- 1 file changed, 20 insertions(+), 9 deletions(-) diff --git a/py/objstr.c b/py/objstr.c index c5acca325..39b035910 100644 --- a/py/objstr.c +++ b/py/objstr.c @@ -43,7 +43,7 @@ STATIC mp_obj_t str_modulo_format(mp_obj_t pattern, uint n_args, const mp_obj_t *args, mp_obj_t dict); const mp_obj_t mp_const_empty_bytes; -STATIC mp_obj_t mp_obj_new_str_iterator(mp_obj_t str); +mp_obj_t mp_obj_new_str_iterator(mp_obj_t str); STATIC mp_obj_t mp_obj_new_bytes_iterator(mp_obj_t str); STATIC NORETURN void bad_implicit_conversion(mp_obj_t self_in); STATIC NORETURN void arg_type_mixup(); @@ -1649,7 +1649,14 @@ MP_DEFINE_CONST_FUN_OBJ_1(str_islower_obj, str_islower); STATIC const mp_map_elem_t str_locals_dict_table[] = { #if MICROPY_CPYTHON_COMPAT { MP_OBJ_NEW_QSTR(MP_QSTR_decode), (mp_obj_t)&bytes_decode_obj }, + #if !MICROPY_PY_BUILTINS_STR_UNICODE + // If we have separate unicode type, then here we have methods only + // for bytes type, and it should not have encode() methods. Otherwise, + // we have non-compliant-but-practical bytestring type, which shares + // method table with bytes, so they both have encode() and decode() + // methods (which should do type checking at runtime). { MP_OBJ_NEW_QSTR(MP_QSTR_encode), (mp_obj_t)&str_encode_obj }, + #endif #endif { MP_OBJ_NEW_QSTR(MP_QSTR_find), (mp_obj_t)&str_find_obj }, { MP_OBJ_NEW_QSTR(MP_QSTR_rfind), (mp_obj_t)&str_rfind_obj }, @@ -1679,6 +1686,7 @@ STATIC const mp_map_elem_t str_locals_dict_table[] = { STATIC MP_DEFINE_CONST_DICT(str_locals_dict, str_locals_dict_table); +#if !MICROPY_PY_BUILTINS_STR_UNICODE const mp_obj_type_t mp_type_str = { { &mp_type_type }, .name = MP_QSTR_str, @@ -1690,6 +1698,7 @@ const mp_obj_type_t mp_type_str = { .buffer_p = { .get_buffer = str_get_buffer }, .locals_dict = (mp_obj_t)&str_locals_dict, }; +#endif // Reuses most of methods from str const mp_obj_type_t mp_type_bytes = { @@ -1857,6 +1866,7 @@ typedef struct _mp_obj_str_it_t { machine_uint_t cur; } mp_obj_str_it_t; +#if !MICROPY_PY_BUILTINS_STR_UNICODE STATIC mp_obj_t str_it_iternext(mp_obj_t self_in) { mp_obj_str_it_t *self = self_in; GET_STR_DATA_LEN(self->str, str, len); @@ -1876,6 +1886,15 @@ STATIC const mp_obj_type_t mp_type_str_it = { .iternext = str_it_iternext, }; +mp_obj_t mp_obj_new_str_iterator(mp_obj_t str) { + mp_obj_str_it_t *o = m_new_obj(mp_obj_str_it_t); + o->base.type = &mp_type_str_it; + o->str = str; + o->cur = 0; + return o; +} +#endif + STATIC mp_obj_t bytes_it_iternext(mp_obj_t self_in) { mp_obj_str_it_t *self = self_in; GET_STR_DATA_LEN(self->str, str, len); @@ -1895,14 +1914,6 @@ STATIC const mp_obj_type_t mp_type_bytes_it = { .iternext = bytes_it_iternext, }; -mp_obj_t mp_obj_new_str_iterator(mp_obj_t str) { - mp_obj_str_it_t *o = m_new_obj(mp_obj_str_it_t); - o->base.type = &mp_type_str_it; - o->str = str; - o->cur = 0; - return o; -} - mp_obj_t mp_obj_new_bytes_iterator(mp_obj_t str) { mp_obj_str_it_t *o = m_new_obj(mp_obj_str_it_t); o->base.type = &mp_type_bytes_it; From 86d3898e709f32bef9e44dd558e7ea5569398011 Mon Sep 17 00:00:00 2001 From: Paul Sokolovsky Date: Fri, 13 Jun 2014 23:00:15 +0300 Subject: [PATCH 12/32] objstrunicode: Get rid of bytes checking, it's separate type. --- py/objstrunicode.c | 22 +++++++--------------- 1 file changed, 7 insertions(+), 15 deletions(-) diff --git a/py/objstrunicode.c b/py/objstrunicode.c index 741f1b76e..d41e92db4 100644 --- a/py/objstrunicode.c +++ b/py/objstrunicode.c @@ -47,7 +47,7 @@ STATIC mp_obj_t mp_obj_new_str_iterator(mp_obj_t str); /******************************************************************************/ /* str */ -STATIC void uni_print_quoted(void (*print)(void *env, const char *fmt, ...), void *env, const byte *str_data, uint str_len, bool is_bytes) { +STATIC void uni_print_quoted(void (*print)(void *env, const char *fmt, ...), void *env, const byte *str_data, uint str_len) { // this escapes characters, but it will be very slow to print (calling print many times) bool has_single_quote = false; bool has_double_quote = false; @@ -66,12 +66,8 @@ STATIC void uni_print_quoted(void (*print)(void *env, const char *fmt, ...), voi const char *s = (const char *)str_data, *top = (const char *)str_data + str_len; while (s < top) { unichar ch; - if (is_bytes) { - ch = *(unsigned char *)s++; // Don't sign-extend bytes - } else { - ch = utf8_get_char(s); - s = utf8_next_char(s); - } + ch = utf8_get_char(s); + s = utf8_next_char(s); if (ch == quote_char) { print(env, "\\%c", quote_char); } else if (ch == '\\') { @@ -95,16 +91,12 @@ STATIC void uni_print_quoted(void (*print)(void *env, const char *fmt, ...), voi print(env, "%c", quote_char); } -STATIC void str_print(void (*print)(void *env, const char *fmt, ...), void *env, mp_obj_t self_in, mp_print_kind_t kind) { +STATIC void uni_print(void (*print)(void *env, const char *fmt, ...), void *env, mp_obj_t self_in, mp_print_kind_t kind) { GET_STR_DATA_LEN(self_in, str_data, str_len); - bool is_bytes = MP_OBJ_IS_TYPE(self_in, &mp_type_bytes); - if (kind == PRINT_STR && !is_bytes) { + if (kind == PRINT_STR) { print(env, "%.*s", str_len, str_data); } else { - if (is_bytes) { - print(env, "b"); - } - uni_print_quoted(print, env, str_data, str_len, is_bytes); + uni_print_quoted(print, env, str_data, str_len); } } @@ -303,7 +295,7 @@ STATIC MP_DEFINE_CONST_DICT(str_locals_dict, str_locals_dict_table); const mp_obj_type_t mp_type_str = { { &mp_type_type }, .name = MP_QSTR_str, - .print = str_print, + .print = uni_print, .make_new = str_make_new, .binary_op = str_binary_op, .subscr = str_subscr, From e7f2b4c875fa3130e4ad37721a7d231380456895 Mon Sep 17 00:00:00 2001 From: Paul Sokolovsky Date: Fri, 13 Jun 2014 23:37:18 +0300 Subject: [PATCH 13/32] objstrunicode: Revamp len() handling for unicode, and optimize bool(). --- py/obj.c | 7 ++++++- py/objstrunicode.c | 13 +++++++++++++ 2 files changed, 19 insertions(+), 1 deletion(-) diff --git a/py/obj.c b/py/obj.c index a0f55d65d..d951abbda 100644 --- a/py/obj.c +++ b/py/obj.c @@ -354,7 +354,12 @@ uint mp_get_index(const mp_obj_type_t *type, machine_uint_t len, mp_obj_t index, // may return MP_OBJ_NULL mp_obj_t mp_obj_len_maybe(mp_obj_t o_in) { - if (MP_OBJ_IS_STR(o_in) || MP_OBJ_IS_TYPE(o_in, &mp_type_bytes)) { + if ( +#if !MICROPY_PY_BUILTINS_STR_UNICODE + // It's simple - unicode is slow, non-unicode is fast + MP_OBJ_IS_STR(o_in) || +#endif + MP_OBJ_IS_TYPE(o_in, &mp_type_bytes)) { return MP_OBJ_NEW_SMALL_INT((machine_int_t)mp_obj_str_get_len(o_in)); } else { mp_obj_type_t *type = mp_obj_get_type(o_in); diff --git a/py/objstrunicode.c b/py/objstrunicode.c index d41e92db4..8fbe81532 100644 --- a/py/objstrunicode.c +++ b/py/objstrunicode.c @@ -100,6 +100,18 @@ STATIC void uni_print(void (*print)(void *env, const char *fmt, ...), void *env, } } +STATIC mp_obj_t uni_unary_op(int op, mp_obj_t self_in) { + GET_STR_DATA_LEN(self_in, str_data, str_len); + switch (op) { + case MP_UNARY_OP_BOOL: + return MP_BOOL(str_len != 0); + case MP_UNARY_OP_LEN: + return MP_OBJ_NEW_SMALL_INT(unichar_charlen((const char *)str_data, str_len)); + default: + return MP_OBJ_NULL; // op not supported + } +} + STATIC mp_obj_t str_make_new(mp_obj_t type_in, uint n_args, uint n_kw, const mp_obj_t *args) { #if MICROPY_CPYTHON_COMPAT if (n_kw != 0) { @@ -297,6 +309,7 @@ const mp_obj_type_t mp_type_str = { .name = MP_QSTR_str, .print = uni_print, .make_new = str_make_new, + .unary_op = uni_unary_op, .binary_op = str_binary_op, .subscr = str_subscr, .getiter = mp_obj_new_str_iterator, From cdc020da4b3da88418e248b8b4bca2247d238923 Mon Sep 17 00:00:00 2001 From: Paul Sokolovsky Date: Sat, 14 Jun 2014 01:19:52 +0300 Subject: [PATCH 14/32] objstrunicode: Re-add buffer protocol back for now, required for io.StringIO. --- py/objstr.c | 2 +- py/objstr.h | 1 + py/objstrunicode.c | 2 +- 3 files changed, 3 insertions(+), 2 deletions(-) diff --git a/py/objstr.c b/py/objstr.c index 39b035910..acbf00283 100644 --- a/py/objstr.c +++ b/py/objstr.c @@ -1601,7 +1601,7 @@ STATIC mp_obj_t str_encode(uint n_args, const mp_obj_t *args) { } #endif -STATIC machine_int_t str_get_buffer(mp_obj_t self_in, mp_buffer_info_t *bufinfo, int flags) { +machine_int_t str_get_buffer(mp_obj_t self_in, mp_buffer_info_t *bufinfo, int flags) { if (flags == MP_BUFFER_READ) { GET_STR_DATA_LEN(self_in, str_data, str_len); bufinfo->buf = (void*)str_data; diff --git a/py/objstr.h b/py/objstr.h index 2e5aaeb09..75de50d29 100644 --- a/py/objstr.h +++ b/py/objstr.h @@ -55,6 +55,7 @@ mp_obj_t mp_obj_str_format(uint n_args, const mp_obj_t *args); mp_obj_t mp_obj_new_str_of_type(const mp_obj_type_t *type, const byte* data, uint len); mp_obj_t str_binary_op(int op, mp_obj_t lhs_in, mp_obj_t rhs_in); +machine_int_t str_get_buffer(mp_obj_t self_in, mp_buffer_info_t *bufinfo, int flags); MP_DECLARE_CONST_FUN_OBJ(str_encode_obj); MP_DECLARE_CONST_FUN_OBJ(str_find_obj); diff --git a/py/objstrunicode.c b/py/objstrunicode.c index 8fbe81532..e6ca35acb 100644 --- a/py/objstrunicode.c +++ b/py/objstrunicode.c @@ -313,7 +313,7 @@ const mp_obj_type_t mp_type_str = { .binary_op = str_binary_op, .subscr = str_subscr, .getiter = mp_obj_new_str_iterator, -// .buffer_p = { .get_buffer = str_get_buffer }, + .buffer_p = { .get_buffer = str_get_buffer }, .locals_dict = (mp_obj_t)&str_locals_dict, }; From 79b7fe2ee58c1ac46c2fcb74bea69f2810bced5b Mon Sep 17 00:00:00 2001 From: Paul Sokolovsky Date: Sat, 14 Jun 2014 02:07:25 +0300 Subject: [PATCH 15/32] objstrunicode: Implement iterator. --- py/objstrunicode.c | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/py/objstrunicode.c b/py/objstrunicode.c index e6ca35acb..d70b33f8f 100644 --- a/py/objstrunicode.c +++ b/py/objstrunicode.c @@ -330,8 +330,10 @@ STATIC mp_obj_t str_it_iternext(mp_obj_t self_in) { mp_obj_str_it_t *self = self_in; GET_STR_DATA_LEN(self->str, str, len); if (self->cur < len) { - mp_obj_t o_out = mp_obj_new_str((const char*)str + self->cur, 1, true); - self->cur += 1; + const byte *cur = str + self->cur; + const byte *end = utf8_next_char(str + self->cur); + mp_obj_t o_out = mp_obj_new_str((const char*)cur, end - cur, true); + self->cur += end - cur; return o_out; } else { return MP_OBJ_STOP_ITERATION; From 17994d1bd3fcf7e37974ecaf0e2ba0cab7c65665 Mon Sep 17 00:00:00 2001 From: Paul Sokolovsky Date: Sat, 14 Jun 2014 02:08:48 +0300 Subject: [PATCH 16/32] tests: Add test for unicode string iteration. --- tests/unicode/unicode_iter.py | 4 ++++ 1 file changed, 4 insertions(+) create mode 100644 tests/unicode/unicode_iter.py diff --git a/tests/unicode/unicode_iter.py b/tests/unicode/unicode_iter.py new file mode 100644 index 000000000..f08a4acee --- /dev/null +++ b/tests/unicode/unicode_iter.py @@ -0,0 +1,4 @@ +for c in "Hello": + print(c) +for c in "Привет": + print(c) From ded0fc77f7ee70b3f7935921bb9631e9763fb1fc Mon Sep 17 00:00:00 2001 From: Paul Sokolovsky Date: Sat, 14 Jun 2014 03:09:39 +0300 Subject: [PATCH 17/32] py: Add dedicated unicode header. --- py/unicode.h | 1 + 1 file changed, 1 insertion(+) create mode 100644 py/unicode.h diff --git a/py/unicode.h b/py/unicode.h new file mode 100644 index 000000000..2468b2fec --- /dev/null +++ b/py/unicode.h @@ -0,0 +1 @@ +machine_uint_t utf8_ptr_to_index(const byte *s, const byte *ptr); From 46d31e9ca90635707031afedf09da7aeed25a321 Mon Sep 17 00:00:00 2001 From: Paul Sokolovsky Date: Sat, 14 Jun 2014 03:16:17 +0300 Subject: [PATCH 18/32] unicode: Add utf8_ptr_to_index(). Useful when we have pointer to char inside string, but need to return char index. (E.g. str.find()). --- py/unicode.c | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/py/unicode.c b/py/unicode.c index 0da247889..c38120072 100644 --- a/py/unicode.c +++ b/py/unicode.c @@ -86,6 +86,17 @@ char *utf8_next_char(const char *s) { return (char *)s; } +machine_uint_t utf8_ptr_to_index(const char *s, const char *ptr) { + machine_uint_t i = 0; + while (ptr > s) { + if (!UTF8_IS_CONT(*--ptr)) { + i++; + } + } + + return i; +} + uint unichar_charlen(const char *str, uint len) { uint charlen = 0; From 5048df0b7c1cfb951e28fc2d4b1f1385b1c8046e Mon Sep 17 00:00:00 2001 From: Paul Sokolovsky Date: Sat, 14 Jun 2014 03:15:00 +0300 Subject: [PATCH 19/32] objstr: find(), rfind(), index(): Make return value be unicode-aware. --- py/objstr.c | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/py/objstr.c b/py/objstr.c index acbf00283..c732719dd 100644 --- a/py/objstr.c +++ b/py/objstr.c @@ -32,6 +32,7 @@ #include "mpconfig.h" #include "nlr.h" #include "misc.h" +#include "unicode.h" #include "qstr.h" #include "obj.h" #include "runtime0.h" @@ -591,6 +592,11 @@ STATIC mp_obj_t str_finder(uint n_args, const mp_obj_t *args, machine_int_t dire } } else { // found + #if MICROPY_PY_BUILTINS_STR_UNICODE + if (self_type == &mp_type_str) { + return MP_OBJ_NEW_SMALL_INT(utf8_ptr_to_index(haystack, p)); + } + #endif return MP_OBJ_NEW_SMALL_INT(p - haystack); } } From b1949e4c0984c1f254c5877da1b977c01567bf4d Mon Sep 17 00:00:00 2001 From: Paul Sokolovsky Date: Sat, 14 Jun 2014 03:36:06 +0300 Subject: [PATCH 20/32] tests: Add tests for unicode find()/rfind()/index(). --- tests/unicode/unicode_index.py | 6 ++++++ 1 file changed, 6 insertions(+) create mode 100644 tests/unicode/unicode_index.py diff --git a/tests/unicode/unicode_index.py b/tests/unicode/unicode_index.py new file mode 100644 index 000000000..3c31468a4 --- /dev/null +++ b/tests/unicode/unicode_index.py @@ -0,0 +1,6 @@ +print("Привет".find("т")) +print("Привет".find("П")) +print("Привет".rfind("т")) +print("Привет".rfind("П")) +print("Привет".index("т")) +print("Привет".index("П")) From 1044c3dfe6e93b143fc80b3cd8a5096da9dddc2b Mon Sep 17 00:00:00 2001 From: Paul Sokolovsky Date: Sat, 14 Jun 2014 06:39:20 +0300 Subject: [PATCH 21/32] unicode: Make get_char()/next_char()/charlen() be 8-bit compatible. Based on config define. --- py/unicode.c | 20 +++++++++++++++++--- 1 file changed, 17 insertions(+), 3 deletions(-) diff --git a/py/unicode.c b/py/unicode.c index c38120072..a58cdb14a 100644 --- a/py/unicode.c +++ b/py/unicode.c @@ -65,7 +65,9 @@ STATIC const uint8_t attr[] = { AT_LO, AT_LO, AT_LO, AT_PR, AT_PR, AT_PR, AT_PR, 0 }; -unichar utf8_get_char(const char *s) { +// TODO: Rename to str_get_char +unichar utf8_get_char(const byte *s) { +#if MICROPY_PY_BUILTINS_STR_UNICODE unichar ord = *s++; if (!UTF8_IS_NONASCII(ord)) return ord; ord &= 0x7F; @@ -76,14 +78,22 @@ unichar utf8_get_char(const char *s) { ord = (ord << 6) | (*s++ & 0x3F); } return ord; +#else + return *s; +#endif } -char *utf8_next_char(const char *s) { +// TODO: Rename to str_next_char +const byte *utf8_next_char(const byte *s) { +#if MICROPY_PY_BUILTINS_STR_UNICODE ++s; while (UTF8_IS_CONT(*s)) { ++s; } - return (char *)s; + return s; +#else + return s + 1; +#endif } machine_uint_t utf8_ptr_to_index(const char *s, const char *ptr) { @@ -99,6 +109,7 @@ machine_uint_t utf8_ptr_to_index(const char *s, const char *ptr) { uint unichar_charlen(const char *str, uint len) { +#if MICROPY_PY_BUILTINS_STR_UNICODE uint charlen = 0; for (const char *top = str + len; str < top; ++str) { if (!UTF8_IS_CONT(*str)) { @@ -106,6 +117,9 @@ uint unichar_charlen(const char *str, uint len) } } return charlen; +#else + return len; +#endif } // Be aware: These unichar_is* functions are actually ASCII-only! From 00c904b47a4cac2cdf276346039f470f59c90c04 Mon Sep 17 00:00:00 2001 From: Paul Sokolovsky Date: Sat, 14 Jun 2014 17:48:40 +0300 Subject: [PATCH 22/32] objstrunicode: Signedness issues. --- py/objstrunicode.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/py/objstrunicode.c b/py/objstrunicode.c index d70b33f8f..334d9895e 100644 --- a/py/objstrunicode.c +++ b/py/objstrunicode.c @@ -63,7 +63,7 @@ STATIC void uni_print_quoted(void (*print)(void *env, const char *fmt, ...), voi quote_char = '"'; } print(env, "%c", quote_char); - const char *s = (const char *)str_data, *top = (const char *)str_data + str_len; + const byte *s = str_data, *top = str_data + str_len; while (s < top) { unichar ch; ch = utf8_get_char(s); From 26fda6dc8ef77e1acdc3529360a6d1cef23608e4 Mon Sep 17 00:00:00 2001 From: Paul Sokolovsky Date: Sat, 14 Jun 2014 18:19:16 +0300 Subject: [PATCH 23/32] objstr: 64-bit issues. --- py/objstr.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/py/objstr.c b/py/objstr.c index c732719dd..714a170d0 100644 --- a/py/objstr.c +++ b/py/objstr.c @@ -1446,7 +1446,7 @@ STATIC mp_obj_t str_count(uint n_args, const mp_obj_t *args) { // if needle_len is zero then we count each gap between characters as an occurrence if (needle_len == 0) { - return MP_OBJ_NEW_SMALL_INT(unichar_charlen((const char*)start, end - start) + 1); + return MP_OBJ_NEW_SMALL_INT((machine_uint_t)unichar_charlen((const char*)start, end - start) + 1); } // count the occurrences From ea2c936c7e6af8e6d8168d039bdf02797b0777bf Mon Sep 17 00:00:00 2001 From: Paul Sokolovsky Date: Sun, 15 Jun 2014 00:35:09 +0300 Subject: [PATCH 24/32] objstrunicode: Refactor str_index_to_ptr() following objstr. --- py/objstr.c | 3 +++ py/objstr.h | 3 +++ py/objstrunicode.c | 19 ++++++++++--------- 3 files changed, 16 insertions(+), 9 deletions(-) diff --git a/py/objstr.c b/py/objstr.c index 714a170d0..6ddba18a3 100644 --- a/py/objstr.c +++ b/py/objstr.c @@ -344,11 +344,14 @@ uncomparable: return MP_OBJ_NULL; // op not supported } +#if !MICROPY_PY_BUILTINS_STR_UNICODE +// objstrunicode defines own version const byte *str_index_to_ptr(const mp_obj_type_t *type, const byte *self_data, uint self_len, mp_obj_t index, bool is_slice) { machine_uint_t index_val = mp_get_index(type, self_len, index, is_slice); return self_data + index_val; } +#endif STATIC mp_obj_t str_subscr(mp_obj_t self_in, mp_obj_t index, mp_obj_t value) { mp_obj_type_t *type = mp_obj_get_type(self_in); diff --git a/py/objstr.h b/py/objstr.h index 75de50d29..da2c9cd51 100644 --- a/py/objstr.h +++ b/py/objstr.h @@ -57,6 +57,9 @@ mp_obj_t mp_obj_new_str_of_type(const mp_obj_type_t *type, const byte* data, uin mp_obj_t str_binary_op(int op, mp_obj_t lhs_in, mp_obj_t rhs_in); machine_int_t str_get_buffer(mp_obj_t self_in, mp_buffer_info_t *bufinfo, int flags); +const byte *str_index_to_ptr(const mp_obj_type_t *type, const byte *self_data, uint self_len, + mp_obj_t index, bool is_slice); + MP_DECLARE_CONST_FUN_OBJ(str_encode_obj); MP_DECLARE_CONST_FUN_OBJ(str_find_obj); MP_DECLARE_CONST_FUN_OBJ(str_rfind_obj); diff --git a/py/objstrunicode.c b/py/objstrunicode.c index 334d9895e..052028eee 100644 --- a/py/objstrunicode.c +++ b/py/objstrunicode.c @@ -154,7 +154,8 @@ STATIC mp_obj_t str_make_new(mp_obj_t type_in, uint n_args, uint n_kw, const mp_ // Convert an index into a pointer to its lead byte. Out of bounds indexing will raise IndexError or // be capped to the first/last character of the string, depending on is_slice. -STATIC const char *str_index_to_ptr(const char *self_data, uint self_len, mp_obj_t index, bool is_slice) { +const byte *str_index_to_ptr(const mp_obj_type_t *type, const byte *self_data, uint self_len, + mp_obj_t index, bool is_slice) { machine_int_t i; // Copied from mp_get_index; I don't want bounds checking, just give me // the integer as-is. (I can't bounds-check without scanning the whole @@ -164,7 +165,7 @@ STATIC const char *str_index_to_ptr(const char *self_data, uint self_len, mp_obj } else if (!mp_obj_get_int_maybe(index, &i)) { nlr_raise(mp_obj_new_exception_msg_varg(&mp_type_TypeError, "string indices must be integers, not %s", mp_obj_get_type_str(index))); } - const char *s, *top = self_data + self_len; + const byte *s, *top = self_data + self_len; if (i < 0) { // Negative indexing is performed by counting from the end of the string. @@ -235,18 +236,18 @@ STATIC mp_obj_t str_subscr(mp_obj_t self_in, mp_obj_t index, mp_obj_t value) { } return mp_obj_new_str_of_type(type, self_data + start, stop - start); } - const char *pstart, *pstop; + const byte *pstart, *pstop; if (ostart != mp_const_none) { - pstart = str_index_to_ptr((const char *)self_data, self_len, ostart, true); + pstart = str_index_to_ptr(type, self_data, self_len, ostart, true); } else { - pstart = (const char *)self_data; + pstart = self_data; } if (ostop != mp_const_none) { // pstop will point just after the stop character. This depends on // the \0 at the end of the string. - pstop = str_index_to_ptr((const char *)self_data, self_len, ostop, true); + pstop = str_index_to_ptr(type, self_data, self_len, ostop, true); } else { - pstop = (const char *)self_data + self_len; + pstop = self_data + self_len; } if (pstop < pstart) { return MP_OBJ_NEW_QSTR(MP_QSTR_); @@ -258,7 +259,7 @@ STATIC mp_obj_t str_subscr(mp_obj_t self_in, mp_obj_t index, mp_obj_t value) { uint index_val = mp_get_index(type, self_len, index, false); return MP_OBJ_NEW_SMALL_INT((mp_small_int_t)self_data[index_val]); } - const char *s = str_index_to_ptr((const char *)self_data, self_len, index, false); + const byte *s = str_index_to_ptr(type, self_data, self_len, index, false); int len = 1; if (UTF8_IS_NONASCII(*s)) { // Count the number of 1 bits (after the first) @@ -266,7 +267,7 @@ STATIC mp_obj_t str_subscr(mp_obj_t self_in, mp_obj_t index, mp_obj_t value) { ++len; } } - return mp_obj_new_str(s, len, true); // This will create a one-character string + return mp_obj_new_str((const char*)s, len, true); // This will create a one-character string } else { return MP_OBJ_NULL; // op not supported } From 63143c94cef353d7bae13f7b13650801bb901c94 Mon Sep 17 00:00:00 2001 From: Paul Sokolovsky Date: Sun, 15 Jun 2014 00:45:40 +0300 Subject: [PATCH 25/32] tests: Test for explicit start/end args to str methods for unicode. --- tests/unicode/unicode_pos.py | 5 +++++ 1 file changed, 5 insertions(+) create mode 100644 tests/unicode/unicode_pos.py diff --git a/tests/unicode/unicode_pos.py b/tests/unicode/unicode_pos.py new file mode 100644 index 000000000..6a5982920 --- /dev/null +++ b/tests/unicode/unicode_pos.py @@ -0,0 +1,5 @@ +# str methods with explicit start/end pos +print("Привет".startswith("П")) +print("Привет".startswith("р", 1)) +print("абвба".find("а", 1)) +print("абвба".find("а", 1, -1)) From ce81312d8a81c7579ecc35ccdc53a49543c0b0d5 Mon Sep 17 00:00:00 2001 From: Paul Sokolovsky Date: Sun, 15 Jun 2014 23:13:41 +0300 Subject: [PATCH 26/32] misc: Add count_lead_ones() function, useful for UTF-8 handling. --- py/misc.h | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/py/misc.h b/py/misc.h index 97e9b30ed..7a736560a 100644 --- a/py/misc.h +++ b/py/misc.h @@ -166,4 +166,17 @@ int DEBUG_printf(const char *fmt, ...); extern uint mp_verbose_flag; +// This is useful for unicode handling. Some CPU archs has +// special instructions for efficient implentation of this +// function (e.g. CLZ on ARM). +#ifndef count_lead_ones +static inline uint count_lead_ones(byte val) { + uint c = 0; + for (byte mask = 0x80; val & mask; mask >>= 1) { + c++; + } + return c; +} +#endif + #endif // _INCLUDED_MINILIB_H From f5f6c3b7920457e99b71cf58cfb3cb0be0e4e890 Mon Sep 17 00:00:00 2001 From: Paul Sokolovsky Date: Sun, 15 Jun 2014 23:23:36 +0300 Subject: [PATCH 27/32] streams: Reading by char count from unicode text streams is not implemented. --- py/stream.c | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/py/stream.c b/py/stream.c index 07a79248a..a5a96a868 100644 --- a/py/stream.c +++ b/py/stream.c @@ -33,6 +33,7 @@ #include "qstr.h" #include "obj.h" #include "objstr.h" +#include "runtime.h" #include "stream.h" #if MICROPY_STREAMS_NON_BLOCK #include @@ -67,6 +68,13 @@ STATIC mp_obj_t stream_read(uint n_args, const mp_obj_t *args) { if (n_args == 1 || ((sz = mp_obj_get_int(args[1])) == -1)) { return stream_readall(args[0]); } + + #if MICROPY_PY_BUILTINS_STR_UNICODE + if (!o->type->stream_p->is_bytes) { + mp_not_implemented("Reading from unicode text streams by character count"); + } + #endif + byte *buf = m_new(byte, sz); int error; machine_int_t out_sz = o->type->stream_p->read(o, buf, sz, &error); From ed07d035d52764b3b02bfa8488f2e0445c28ea2f Mon Sep 17 00:00:00 2001 From: Paul Sokolovsky Date: Sun, 15 Jun 2014 23:28:39 +0300 Subject: [PATCH 28/32] tests: Add basic test for unicode file i/o. --- tests/unicode/data/utf-8_1.txt | 1 + tests/unicode/file1.py | 4 ++++ 2 files changed, 5 insertions(+) create mode 100644 tests/unicode/data/utf-8_1.txt create mode 100644 tests/unicode/file1.py diff --git a/tests/unicode/data/utf-8_1.txt b/tests/unicode/data/utf-8_1.txt new file mode 100644 index 000000000..d84c480d1 --- /dev/null +++ b/tests/unicode/data/utf-8_1.txt @@ -0,0 +1 @@ +Привет diff --git a/tests/unicode/file1.py b/tests/unicode/file1.py new file mode 100644 index 000000000..554e88674 --- /dev/null +++ b/tests/unicode/file1.py @@ -0,0 +1,4 @@ +f = open("unicode/data/utf-8_1.txt") +l = f.readline() +print(l) +print(len(l)) From e04a44e2f65d1ac49e5cce164d6010ff968f4d48 Mon Sep 17 00:00:00 2001 From: Damien George Date: Sat, 28 Jun 2014 10:27:23 +0100 Subject: [PATCH 29/32] py: Small comments, name changes, use of machine_int_t. --- py/builtin.c | 4 ++-- py/misc.h | 3 ++- py/objstr.c | 13 ++++++------- py/objstr.h | 4 ++-- py/objstrunicode.c | 6 +++--- py/unicode.c | 1 + 6 files changed, 16 insertions(+), 15 deletions(-) diff --git a/py/builtin.c b/py/builtin.c index 1dd364911..9986b90d3 100644 --- a/py/builtin.c +++ b/py/builtin.c @@ -173,7 +173,7 @@ MP_DEFINE_CONST_FUN_OBJ_1(mp_builtin_callable_obj, mp_builtin_callable); STATIC mp_obj_t mp_builtin_chr(mp_obj_t o_in) { #if MICROPY_PY_BUILTINS_STR_UNICODE - int c = mp_obj_get_int(o_in); + machine_int_t c = mp_obj_get_int(o_in); char str[4]; int len = 0; if (c < 0x80) { @@ -198,7 +198,7 @@ STATIC mp_obj_t mp_builtin_chr(mp_obj_t o_in) { } return mp_obj_new_str(str, len, true); #else - int ord = mp_obj_get_int(o_in); + machine_int_t ord = mp_obj_get_int(o_in); if (0 <= ord && ord <= 0x10ffff) { char str[1] = {ord}; return mp_obj_new_str(str, 1, true); diff --git a/py/misc.h b/py/misc.h index 7a736560a..bf63ce135 100644 --- a/py/misc.h +++ b/py/misc.h @@ -100,7 +100,7 @@ bool unichar_isupper(unichar c); bool unichar_islower(unichar c); unichar unichar_tolower(unichar c); unichar unichar_toupper(unichar c); -uint unichar_charlen(const char *str, uint len); +uint unichar_charlen(const char *str, uint len); // TODO this should return machine_uint_t #define UTF8_IS_NONASCII(ch) ((ch) & 0x80) #define UTF8_IS_CONT(ch) (((ch) & 0xC0) == 0x80) @@ -169,6 +169,7 @@ extern uint mp_verbose_flag; // This is useful for unicode handling. Some CPU archs has // special instructions for efficient implentation of this // function (e.g. CLZ on ARM). +// NOTE: this function is unused at the moment #ifndef count_lead_ones static inline uint count_lead_ones(byte val) { uint c = 0; diff --git a/py/objstr.c b/py/objstr.c index 6ddba18a3..a4749369d 100644 --- a/py/objstr.c +++ b/py/objstr.c @@ -251,7 +251,7 @@ STATIC const byte *find_subbytes(const byte *haystack, machine_uint_t hlen, cons return NULL; } -mp_obj_t str_binary_op(int op, mp_obj_t lhs_in, mp_obj_t rhs_in) { +mp_obj_t mp_obj_str_binary_op(int op, mp_obj_t lhs_in, mp_obj_t rhs_in) { GET_STR_DATA_LEN(lhs_in, lhs_data, lhs_len); mp_obj_type_t *lhs_type = mp_obj_get_type(lhs_in); mp_obj_type_t *rhs_type = mp_obj_get_type(rhs_in); @@ -566,7 +566,6 @@ STATIC mp_obj_t str_rsplit(uint n_args, const mp_obj_t *args) { return res; } - STATIC mp_obj_t str_finder(uint n_args, const mp_obj_t *args, machine_int_t direction, bool is_index) { const mp_obj_type_t *self_type = mp_obj_get_type(args[0]); assert(2 <= n_args && n_args <= 4); @@ -1610,7 +1609,7 @@ STATIC mp_obj_t str_encode(uint n_args, const mp_obj_t *args) { } #endif -machine_int_t str_get_buffer(mp_obj_t self_in, mp_buffer_info_t *bufinfo, int flags) { +machine_int_t mp_obj_str_get_buffer(mp_obj_t self_in, mp_buffer_info_t *bufinfo, int flags) { if (flags == MP_BUFFER_READ) { GET_STR_DATA_LEN(self_in, str_data, str_len); bufinfo->buf = (void*)str_data; @@ -1701,10 +1700,10 @@ const mp_obj_type_t mp_type_str = { .name = MP_QSTR_str, .print = str_print, .make_new = str_make_new, - .binary_op = str_binary_op, + .binary_op = mp_obj_str_binary_op, .subscr = str_subscr, .getiter = mp_obj_new_str_iterator, - .buffer_p = { .get_buffer = str_get_buffer }, + .buffer_p = { .get_buffer = mp_obj_str_get_buffer }, .locals_dict = (mp_obj_t)&str_locals_dict, }; #endif @@ -1715,10 +1714,10 @@ const mp_obj_type_t mp_type_bytes = { .name = MP_QSTR_bytes, .print = str_print, .make_new = bytes_make_new, - .binary_op = str_binary_op, + .binary_op = mp_obj_str_binary_op, .subscr = str_subscr, .getiter = mp_obj_new_bytes_iterator, - .buffer_p = { .get_buffer = str_get_buffer }, + .buffer_p = { .get_buffer = mp_obj_str_get_buffer }, .locals_dict = (mp_obj_t)&str_locals_dict, }; diff --git a/py/objstr.h b/py/objstr.h index da2c9cd51..515890c6e 100644 --- a/py/objstr.h +++ b/py/objstr.h @@ -54,8 +54,8 @@ typedef struct _mp_obj_str_t { mp_obj_t mp_obj_str_format(uint n_args, const mp_obj_t *args); mp_obj_t mp_obj_new_str_of_type(const mp_obj_type_t *type, const byte* data, uint len); -mp_obj_t str_binary_op(int op, mp_obj_t lhs_in, mp_obj_t rhs_in); -machine_int_t str_get_buffer(mp_obj_t self_in, mp_buffer_info_t *bufinfo, int flags); +mp_obj_t mp_obj_str_binary_op(int op, mp_obj_t lhs_in, mp_obj_t rhs_in); +machine_int_t mp_obj_str_get_buffer(mp_obj_t self_in, mp_buffer_info_t *bufinfo, int flags); const byte *str_index_to_ptr(const mp_obj_type_t *type, const byte *self_data, uint self_len, mp_obj_t index, bool is_slice); diff --git a/py/objstrunicode.c b/py/objstrunicode.c index 052028eee..35cc73bd8 100644 --- a/py/objstrunicode.c +++ b/py/objstrunicode.c @@ -106,7 +106,7 @@ STATIC mp_obj_t uni_unary_op(int op, mp_obj_t self_in) { case MP_UNARY_OP_BOOL: return MP_BOOL(str_len != 0); case MP_UNARY_OP_LEN: - return MP_OBJ_NEW_SMALL_INT(unichar_charlen((const char *)str_data, str_len)); + return MP_OBJ_NEW_SMALL_INT((machine_int_t)unichar_charlen((const char *)str_data, str_len)); default: return MP_OBJ_NULL; // op not supported } @@ -311,10 +311,10 @@ const mp_obj_type_t mp_type_str = { .print = uni_print, .make_new = str_make_new, .unary_op = uni_unary_op, - .binary_op = str_binary_op, + .binary_op = mp_obj_str_binary_op, .subscr = str_subscr, .getiter = mp_obj_new_str_iterator, - .buffer_p = { .get_buffer = str_get_buffer }, + .buffer_p = { .get_buffer = mp_obj_str_get_buffer }, .locals_dict = (mp_obj_t)&str_locals_dict, }; diff --git a/py/unicode.c b/py/unicode.c index a58cdb14a..a91e08078 100644 --- a/py/unicode.c +++ b/py/unicode.c @@ -107,6 +107,7 @@ machine_uint_t utf8_ptr_to_index(const char *s, const char *ptr) { return i; } +// TODO: Rename to str_charlen; return machine_uint_t uint unichar_charlen(const char *str, uint len) { #if MICROPY_PY_BUILTINS_STR_UNICODE From 41736f8201fd645bbcca6741c03eccb8f4c6bde3 Mon Sep 17 00:00:00 2001 From: Damien George Date: Sat, 28 Jun 2014 10:29:12 +0100 Subject: [PATCH 30/32] tests: Write output in byte mode, not text mode. This enables testing unicode and non-unicode implementations. --- tests/run-tests | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/tests/run-tests b/tests/run-tests index 8f5f7d470..c6bc4020d 100755 --- a/tests/run-tests +++ b/tests/run-tests @@ -88,10 +88,10 @@ def run_tests(pyb, tests): rm_f(filename_expected) rm_f(filename_mupy) else: - with open(filename_expected, "w") as f: - f.write(str(output_expected, "ascii")) - with open(filename_mupy, "w") as f: - f.write(str(output_mupy, "ascii")) + with open(filename_expected, "wb") as f: + f.write(output_expected) + with open(filename_mupy, "wb") as f: + f.write(output_mupy) print("FAIL ", test_file) failed_tests.append(test_name) From 8546ce1e2894f15e9d683ff2294f22c1419cbe78 Mon Sep 17 00:00:00 2001 From: Damien George Date: Sat, 28 Jun 2014 10:29:22 +0100 Subject: [PATCH 31/32] py: Add missing #endif. --- py/mpconfig.h | 1 + 1 file changed, 1 insertion(+) diff --git a/py/mpconfig.h b/py/mpconfig.h index 7ed99ac8e..0dd84d1f8 100644 --- a/py/mpconfig.h +++ b/py/mpconfig.h @@ -252,6 +252,7 @@ typedef double mp_float_t; // Whether str object is proper unicode #ifndef MICROPY_PY_BUILTINS_STR_UNICODE #define MICROPY_PY_BUILTINS_STR_UNICODE (0) +#endif // Whether to support bytearray object #ifndef MICROPY_PY_BUILTINS_BYTEARRAY From 635b60e299509a85722db77c4409c78ca86dbdc7 Mon Sep 17 00:00:00 2001 From: Damien George Date: Sat, 28 Jun 2014 10:29:52 +0100 Subject: [PATCH 32/32] unix, stmhal: Add option for STR_UNICODE to mpconfigport.h. Unicode is disabled by default for now, since FileIO.read(n) is currently not implemented for text-mode files, and this is an often function. --- stmhal/mpconfigport.h | 1 + unix/mpconfigport.h | 1 + 2 files changed, 2 insertions(+) diff --git a/stmhal/mpconfigport.h b/stmhal/mpconfigport.h index 28cd90bb0..f5110d8f0 100644 --- a/stmhal/mpconfigport.h +++ b/stmhal/mpconfigport.h @@ -44,6 +44,7 @@ */ #define MICROPY_ENABLE_LFN (1) #define MICROPY_LFN_CODE_PAGE (437) /* 1=SFN/ANSI 437=LFN/U.S.(OEM) */ +#define MICROPY_PY_BUILTINS_STR_UNICODE (0) #define MICROPY_PY_BUILTINS_FROZENSET (1) #define MICROPY_PY_SYS_EXIT (1) #define MICROPY_PY_SYS_STDFILES (1) diff --git a/unix/mpconfigport.h b/unix/mpconfigport.h index 1559bdb35..763b34ba3 100644 --- a/unix/mpconfigport.h +++ b/unix/mpconfigport.h @@ -41,6 +41,7 @@ #define MICROPY_LONGINT_IMPL (MICROPY_LONGINT_IMPL_MPZ) #define MICROPY_STREAMS_NON_BLOCK (1) #define MICROPY_OPT_COMPUTED_GOTO (1) +#define MICROPY_PY_BUILTINS_STR_UNICODE (0) #define MICROPY_PY_BUILTINS_FROZENSET (1) #define MICROPY_PY_SYS_EXIT (1) #define MICROPY_PY_SYS_PLATFORM "linux"