py/objringio: Add micropython.RingIO() interface for general use.

This commit adds a new `RingIO` type which exposes the internal ring-buffer
code for general use in Python programs.  It has the stream interface
making it similar to `StringIO` and `BytesIO`, except `RingIO` has a fixed
buffer size and is automatically safe when reads and writes are in
different threads or an IRQ.

This new type is enabled at the "extra features" ROM level.

Signed-off-by: Andrew Leech <andrew.leech@planetinnovation.com.au>

docs/library/micropython.rst

@@ -155,3 +155,71 @@ Functions
 
    There is a finite queue to hold the scheduled functions and `schedule()`
    will raise a `RuntimeError` if the queue is full.
+
+Classes
+-------
+
+.. class:: RingIO(size)
+.. class:: RingIO(buffer)
+   :noindex:
+
+   Provides a fixed-size ringbuffer for bytes with a stream interface. Can be
+   considered like a fifo queue variant of `io.BytesIO`.
+
+   When created with integer size a suitable buffer will be allocated.
+   Alternatively a `bytearray` or similar buffer protocol object can be provided
+   to the constructor for in-place use.
+
+   The classic ringbuffer algorithm is used which allows for any size buffer
+   to be used however one byte will be consumed for tracking. If initialised
+   with an integer size this will be accounted for, for example ``RingIO(16)``
+   will allocate a 17 byte buffer internally so it can hold 16 bytes of data.
+   When passing in a pre-allocated buffer however one byte less than its
+   original length will be available for storage, eg. ``RingIO(bytearray(16))``
+   will only hold 15 bytes of data.
+
+   A RingIO instance can be IRQ / thread safe when used to pass data in a single
+   direction eg. when written to in an IRQ and read from in a non-IRQ function
+   (or vice versa). This does not hold if you try to eg. write to a single instance
+   from both IRQ and non-IRQ code, this would often cause data corruption.
+
+    .. method:: RingIO.any()
+
+        Returns an integer counting the number of characters that can be read.
+
+    .. method:: RingIO.read([nbytes])
+
+        Read available characters. This is a non-blocking function. If ``nbytes``
+        is specified then read at most that many bytes, otherwise read as much
+        data as possible.
+
+        Return value: a bytes object containing the bytes read. Will be
+        zero-length bytes object if no data is available.
+
+    .. method:: RingIO.readline([nbytes])
+
+        Read a line, ending in a newline character or return if one exists in
+        the buffer, else return available bytes in buffer. If ``nbytes`` is
+        specified then read at most that many bytes.
+
+        Return value: a bytes object containing the line read.
+
+    .. method:: RingIO.readinto(buf[, nbytes])
+
+        Read available bytes into the provided ``buf``.  If ``nbytes`` is
+        specified then read at most that many bytes.  Otherwise, read at
+        most ``len(buf)`` bytes.
+
+        Return value: Integer count of the number of bytes read into ``buf``.
+
+    .. method:: RingIO.write(buf)
+
+        Non-blocking write of bytes from ``buf`` into the ringbuffer, limited
+        by the available space in the ringbuffer.
+
+        Return value: Integer count of bytes written.
+
+    .. method:: RingIO.close()
+
+        No-op provided as part of standard `stream` interface. Has no effect
+        on data in the ringbuffer.

ports/unix/coverage.c

@@ -692,6 +692,24 @@ static mp_obj_t extra_coverage(void) {
         ringbuf.iget = 0;
         ringbuf_put(&ringbuf, 0xaa);
         mp_printf(&mp_plat_print, "%d\n", ringbuf_get16(&ringbuf));
+
+        // ringbuf_put_bytes() / ringbuf_get_bytes() functions.
+        ringbuf.iput = 0;
+        ringbuf.iget = 0;
+        uint8_t *put = (uint8_t *)"abc123";
+        uint8_t get[7] = {0};
+        mp_printf(&mp_plat_print, "%d\n", ringbuf_put_bytes(&ringbuf, put, 7));
+        mp_printf(&mp_plat_print, "%d\n", ringbuf_get_bytes(&ringbuf, get, 7));
+        mp_printf(&mp_plat_print, "%s\n", get);
+        // Prefill ringbuffer.
+        for (size_t i = 0; i < sizeof(buf) - 3; ++i) {
+            ringbuf_put(&ringbuf, i);
+        }
+        // Should fail - too full.
+        mp_printf(&mp_plat_print, "%d\n", ringbuf_put_bytes(&ringbuf, put, 7));
+        // Should fail - buffer too big.
+        uint8_t large[sizeof(buf) + 5] = {0};
+        mp_printf(&mp_plat_print, "%d\n", ringbuf_put_bytes(&ringbuf, large, sizeof(large)));
     }
 
     // pairheap

py/modmicropython.c

@@ -200,6 +200,9 @@ static const mp_rom_map_elem_t mp_module_micropython_globals_table[] = {
     #if MICROPY_KBD_EXCEPTION
     { MP_ROM_QSTR(MP_QSTR_kbd_intr), MP_ROM_PTR(&mp_micropython_kbd_intr_obj) },
     #endif
+    #if MICROPY_PY_MICROPYTHON_RINGIO
+    { MP_ROM_QSTR(MP_QSTR_RingIO), MP_ROM_PTR(&mp_type_ringio) },
+    #endif
     #if MICROPY_ENABLE_SCHEDULER
     { MP_ROM_QSTR(MP_QSTR_schedule), MP_ROM_PTR(&mp_micropython_schedule_obj) },
     #endif

py/mpconfig.h

@@ -1306,6 +1306,11 @@ typedef double mp_float_t;
 #define MICROPY_PY_MICROPYTHON_HEAP_LOCKED (MICROPY_CONFIG_ROM_LEVEL_AT_LEAST_EVERYTHING)
 #endif
 
+// Support for micropython.RingIO()
+#ifndef MICROPY_PY_MICROPYTHON_RINGIO
+#define MICROPY_PY_MICROPYTHON_RINGIO (MICROPY_CONFIG_ROM_LEVEL_AT_LEAST_EXTRA_FEATURES)
+#endif
+
 // Whether to provide "array" module. Note that large chunk of the
 // underlying code is shared with "bytearray" builtin type, so to
 // get real savings, it should be disabled too.

py/obj.h

@@ -843,6 +843,7 @@ extern const mp_obj_type_t mp_type_bound_meth;
 extern const mp_obj_type_t mp_type_property;
 extern const mp_obj_type_t mp_type_stringio;
 extern const mp_obj_type_t mp_type_bytesio;
+extern const mp_obj_type_t mp_type_ringio;
 extern const mp_obj_type_t mp_type_reversed;
 extern const mp_obj_type_t mp_type_polymorph_iter;
 #if MICROPY_ENABLE_FINALISER

py/objringio.c

@@ -0,0 +1,130 @@
+/*
+ * This file is part of the MicroPython project, http://micropython.org/
+ *
+ * The MIT License (MIT)
+ *
+ * Copyright (c) 2024 Andrew Leech
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+ * THE SOFTWARE.
+ */
+
+#include "ringbuf.h"
+#include "py/mpconfig.h"
+
+#if MICROPY_PY_MICROPYTHON_RINGIO
+
+#include "py/runtime.h"
+#include "py/stream.h"
+
+typedef struct _micropython_ringio_obj_t {
+    mp_obj_base_t base;
+    ringbuf_t ringbuffer;
+} micropython_ringio_obj_t;
+
+static mp_obj_t micropython_ringio_make_new(const mp_obj_type_t *type, size_t n_args, size_t n_kw, const mp_obj_t *args) {
+    mp_arg_check_num(n_args, n_kw, 1, 1, false);
+    mp_int_t buff_size = -1;
+    mp_buffer_info_t bufinfo = {NULL, 0, 0};
+
+    if (!mp_get_buffer(args[0], &bufinfo, MP_BUFFER_RW)) {
+        buff_size = mp_obj_get_int(args[0]);
+    }
+    micropython_ringio_obj_t *self = mp_obj_malloc(micropython_ringio_obj_t, type);
+    if (bufinfo.buf != NULL) {
+        // buffer passed in, use it directly for ringbuffer.
+        self->ringbuffer.buf = bufinfo.buf;
+        self->ringbuffer.size = bufinfo.len;
+        self->ringbuffer.iget = self->ringbuffer.iput = 0;
+    } else {
+        // Allocate new buffer, add one extra to buff_size as ringbuf consumes one byte for tracking.
+        ringbuf_alloc(&(self->ringbuffer), buff_size + 1);
+    }
+    return MP_OBJ_FROM_PTR(self);
+}
+
+static mp_uint_t micropython_ringio_read(mp_obj_t self_in, void *buf_in, mp_uint_t size, int *errcode) {
+    micropython_ringio_obj_t *self = MP_OBJ_TO_PTR(self_in);
+    size = MIN(size, ringbuf_avail(&self->ringbuffer));
+    ringbuf_memcpy_get_internal(&(self->ringbuffer), buf_in, size);
+    *errcode = 0;
+    return size;
+}
+
+static mp_uint_t micropython_ringio_write(mp_obj_t self_in, const void *buf_in, mp_uint_t size, int *errcode) {
+    micropython_ringio_obj_t *self = MP_OBJ_TO_PTR(self_in);
+    size = MIN(size, ringbuf_free(&self->ringbuffer));
+    ringbuf_memcpy_put_internal(&(self->ringbuffer), buf_in, size);
+    *errcode = 0;
+    return size;
+}
+
+static mp_uint_t micropython_ringio_ioctl(mp_obj_t self_in, mp_uint_t request, uintptr_t arg, int *errcode) {
+    micropython_ringio_obj_t *self = MP_OBJ_TO_PTR(self_in);
+    switch (request) {
+        case MP_STREAM_POLL: {
+            mp_uint_t ret = 0;
+            if ((arg & MP_STREAM_POLL_RD) && ringbuf_avail(&self->ringbuffer) > 0) {
+                ret |= MP_STREAM_POLL_RD;
+            }
+            if ((arg & MP_STREAM_POLL_WR) && ringbuf_free(&self->ringbuffer) > 0) {
+                ret |= MP_STREAM_POLL_WR;
+            }
+            return ret;
+        }
+        case MP_STREAM_CLOSE:
+            return 0;
+    }
+    *errcode = MP_EINVAL;
+    return MP_STREAM_ERROR;
+}
+
+static mp_obj_t micropython_ringio_any(mp_obj_t self_in) {
+    micropython_ringio_obj_t *self = MP_OBJ_TO_PTR(self_in);
+    return MP_OBJ_NEW_SMALL_INT(ringbuf_avail(&self->ringbuffer));
+}
+static MP_DEFINE_CONST_FUN_OBJ_1(micropython_ringio_any_obj, micropython_ringio_any);
+
+static const mp_rom_map_elem_t micropython_ringio_locals_dict_table[] = {
+    { MP_ROM_QSTR(MP_QSTR_any), MP_ROM_PTR(&micropython_ringio_any_obj) },
+    { MP_ROM_QSTR(MP_QSTR_read), MP_ROM_PTR(&mp_stream_read_obj) },
+    { MP_ROM_QSTR(MP_QSTR_readline), MP_ROM_PTR(&mp_stream_unbuffered_readline_obj) },
+    { MP_ROM_QSTR(MP_QSTR_readinto), MP_ROM_PTR(&mp_stream_readinto_obj) },
+    { MP_ROM_QSTR(MP_QSTR_write), MP_ROM_PTR(&mp_stream_write_obj) },
+    { MP_ROM_QSTR(MP_QSTR_close), MP_ROM_PTR(&mp_stream_close_obj) },
+
+};
+static MP_DEFINE_CONST_DICT(micropython_ringio_locals_dict, micropython_ringio_locals_dict_table);
+
+static const mp_stream_p_t ringio_stream_p = {
+    .read = micropython_ringio_read,
+    .write = micropython_ringio_write,
+    .ioctl = micropython_ringio_ioctl,
+    .is_text = false,
+};
+
+MP_DEFINE_CONST_OBJ_TYPE(
+    mp_type_ringio,
+    MP_QSTR_RingIO,
+    MP_TYPE_FLAG_NONE,
+    make_new, micropython_ringio_make_new,
+    protocol, &ringio_stream_p,
+    locals_dict, &micropython_ringio_locals_dict
+    );
+
+#endif // MICROPY_PY_MICROPYTHON_RINGIO

py/py.cmake

@@ -95,6 +95,7 @@ set(MICROPY_SOURCE_PY
     ${MICROPY_PY_DIR}/objproperty.c
     ${MICROPY_PY_DIR}/objrange.c
     ${MICROPY_PY_DIR}/objreversed.c
+    ${MICROPY_PY_DIR}/objringio.c
     ${MICROPY_PY_DIR}/objset.c
     ${MICROPY_PY_DIR}/objsingleton.c
     ${MICROPY_PY_DIR}/objslice.c

py/py.mk

@@ -167,6 +167,7 @@ PY_CORE_O_BASENAME = $(addprefix py/,\
 	objnamedtuple.o \
 	objrange.o \
 	objreversed.o \
+	objringio.o \
 	objset.o \
 	objsingleton.o \
 	objslice.o \

py/ringbuf.c

@@ -24,8 +24,6 @@
  * THE SOFTWARE.
  */
 
-#include <string.h>
-
 #include "ringbuf.h"
 
 int ringbuf_get16(ringbuf_t *r) {
@@ -83,17 +81,7 @@ int ringbuf_get_bytes(ringbuf_t *r, uint8_t *data, size_t data_len) {
     if (ringbuf_avail(r) < data_len) {
         return (r->size <= data_len) ? -2 : -1;
     }
-    uint32_t iget = r->iget;
-    uint32_t iget_a = (iget + data_len) % r->size;
-    uint8_t *datap = data;
-    if (iget_a < iget) {
-        // Copy part of the data from the space left at the end of the buffer
-        memcpy(datap, r->buf + iget, r->size - iget);
-        datap += (r->size - iget);
-        iget = 0;
-    }
-    memcpy(datap, r->buf + iget, iget_a - iget);
-    r->iget = iget_a;
+    ringbuf_memcpy_get_internal(r, data, data_len);
     return 0;
 }
 
@@ -105,16 +93,6 @@ int ringbuf_put_bytes(ringbuf_t *r, const uint8_t *data, size_t data_len) {
     if (ringbuf_free(r) < data_len) {
         return (r->size <= data_len) ? -2 : -1;
     }
-    uint32_t iput = r->iput;
-    uint32_t iput_a = (iput + data_len) % r->size;
-    const uint8_t *datap = data;
-    if (iput_a < iput) {
-        // Copy part of the data to the end of the buffer
-        memcpy(r->buf + iput, datap, r->size - iput);
-        datap += (r->size - iput);
-        iput = 0;
-    }
-    memcpy(r->buf + iput, datap, iput_a - iput);
-    r->iput = iput_a;
+    ringbuf_memcpy_put_internal(r, data, data_len);
     return 0;
 }

py/ringbuf.h

@@ -28,10 +28,9 @@
 
 #include <stddef.h>
 #include <stdint.h>
+#include <string.h>
 
-#ifdef _MSC_VER
-#include "py/mpconfig.h" // For inline.
-#endif
+#include "py/mpconfig.h"
 
 typedef struct _ringbuf_t {
     uint8_t *buf;
@@ -91,6 +90,38 @@ static inline size_t ringbuf_avail(ringbuf_t *r) {
     return (r->size + r->iput - r->iget) % r->size;
 }
 
+static inline void ringbuf_memcpy_get_internal(ringbuf_t *r, uint8_t *data, size_t data_len) {
+    // No bounds / space checking is performed here so ensure available size is checked before running this
+    // otherwise data loss or buffer overflow can occur.
+    uint32_t iget = r->iget;
+    uint32_t iget_a = (iget + data_len) % r->size;
+    uint8_t *datap = data;
+    if (iget_a < iget) {
+        // Copy part of the data from the space left at the end of the buffer
+        memcpy(datap, r->buf + iget, r->size - iget);
+        datap += (r->size - iget);
+        iget = 0;
+    }
+    memcpy(datap, r->buf + iget, iget_a - iget);
+    r->iget = iget_a;
+}
+
+static inline void ringbuf_memcpy_put_internal(ringbuf_t *r, const uint8_t *data, size_t data_len) {
+    // No bounds / space checking is performed here so ensure free size is checked before running this
+    // otherwise data loss or buffer overflow can occur.
+    uint32_t iput = r->iput;
+    uint32_t iput_a = (iput + data_len) % r->size;
+    const uint8_t *datap = data;
+    if (iput_a < iput) {
+        // Copy part of the data to the end of the buffer
+        memcpy(r->buf + iput, datap, r->size - iput);
+        datap += (r->size - iput);
+        iput = 0;
+    }
+    memcpy(r->buf + iput, datap, iput_a - iput);
+    r->iput = iput_a;
+}
+
 // Note: big-endian. No-op if not enough room available for both bytes.
 int ringbuf_get16(ringbuf_t *r);
 int ringbuf_peek16(ringbuf_t *r);

tests/micropython/ringio.py

@@ -0,0 +1,48 @@
+# Check that micropython.RingIO works correctly.
+
+import micropython
+
+try:
+    micropython.RingIO
+except AttributeError:
+    print("SKIP")
+    raise SystemExit
+
+rb = micropython.RingIO(16)
+print(rb)
+
+print(rb.any())
+
+rb.write(b"\x00")
+print(rb.any())
+
+rb.write(b"\x00")
+print(rb.any())
+
+print(rb.read(2))
+print(rb.any())
+
+
+rb.write(b"\x00\x01")
+print(rb.read())
+
+print(rb.read(1))
+
+# Try to write more data than can fit at one go.
+print(rb.write(b"\x00\x01" * 10))
+print(rb.write(b"\x00"))
+print(rb.read())
+
+
+ba = bytearray(17)
+rb = micropython.RingIO(ba)
+print(rb)
+print(rb.write(b"\x00\x01" * 10))
+print(rb.write(b"\x00"))
+print(rb.read())
+
+try:
+    # Size must be int.
+    micropython.RingIO(None)
+except TypeError as ex:
+    print(ex)

tests/micropython/ringio.py.exp

@@ -0,0 +1,16 @@
+<RingIO>
+0
+1
+2
+b'\x00\x00'
+0
+b'\x00\x01'
+b''
+16
+0
+b'\x00\x01\x00\x01\x00\x01\x00\x01\x00\x01\x00\x01\x00\x01\x00\x01'
+<RingIO>
+16
+0
+b'\x00\x01\x00\x01\x00\x01\x00\x01\x00\x01\x00\x01\x00\x01\x00\x01'
+can't convert NoneType to int

tests/micropython/ringio_async.py

@@ -0,0 +1,36 @@
+# Check that micropython.RingIO works correctly with asyncio.Stream.
+
+import micropython
+
+try:
+    import asyncio
+
+    asyncio.StreamWriter
+    micropython.RingIO
+except (AttributeError, ImportError):
+    print("SKIP")
+    raise SystemExit
+
+rb = micropython.RingIO(16)
+rba = asyncio.StreamWriter(rb)
+
+data = b"ABC123" * 20
+print("w", len(data))
+
+
+async def data_writer():
+    global data
+    rba.write(data)
+    await rba.drain()
+
+
+async def main():
+    task = asyncio.create_task(data_writer())
+    await asyncio.sleep_ms(10)
+    read = await rba.readexactly(len(data))
+    print(read)
+    print("r", len(read))
+    print(read == data)
+
+
+asyncio.run(main())

tests/micropython/ringio_async.py.exp

@@ -0,0 +1,4 @@
+w 120
+b'ABC123ABC123ABC123ABC123ABC123ABC123ABC123ABC123ABC123ABC123ABC123ABC123ABC123ABC123ABC123ABC123ABC123ABC123ABC123ABC123'
+r 120
+True

tests/ports/unix/extra_coverage.py.exp

@@ -144,6 +144,11 @@ cc99
 22ff
 -1
 -1
+0
+0
+abc123
+-1
+-2
 # pairheap
 create: 0 0 0 0
 pop all: 0 1 2 3