Merge v1.12

.travis.yml

@@ -62,7 +62,10 @@ jobs:
     - stage: test
       env: NAME="unix coverage build and tests"
       install:
+        - sudo apt-get install python3-pip
         - sudo pip install cpp-coveralls
+        - sudo pip3 install setuptools
+        - sudo pip3 install pyelftools
         - gcc --version
         - python3 --version
       script:
@@ -78,6 +81,18 @@ jobs:
         - (cd tests && MICROPY_CPYTHON3=python3 MICROPY_MICROPYTHON=../ports/unix/micropython_coverage ./run-tests --via-mpy --emit native -d basics float micropython)
         # test when input script comes from stdin
         - cat tests/basics/0prelim.py | ports/unix/micropython_coverage | grep -q 'abc'
+        # test building native mpy modules
+        - make -C examples/natmod/features1 ARCH=x64
+        - make -C examples/natmod/features2 ARCH=x64
+        - make -C examples/natmod/btree ARCH=x64
+        - make -C examples/natmod/framebuf ARCH=x64
+        - make -C examples/natmod/uheapq ARCH=x64
+        - make -C examples/natmod/urandom ARCH=x64
+        - make -C examples/natmod/ure ARCH=x64
+        - make -C examples/natmod/uzlib ARCH=x64
+        # test importing .mpy generated by mpy_ld.py
+        - MICROPYPATH=examples/natmod/features2 ./ports/unix/micropython_coverage -m features2
+        - (cd tests && ./run-natmodtests.py extmod/{btree*,framebuf*,uheapq*,ure*,uzlib*}.py)
         # run coveralls coverage analysis (try to, even if some builds/tests failed)
         - (cd ports/unix && coveralls --root ../.. --build-root . --gcov $(which gcov) --gcov-options '\-o build-coverage/' --include py --include extmod)
       after_failure:

docs/conf.py

@@ -74,7 +74,7 @@ copyright = '2014-2019, Damien P. George, Paul Sokolovsky, and contributors'
 #
 # We don't follow "The short X.Y version" vs "The full version, including alpha/beta/rc tags"
 # breakdown, so use the same version identifier for both to avoid confusion.
-version = release = '1.11'
+version = release = '1.12'
 
 # The language for content autogenerated by Sphinx. Refer to documentation
 # for a list of supported languages.

docs/develop/cmodules.rst

@@ -1,3 +1,5 @@
+.. _cmodules:
+
 MicroPython external C modules
 ==============================
 
@@ -17,6 +19,10 @@ more sense to keep this external to the main MicroPython repository.
 This chapter describes how to compile such external modules into the
 MicroPython executable or firmware image.
 
+An alternative approach is to use :ref:`natmod` which allows writing custom C
+code that is placed in a .mpy file, which can be imported dynamically in to
+a running MicroPython system without the need to recompile the main firmware.
+
 
 Structure of an external C module
 ---------------------------------

docs/develop/index.rst

@@ -11,3 +11,4 @@ See the `getting started guide
 
    cmodules.rst
    qstr.rst
+   natmod.rst

docs/develop/natmod.rst

@@ -0,0 +1,202 @@
+.. _natmod:
+
+Native machine code in .mpy files
+=================================
+
+This section describes how to build and work with .mpy files that contain native
+machine code from a language other than Python.  This allows you to
+write code in a language like C, compile and link it into a .mpy file, and then
+import this file like a normal Python module.  This can be used for implementing
+functionality which is performance critical, or for including an existing
+library written in another language.
+
+One of the main advantages of using native .mpy files is that native machine code
+can be imported by a script dynamically, without the need to rebuild the main
+MicroPython firmware.  This is in contrast to :ref:`cmodules` which also allows
+defining custom modules in C but they must be compiled into the main firmware image.
+
+The focus here is on using C to build native modules, but in principle any
+language which can be compiled to stand-alone machine code can be put into a
+.mpy file.
+
+A native .mpy module is built using the ``mpy_ld.py`` tool, which is found in the
+``tools/`` directory of the project.  This tool takes a set of object files
+(.o files) and links them together to create a native .mpy files.
+
+Supported features and limitations
+----------------------------------
+
+A .mpy file can contain MicroPython bytecode and/or native machine code.  If it
+contains native machine code then the .mpy file has a specific architecture
+associated with it.  Current supported architectures are (these are the valid
+options for the ``ARCH`` variable, see below):
+
+* ``x86`` (32 bit)
+* ``x64`` (64 bit x86)
+* ``armv7m`` (ARM Thumb 2, eg Cortex-M3)
+* ``armv7emsp`` (ARM Thumb 2, single precision float, eg Cortex-M4F, Cortex-M7)
+* ``armv7emdp`` (ARM Thumb 2, double precision float, eg Cortex-M7)
+* ``xtensa`` (non-windowed, eg ESP8266)
+* ``xtensawin`` (windowed with window size 8, eg ESP32)
+
+When compiling and linking the native .mpy file the architecture must be chosen
+and the corresponding file can only be imported on that architecture.  For more
+details about .mpy files see :ref:`mpy_files`.
+
+Native code must be compiled as position independent code (PIC) and use a global
+offset table (GOT), although the details of this varies from architecture to
+architecture.  When importing .mpy files with native code the import machinery
+is able to do some basic relocation of the native code.  This includes
+relocating text, rodata and BSS sections.
+
+Supported features of the linker and dynamic loader are:
+
+* executable code (text)
+* read-only data (rodata), including strings and constant data (arrays, structs, etc)
+* zeroed data (BSS)
+* pointers in text to text, rodata and BSS
+* pointers in rodata to text, rodata and BSS
+
+The known limitations are:
+
+* data sections are not supported; workaround: use BSS data and initialise the
+  data values explicitly
+
+* static BSS variables are not supported; workaround: use global BSS variables
+
+So, if your C code has writable data, make sure the data is defined globally,
+without an initialiser, and only written to within functions.
+
+Defining a native module
+------------------------
+
+A native .mpy module is defined by a set of files that are used to build the .mpy.
+The filesystem layout consists of two main parts, the source files and the Makefile:
+
+* In the simplest case only a single C source file is required, which contains all
+  the code that will be compiled into the .mpy module.  This C source code must
+  include the ``py/dynruntime.h`` file to access the MicroPython dynamic API, and
+  must at least define a function called ``mpy_init``.  This function will be the
+  entry point of the module, called when the module is imported.
+
+  The module can be split into multiple C source files if desired.  Parts of the
+  module can also be implemented in Python.  All source files should be listed in
+  the Makefile, by adding them to the ``SRC`` variable (see below).  This includes
+  both C source files as well as any Python files which will be included in the
+  resulting .mpy file.
+
+* The ``Makefile`` contains the build configuration for the module and list the
+  source files used to build the .mpy module.  It should define ``MPY_DIR`` as the
+  location of the MicroPython repository (to find header files, the relevant Makefile
+  fragment, and the ``mpy_ld.py`` tool), ``MOD`` as the name of the module, ``SRC``
+  as the list of source files, optionally specify the machine architecture via ``ARCH``,
+  and then include ``py/dynruntime.mk``.
+
+Minimal example
+---------------
+
+This section provides a fully working example of a simple module named ``factorial``.
+This module provides a single function ``factorial.factorial(x)`` which computes the
+factorial of the input and returns the result.
+
+Directory layout::
+
+    factorial/
+    ├── factorial.c
+    └── Makefile
+
+The file ``factorial.c`` contains:
+
+.. code-block:: c
+
+    // Include the header file to get access to the MicroPython API
+    #include "py/dynruntime.h"
+
+    // Helper function to compute factorial
+    STATIC mp_int_t factorial_helper(mp_int_t x) {
+        if (x == 0) {
+            return 1;
+        }
+        return x * factorial_helper(x - 1);
+    }
+
+    // This is the function which will be called from Python, as factorial(x)
+    STATIC mp_obj_t factorial(mp_obj_t x_obj) {
+        // Extract the integer from the MicroPython input object
+        mp_int_t x = mp_obj_get_int(x_obj);
+        // Calculate the factorial
+        mp_int_t result = factorial_helper(x);
+        // Convert the result to a MicroPython integer object and return it
+        return mp_obj_new_int(result);
+    }
+    // Define a Python reference to the function above
+    STATIC MP_DEFINE_CONST_FUN_OBJ_1(factorial_obj, factorial);
+
+    // This is the entry point and is called when the module is imported
+    mp_obj_t mpy_init(mp_obj_fun_bc_t *self, size_t n_args, size_t n_kw, mp_obj_t *args) {
+        // This must be first, it sets up the globals dict and other things
+        MP_DYNRUNTIME_INIT_ENTRY
+
+        // Make the function available in the module's namespace
+        mp_store_global(MP_QSTR_factorial, MP_OBJ_FROM_PTR(&factorial_obj));
+
+        // This must be last, it restores the globals dict
+        MP_DYNRUNTIME_INIT_EXIT
+    }
+
+The file ``Makefile`` contains:
+
+.. code-block:: make
+
+    # Location of top-level MicroPython directory
+    MPY_DIR = ../../..
+
+    # Name of module
+    MOD = features0
+
+    # Source files (.c or .py)
+    SRC = features0.c
+
+    # Architecture to build for (x86, x64, armv7m, xtensa, xtensawin)
+    ARCH = x64
+
+    # Include to get the rules for compiling and linking the module
+    include $(MPY_DIR)/py/dynruntime.mk
+
+Compiling the module
+--------------------
+
+Be sure to select the correct ``ARCH`` for the target you are going to run on.
+Then build with::
+
+    $ make
+
+Without modifying the Makefile you can specify the target architecture via::
+
+    $ make ARCH=armv7m
+
+Module usage in MicroPython
+---------------------------
+
+Once the module is built there should be a file called ``factorial.mpy``.  Copy
+this so it is accessible on the filesystem of your MicroPython system and can be
+found in the import path.  The module con now be accessed in Python just like any
+other module, for example::
+
+    import factorial
+    print(factorial.factorial(10))
+    # should display 3628800
+
+Further examples
+----------------
+
+See ``examples/natmod/`` for further examples which show many of the available
+features of native .mpy modules.  Such features include:
+
+* using multiple C source files
+* including Python code alongside C code
+* rodata and BSS data
+* memory allocation
+* use of floating point
+* exception handling
+* including external C libraries

docs/esp32/general.rst

@@ -52,6 +52,7 @@ For your convenience, some of technical specifications are provided below:
 * I2S: 2
 * ADC: 12-bit SAR ADC up to 18 channels
 * DAC: 2 8-bit DACs
+* RMT: 8 channels allowing accurate pulse transmit/receive
 * Programming: using BootROM bootloader from UART - due to external FlashROM
   and always-available BootROM bootloader, the ESP32 is not brickable
 

docs/esp32/quickref.rst

@@ -359,12 +359,26 @@ Notes:
   To further reduce power consumption it is possible to disable the internal pullups::
 
     p1 = Pin(4, Pin.IN, Pin.PULL_HOLD)
-    
+
   After leaving deepsleep it may be necessary to un-hold the pin explicitly (e.g. if
   it is an output pin) via::
-    
+
     p1 = Pin(4, Pin.OUT, None)
 
+RMT
+---
+
+The RMT is ESP32-specific and allows generation of accurate digital pulses with
+12.5ns resolution.  See :ref:`esp32.RMT <esp32.RMT>` for details.  Usage is::
+
+    import esp32
+    from machine import Pin
+
+    r = esp32.RMT(0, pin=Pin(18), clock_div=8)
+    r   # RMT(channel=0, pin=18, source_freq=80000000, clock_div=8)
+    # The channel resolution is 100ns (1/(source_freq/clock_div)).
+    r.write_pulses((1, 20, 2, 40), start=0) # Send 0 for 100ns, 1 for 2000ns, 0 for 200ns, 1 for 4000ns
+
 OneWire driver
 --------------
 

docs/esp8266/quickref.rst

@@ -249,6 +249,10 @@ See :ref:`machine.RTC <machine.RTC>` ::
     ntptime.settime() # set the rtc datetime from the remote server
     rtc.datetime()    # get the date and time in UTC
 
+.. note:: Not all methods are implemented: `RTC.now()`, `RTC.irq(handler=*) <RTC.irq>`
+          (using a custom handler), `RTC.init()` and `RTC.deinit()` are
+          currently not supported.
+
 Deep-sleep mode
 ---------------
 

docs/esp8266/tutorial/network_tcp.rst

@@ -44,7 +44,7 @@ Now that we are connected we can download and display the data::
     ...     data = s.recv(500)
     ...     print(str(data, 'utf8'), end='')
     ...
- 
+
 When this loop executes it should start showing the animation (use ctrl-C to
 interrupt it).
 

docs/library/esp32.rst

@@ -1,3 +1,5 @@
+.. currentmodule:: esp32
+
 :mod:`esp32` --- functionality specific to the ESP32
 ====================================================
 
@@ -56,10 +58,14 @@ This class gives access to the partitions in the device's flash memory.
     Returns a 6-tuple ``(type, subtype, addr, size, label, encrypted)``.
 
 .. method:: Partition.readblocks(block_num, buf)
+.. method:: Partition.readblocks(block_num, buf, offset)
 .. method:: Partition.writeblocks(block_num, buf)
+.. method:: Partition.writeblocks(block_num, buf, offset)
 .. method:: Partition.ioctl(cmd, arg)
 
-    These methods implement the block protocol defined by :class:`uos.AbstractBlockDev`.
+    These methods implement the simple and :ref:`extended
+    <block-device-interface>` block protocol defined by
+    :class:`uos.AbstractBlockDev`.
 
 .. method:: Partition.set_boot()
 
@@ -82,6 +88,91 @@ Constants
 
     Used in `Partition.find` to specify the partition type.
 
+
+.. _esp32.RMT:
+
+RMT
+---
+
+The RMT (Remote Control) module, specific to the ESP32, was originally designed
+to send and receive infrared remote control signals. However, due to a flexible
+design and very accurate (as low as 12.5ns) pulse generation, it can also be
+used to transmit or receive many other types of digital signals::
+
+    import esp32
+    from machine import Pin
+
+    r = esp32.RMT(0, pin=Pin(18), clock_div=8)
+    r  # RMT(channel=0, pin=18, source_freq=80000000, clock_div=8)
+    # The channel resolution is 100ns (1/(source_freq/clock_div)).
+    r.write_pulses((1, 20, 2, 40), start=0)  # Send 0 for 100ns, 1 for 2000ns, 0 for 200ns, 1 for 4000ns
+
+The input to the RMT module is an 80MHz clock (in the future it may be able to
+configure the input clock but, for now, it's fixed). ``clock_div`` *divides*
+the clock input which determines the resolution of the RMT channel. The
+numbers specificed in ``write_pulses`` are multiplied by the resolution to
+define the pulses.
+
+``clock_div`` is an 8-bit divider (0-255) and each pulse can be defined by
+multiplying the resolution by a 15-bit (0-32,768) number. There are eight
+channels (0-7) and each can have a different clock divider.
+
+So, in the example above, the 80MHz clock is divided by 8. Thus the
+resolution is (1/(80Mhz/8)) 100ns. Since the ``start`` level is 0 and toggles
+with each number, the bitstream is ``0101`` with durations of [100ns, 2000ns,
+100ns, 4000ns].
+
+For more details see Espressif's `ESP-IDF RMT documentation.
+<https://docs.espressif.com/projects/esp-idf/en/latest/api-reference/peripherals/rmt.html>`_.
+
+.. Warning::
+   The current MicroPython RMT implementation lacks some features, most notably
+   receiving pulses and carrier transmit. RMT should be considered a
+   *beta feature* and the interface may change in the future.
+
+
+.. class:: RMT(channel, \*, pin=None, clock_div=8)
+
+    This class provides access to one of the eight RMT channels. *channel* is
+    required and identifies which RMT channel (0-7) will be configured. *pin*,
+    also required, configures which Pin is bound to the RMT channel. *clock_div*
+    is an 8-bit clock divider that divides the source clock (80MHz) to the RMT
+    channel allowing the resolution to be specified.
+
+.. method:: RMT.source_freq()
+
+    Returns the source clock frequency. Currently the source clock is not
+    configurable so this will always return 80MHz.
+
+.. method:: RMT.clock_div()
+
+    Return the clock divider. Note that the channel resolution is
+    ``1 / (source_freq / clock_div)``.
+
+.. method:: RMT.wait_done(timeout=0)
+
+    Returns True if `RMT.write_pulses` has completed.
+
+    If *timeout* (defined in ticks of ``source_freq / clock_div``) is specified
+    the method will wait for *timeout* or until `RMT.write_pulses` is complete,
+    returning ``False`` if the channel continues to transmit.
+
+.. Warning::
+    Avoid using ``wait_done()`` if looping is enabled.
+
+.. method:: RMT.loop(enable_loop)
+
+    Configure looping on the channel, allowing a stream of pulses to be
+    indefinitely repeated. *enable_loop* is bool, set to True to enable looping.
+
+.. method:: RMT.write_pulses(pulses, start)
+
+    Begin sending *pulses*, a list or tuple defining the stream of pulses. The
+    length of each pulse is defined by a number to be multiplied by the channel
+    resolution ``(1 / (source_freq / clock_div))``. *start* defines whether the
+    stream starts at 0 or 1.
+
+
 The Ultra-Low-Power co-processor
 --------------------------------
 

docs/library/lcd160cr.rst

@@ -340,7 +340,7 @@ Advanced commands
 .. method:: LCD160CR.set_scroll_win_param(win, param, value)
 
     Set a single parameter of a scrolling window region:
-    
+
         - *win* is the window id, 0..8.
         - *param* is the parameter number to configure, 0..7, and corresponds
           to the parameters in the `set_scroll_win` method.

docs/library/machine.RTC.rst

@@ -27,7 +27,7 @@ Methods
 .. method:: RTC.init(datetime)
 
    Initialise the RTC. Datetime is a tuple of the form:
-   
+
       ``(year, month, day[, hour[, minute[, second[, microsecond[, tzinfo]]]]])``
 
 .. method:: RTC.now()

docs/library/machine.TimerWiPy.rst

@@ -47,9 +47,9 @@ Methods
        tim.init(Timer.ONE_SHOT, width=32)   # one shot 32-bit timer
 
    Keyword arguments:
-   
+
      - ``mode`` can be one of:
-     
+
        - ``TimerWiPy.ONE_SHOT`` - The timer runs once until the configured 
          period of the channel expires.
        - ``TimerWiPy.PERIODIC`` - The timer runs periodically at the configured 
@@ -70,7 +70,7 @@ Methods
    object is returned (or ``None`` if there is no previous channel).
 
    Otherwise, a TimerChannel object is initialized and returned.
-   
+
    The operating mode is is the one configured to the Timer object that was used to
    create the channel.
 

docs/library/pyb.ADC.rst

@@ -19,7 +19,7 @@ Usage::
     val = adc.read_core_vref()          # read MCU VREF
     val = adc.read_vref()               # read MCU supply voltage
 
- 
+
 Constructors
 ------------
 

docs/library/pyb.Accel.rst

@@ -19,7 +19,7 @@ Constructors
 .. class:: pyb.Accel()
 
    Create and return an accelerometer object.
-   
+
 Methods
 -------
 

docs/library/pyb.CAN.rst

@@ -92,7 +92,7 @@ Methods
 
    Force a software restart of the CAN controller without resetting its
    configuration.
-   
+
    If the controller enters the bus-off state then it will no longer participate
    in bus activity.  If the controller is not configured to automatically restart
    (see :meth:`~CAN.init()`) then this method can be used to trigger a restart,

docs/library/pyb.ExtInt.rst

@@ -54,7 +54,7 @@ Constructors
 .. class:: pyb.ExtInt(pin, mode, pull, callback)
 
    Create an ExtInt object:
-   
+
      - ``pin`` is the pin on which to enable the interrupt (can be a pin object or any valid pin name).
      - ``mode`` can be one of:
        - ``ExtInt.IRQ_RISING`` - trigger on a rising edge;

docs/library/pyb.Flash.rst

@@ -0,0 +1,52 @@
+.. currentmodule:: pyb
+.. _pyb.Flash:
+
+class Flash -- access to built-in flash storage
+===============================================
+
+The Flash class allows direct access to the primary flash device on the pyboard.
+
+In most cases, to store persistent data on the device, you'll want to use a
+higher-level abstraction, for example the filesystem via Python's standard file
+API, but this interface is useful to :ref:`customise the filesystem
+configuration <filesystem>` or implement a low-level storage system for your
+application.
+
+Constructors
+------------
+
+.. class:: pyb.Flash()
+
+   Create and return a block device that represents the flash device presented
+   to the USB mass storage interface.
+
+   It includes a virtual partition table at the start, and the actual flash
+   starts at block ``0x100``.
+
+   This constructor is deprecated and will be removed in a future version of MicroPython.
+
+.. class:: pyb.Flash(\*, start=-1, len=-1)
+
+   Create and return a block device that accesses the flash at the specified offset. The length defaults to the remaining size of the device.
+
+   The *start* and *len* offsets are in bytes, and must be a multiple of the block size (typically 512 for internal flash).
+
+Methods
+-------
+
+.. method:: Flash.readblocks(block_num, buf)
+.. method:: Flash.readblocks(block_num, buf, offset)
+.. method:: Flash.writeblocks(block_num, buf)
+.. method:: Flash.writeblocks(block_num, buf, offset)
+.. method:: Flash.ioctl(cmd, arg)
+
+    These methods implement the simple and :ref:`extended
+    <block-device-interface>` block protocol defined by
+    :class:`uos.AbstractBlockDev`.
+
+Hardware Note
+-------------
+
+On boards with external spiflash (e.g. Pyboard D), the MicroPython firmware will
+be configured to use that as the primary flash storage. On all other boards, the
+internal flash inside the :term:`MCU` will be used.

docs/library/pyb.LCD.rst

@@ -63,13 +63,13 @@ Methods
 .. method:: LCD.fill(colour)
 
    Fill the screen with the given colour (0 or 1 for white or black).
-   
+
    This method writes to the hidden buffer.  Use ``show()`` to show the buffer.
 
 .. method:: LCD.get(x, y)
 
    Get the pixel at the position ``(x, y)``.  Returns 0 or 1.
-   
+
    This method reads from the visible buffer.
 
 .. method:: LCD.light(value)
@@ -79,7 +79,7 @@ Methods
 .. method:: LCD.pixel(x, y, colour)
 
    Set the pixel at ``(x, y)`` to the given colour (0 or 1).
-   
+
    This method writes to the hidden buffer.  Use ``show()`` to show the buffer.
 
 .. method:: LCD.show()
@@ -89,7 +89,7 @@ Methods
 .. method:: LCD.text(str, x, y, colour)
 
    Draw the given text to the position ``(x, y)`` using the given colour (0 or 1).
-   
+
    This method writes to the hidden buffer.  Use ``show()`` to show the buffer.
 
 .. method:: LCD.write(str)

docs/library/pyb.LED.rst

@@ -13,7 +13,7 @@ Constructors
 .. class:: pyb.LED(id)
 
    Create an LED object associated with the given LED:
-   
+
      - ``id`` is the LED number, 1-4.
 
 

docs/library/pyb.RTC.rst

@@ -28,13 +28,13 @@ Methods
 .. method:: RTC.datetime([datetimetuple])
 
    Get or set the date and time of the RTC.
-   
+
    With no arguments, this method returns an 8-tuple with the current
    date and time.  With 1 argument (being an 8-tuple) it sets the date
    and time (and ``subseconds`` is reset to 255).
-   
+
    The 8-tuple has the following format:
-   
+
        (year, month, day, weekday, hours, minutes, seconds, subseconds)
 
    ``weekday`` is 1-7 for Monday through Sunday.

docs/library/pyb.Timer.rst

@@ -112,7 +112,7 @@ Methods
 .. method:: Timer.deinit()
 
    Deinitialises the timer.
-   
+
    Disables the callback (and the associated irq).
 
    Disables any channel callbacks (and the associated irq).
@@ -191,7 +191,7 @@ Methods
      - Read the encoder value using the timer.counter() method.
      - Only works on CH1 and CH2 (and not on CH1N or CH2N)
      - The channel number is ignored when setting the encoder mode.
-       
+
    PWM Example::
 
        timer = pyb.Timer(2, freq=1000)

docs/library/pyb.USB_HID.rst

@@ -24,11 +24,11 @@ Methods
 .. method:: USB_HID.recv(data, \*, timeout=5000)
 
    Receive data on the bus:
-   
+
      - ``data`` can be an integer, which is the number of bytes to receive,
        or a mutable buffer, which will be filled with received bytes.
      - ``timeout`` is the timeout in milliseconds to wait for the receive.
-   
+
    Return value: if ``data`` is an integer then a new buffer of the bytes received,
    otherwise the number of bytes read into ``data`` is returned.
 

docs/library/pyb.USB_VCP.rst

@@ -92,21 +92,21 @@ Methods
 .. method:: USB_VCP.recv(data, \*, timeout=5000)
 
    Receive data on the bus:
-   
+
      - ``data`` can be an integer, which is the number of bytes to receive,
        or a mutable buffer, which will be filled with received bytes.
      - ``timeout`` is the timeout in milliseconds to wait for the receive.
-   
+
    Return value: if ``data`` is an integer then a new buffer of the bytes received,
    otherwise the number of bytes read into ``data`` is returned.
 
 .. method:: USB_VCP.send(data, \*, timeout=5000)
 
    Send data over the USB VCP:
-   
+
      - ``data`` is the data to send (an integer to send, or a buffer object).
      - ``timeout`` is the timeout in milliseconds to wait for the send.
-   
+
    Return value: number of bytes sent.
 
 

docs/library/pyb.rst

@@ -20,7 +20,7 @@ Time related functions
 .. function:: millis()
 
    Returns the number of milliseconds since the board was last reset.
-   
+
    The result is always a MicroPython smallint (31-bit signed number), so
    after 2^30 milliseconds (about 12.4 days) this will start to return
    negative numbers.
@@ -32,7 +32,7 @@ Time related functions
 .. function:: micros()
 
    Returns the number of microseconds since the board was last reset.
-   
+
    The result is always a MicroPython smallint (31-bit signed number), so
    after 2^30 microseconds (about 17.8 minutes) this will start to return
    negative numbers.
@@ -44,10 +44,10 @@ Time related functions
 .. function:: elapsed_millis(start)
 
    Returns the number of milliseconds which have elapsed since ``start``.
-   
+
    This function takes care of counter wrap, and always returns a positive
    number. This means it can be used to measure periods up to about 12.4 days.
-   
+
    Example::
 
        start = pyb.millis()
@@ -57,10 +57,10 @@ Time related functions
 .. function:: elapsed_micros(start)
 
    Returns the number of microseconds which have elapsed since ``start``.
-   
+
    This function takes care of counter wrap, and always returns a positive
    number. This means it can be used to measure periods up to about 17.8 minutes.
-   
+
    Example::
 
        start = pyb.micros()
@@ -309,6 +309,7 @@ Classes
    pyb.CAN.rst
    pyb.DAC.rst
    pyb.ExtInt.rst
+   pyb.Flash.rst
    pyb.I2C.rst
    pyb.LCD.rst
    pyb.LED.rst

docs/library/ubluetooth.rst

@@ -12,6 +12,9 @@ roles concurrently.
 This API is intended to match the low-level Bluetooth protocol and provide
 building-blocks for higher-level abstractions such as specific device types.
 
+.. note:: This module is still under development and its classes, functions,
+          methods and constants are subject to change.
+
 class BLE
 ---------
 
@@ -32,14 +35,27 @@ Configuration
 
     The radio must be made active before using any other methods on this class.
 
-.. method:: BLE.config(name)
+.. method:: BLE.config('param')
+            BLE.config(param=value, ...)
 
-    Queries a configuration value by *name*. Currently supported values are:
+    Get or set configuration values of the BLE interface.  To get a value the
+    parameter name should be quoted as a string, and just one parameter is
+    queried at a time.  To set values use the keyword syntax, and one ore more
+    parameter can be set at a time.
+
+    Currently supported values are:
 
     - ``'mac'``: Returns the device MAC address. If a device has a fixed address
       (e.g. PYBD) then it will be returned. Otherwise (e.g. ESP32) a random
       address will be generated when the BLE interface is made active.
 
+    - ``'rxbuf'``: Set the size in bytes of the internal buffer used to store
+      incoming events.  This buffer is global to the entire BLE driver and so
+      handles incoming data for all events, including all characteristics.
+      Increasing this allows better handling of bursty incoming data (for
+      example scan results) and the ability for a central role to receive
+      larger characteristic values.
+
 Event Handling
 --------------
 
@@ -310,12 +326,23 @@ Central Role (GATT Client)
 
     On success, the ``_IRQ_GATTC_READ_RESULT`` event will be raised.
 
-.. method:: BLE.gattc_write(conn_handle, value_handle, data)
+.. method:: BLE.gattc_write(conn_handle, value_handle, data, mode=0)
 
     Issue a remote write to a connected peripheral for the specified
     characteristic or descriptor handle.
 
-    On success, the ``_IRQ_GATTC_WRITE_STATUS`` event will be raised.
+    The argument *mode* specifies the write behaviour, with the currently
+    supported values being:
+
+        * ``mode=0`` (default) is a write-without-response: the write will
+          be sent to the remote peripheral but no confirmation will be
+          returned, and no event will be raised.
+        * ``mode=1`` is a write-with-response: the remote peripheral is
+          requested to send a response/acknowledgement that it received the
+          data.
+
+    If a response is received from the remote peripheral the
+    ``_IRQ_GATTC_WRITE_STATUS`` event will be raised.
 
 
 class UUID

docs/library/uos.rst

@@ -178,6 +178,43 @@ represented by VFS classes.
 
         Build a FAT filesystem on *block_dev*.
 
+.. class:: VfsLfs1(block_dev)
+
+    Create a filesystem object that uses the `littlefs v1 filesystem format`_.
+    Storage of the littlefs filesystem is provided by *block_dev*, which must
+    support the :ref:`extended interface <block-device-interface>`.
+    Objects created by this constructor can be mounted using :func:`mount`.
+
+    See :ref:`filesystem` for more information.
+
+    .. staticmethod:: mkfs(block_dev)
+
+        Build a Lfs1 filesystem on *block_dev*.
+
+    .. note:: There are reports of littlefs v1 failing in certain situations,
+              for details see `littlefs issue 347`_.
+
+.. class:: VfsLfs2(block_dev)
+
+    Create a filesystem object that uses the `littlefs v2 filesystem format`_.
+    Storage of the littlefs filesystem is provided by *block_dev*, which must
+    support the :ref:`extended interface <block-device-interface>`.
+    Objects created by this constructor can be mounted using :func:`mount`.
+
+    See :ref:`filesystem` for more information.
+
+    .. staticmethod:: mkfs(block_dev)
+
+        Build a Lfs2 filesystem on *block_dev*.
+
+    .. note:: There are reports of littlefs v2 failing in certain situations,
+              for details see `littlefs issue 295`_.
+
+.. _littlefs v1 filesystem format: https://github.com/ARMmbed/littlefs/tree/v1
+.. _littlefs v2 filesystem format: https://github.com/ARMmbed/littlefs
+.. _littlefs issue 295: https://github.com/ARMmbed/littlefs/issues/295
+.. _littlefs issue 347: https://github.com/ARMmbed/littlefs/issues/347
+
 Block devices
 -------------
 
@@ -187,9 +224,19 @@ implementation of this class will usually allow access to the memory-like
 functionality a piece of hardware (like flash memory).  A block device can be
 used by a particular filesystem driver to store the data for its filesystem.
 
+.. _block-device-interface:
+
+Simple and extended interface
+.............................
+
 There are two compatible signatures for the ``readblocks`` and ``writeblocks``
 methods (see below), in order to support a variety of use cases.  A given block
-device may implement one form or the other, or both at the same time.
+device may implement one form or the other, or both at the same time. The second
+form (with the offset parameter) is referred to as the "extended interface".
+
+Some filesystems (such as littlefs) that require more control over write
+operations, for example writing to sub-block regions without erasing, may require
+that the block device supports the extended interface.
 
 .. class:: AbstractBlockDev(...)
 
@@ -247,64 +294,5 @@ device may implement one form or the other, or both at the same time.
             (*arg* is unused)
           - 6 -- erase a block, *arg* is the block number to erase
 
-By way of example, the following class will implement a block device that stores
-its data in RAM using a ``bytearray``::
-
-    class RAMBlockDev:
-        def __init__(self, block_size, num_blocks):
-            self.block_size = block_size
-            self.data = bytearray(block_size * num_blocks)
-
-        def readblocks(self, block_num, buf):
-            for i in range(len(buf)):
-                buf[i] = self.data[block_num * self.block_size + i]
-
-        def writeblocks(self, block_num, buf):
-            for i in range(len(buf)):
-                self.data[block_num * self.block_size + i] = buf[i]
-
-        def ioctl(self, op, arg):
-            if op == 4: # get number of blocks
-                return len(self.data) // self.block_size
-            if op == 5: # get block size
-                return self.block_size
-
-It can be used as follows::
-
-    import uos
-
-    bdev = RAMBlockDev(512, 50)
-    uos.VfsFat.mkfs(bdev)
-    vfs = uos.VfsFat(bdev)
-    uos.mount(vfs, '/ramdisk')
-
-An example of a block device that supports both signatures and behaviours of
-the :meth:`readblocks` and :meth:`writeblocks` methods is::
-
-    class RAMBlockDev:
-        def __init__(self, block_size, num_blocks):
-            self.block_size = block_size
-            self.data = bytearray(block_size * num_blocks)
-
-        def readblocks(self, block, buf, offset=0):
-            addr = block_num * self.block_size + offset
-            for i in range(len(buf)):
-                buf[i] = self.data[addr + i]
-
-        def writeblocks(self, block_num, buf, offset=None):
-            if offset is None:
-                # do erase, then write
-                for i in range(len(buf) // self.block_size):
-                    self.ioctl(6, block_num + i)
-                offset = 0
-            addr = block_num * self.block_size + offset
-            for i in range(len(buf)):
-                self.data[addr + i] = buf[i]
-
-        def ioctl(self, op, arg):
-            if op == 4: # block count
-                return len(self.data) // self.block_size
-            if op == 5: # block size
-                return self.block_size
-            if op == 6: # block erase
-                return 0
+See :ref:`filesystem` for example implementations of block devices using both
+protocols.

docs/pyboard/quickref.rst

@@ -66,7 +66,7 @@ See :ref:`pyb.LED <pyb.LED>`. ::
     led.toggle()
     led.on()
     led.off()
-    
+
     # LEDs 3 and 4 support PWM intensity (0-255)
     LED(4).intensity()    # get intensity
     LED(4).intensity(128) # set intensity to half

docs/pyboard/tutorial/amp_skin.rst

@@ -60,7 +60,7 @@ on your pyboard (either on the flash or the SD card in the top-level directory).
 or to convert any file you have with the command::
 
     avconv -i original.wav -ar 22050 -codec pcm_u8 test.wav
-    
+
 Then you can do::
 
     >>> import wave

docs/pyboard/tutorial/fading_led.rst

@@ -27,10 +27,10 @@ For this tutorial, we will use the ``X1`` pin. Connect one end of the resistor t
 Code
 ----
 By examining the :ref:`pyboard_quickref`, we see that ``X1`` is connected to channel 1 of timer 5 (``TIM5 CH1``). Therefore we will first create a ``Timer`` object for timer 5, then create a ``TimerChannel`` object for channel 1::
-    
+
     from pyb import Timer
     from time import sleep
-    
+
     # timer 5 will be created with a frequency of 100 Hz
     tim = pyb.Timer(5, freq=100)
     tchannel = tim.channel(1, Timer.PWM, pin=pyb.Pin.board.X1, pulse_width=0)
@@ -47,16 +47,16 @@ To achieve the fading effect shown at the beginning of this tutorial, we want to
     # how much to change the pulse-width by each step
     wstep = 1500
     cur_width = min_width
-    
+
     while True:
       tchannel.pulse_width(cur_width)
-      
+
       # this determines how often we change the pulse-width. It is
       # analogous to frames-per-second
       sleep(0.01)
-    
+
       cur_width += wstep
-    
+
       if cur_width > max_width:
         cur_width = min_width
 
@@ -67,11 +67,11 @@ If we want to have a breathing effect, where the LED fades from dim to bright th
 
     while True:
       tchannel.pulse_width(cur_width)
-    
+
       sleep(0.01)
-    
+
       cur_width += wstep
-    
+
       if cur_width > max_width:
         cur_width = max_width
         wstep *= -1

docs/pyboard/tutorial/leds.rst

@@ -17,7 +17,7 @@ This is all very well but we would like this process to be automated. Open the f
         pyb.delay(1000)
 
 When you save, the red light on the pyboard should turn on for about a second. To run the script, do a soft reset (CTRL-D). The pyboard will then restart and you should see a green light continuously flashing on and off. Success, the first step on your path to building an army of evil robots! When you are bored of the annoying flashing light then press CTRL-C at your terminal to stop it running.
- 
+
 So what does this code do? First we need some terminology. Python is an object-oriented language, almost everything in python is a *class* and when you create an instance of a class you get an *object*. Classes have *methods* associated to them. A method (also called a member function) is used to interact with or control the object.
 
 The first line of code creates an LED object which we have then called led. When we create the object, it takes a single parameter which must be between 1 and 4, corresponding to the 4 LEDs on the board. The pyb.LED class has three important member functions that we will use: on(), off() and toggle(). The other function that we use is pyb.delay() this simply waits for a given time in milliseconds. Once we have created the LED object, the statement while True: creates an infinite loop which toggles the led between on and off and waits for 1 second.

docs/pyboard/tutorial/repl.rst

@@ -41,7 +41,7 @@ Mac OS X
 Open a terminal and run::
 
     screen /dev/tty.usbmodem*
-    
+
 When you are finished and want to exit screen, type CTRL-A CTRL-\\.
 
 Linux
@@ -50,7 +50,7 @@ Linux
 Open a terminal and run::
 
     screen /dev/ttyACM0
-    
+
 You can also try ``picocom`` or ``minicom`` instead of screen.  You may have to
 use ``/dev/ttyACM1`` or a higher number for ``ttyACM``.  And, you may need to give
 yourself the correct permissions to access this devices (eg group ``uucp`` or ``dialout``,

docs/reference/asm_thumb2_float.rst

@@ -31,7 +31,7 @@ Arithmetic
 * vsqrt(Sd, Sm) ``Sd = sqrt(Sm)``
 
 Registers may be identical: ``vmul(S0, S0, S0)`` will execute ``S0 = S0*S0``
- 
+
 Move between ARM core and FPU registers
 ---------------------------------------
 
@@ -40,7 +40,7 @@ Move between ARM core and FPU registers
 
 The FPU has a register known as FPSCR, similar to the ARM core's APSR, which stores condition
 codes plus other data. The following instructions provide access to this.
- 
+
 * vmrs(APSR\_nzcv, FPSCR)
 
 Move the floating-point N, Z, C, and V flags to the APSR N, Z, C, and V flags.

docs/reference/filesystem.rst

@@ -0,0 +1,290 @@
+.. _filesystem:
+
+Working with filesystems
+========================
+
+.. contents::
+
+This tutorial describes how MicroPython provides an on-device filesystem,
+allowing standard Python file I/O methods to be used with persistent storage.
+
+MicroPython automatically creates a default configuration and auto-detects the
+primary filesystem, so this tutorial will be mostly useful if you want to modify
+the partitioning, filesystem type, or use custom block devices.
+
+The filesystem is typically backed by internal flash memory on the device, but
+can also use external flash, RAM, or a custom block device.
+
+On some ports (e.g. STM32), the filesystem may also be available over USB MSC to
+a host PC. :ref:`pyboard_py` also provides a way for the host PC to access to
+the filesystem on all ports.
+
+Note: This is mainly for use on bare-metal ports like STM32 and ESP32. On ports
+with an operating system (e.g. the Unix port) the filesystem is provided by the
+host OS.
+
+VFS
+---
+
+MicroPython implements a Unix-like Virtual File System (VFS) layer. All mounted
+filesystems are combined into a single virtual filesystem, starting at the root
+``/``. Filesystems are mounted into directories in this structure, and at
+startup the working directory is changed to where the primary filesystem is
+mounted.
+
+On STM32 / Pyboard, the internal flash is mounted at ``/flash``, and optionally
+the SDCard at ``/sd``. On ESP8266/ESP32, the primary filesystem is mounted at
+``/``.
+
+Block devices
+-------------
+
+A block device is an instance of a class that implements the
+:class:`uos.AbstractBlockDev` protocol.
+
+Built-in block devices
+~~~~~~~~~~~~~~~~~~~~~~
+
+Ports provide built-in block devices to access their primary flash.
+
+On power-on, MicroPython will attempt to detect the filesystem on the default
+flash and configure and mount it automatically. If no filesystem is found,
+MicroPython will attempt to create a FAT filesystem spanning the entire flash.
+Ports can also provide a mechanism to "factory reset" the primary flash, usually
+by some combination of button presses at power on.
+
+STM32 / Pyboard
+...............
+
+The :ref:`pyb.Flash <pyb.Flash>` class provides access to the internal flash. On some
+boards which have larger external flash (e.g. Pyboard D), it will use that
+instead. The ``start`` kwarg should always be specified, i.e.
+``pyb.Flash(start=0)``.
+
+Note: For backwards compatibility, when constructed with no arguments (i.e.
+``pyb.Flash()``), it only implements the simple block interface and reflects the
+virtual device presented to USB MSC (i.e. it includes a virtual partition table
+at the start).
+
+ESP8266
+.......
+
+The internal flash is exposed as a block device object which is created in the
+``flashbdev`` module on start up. This object is by default added as a global
+variable so it can usually be accessed simply as ``bdev``. This implements the
+extended interface.
+
+ESP32
+.....
+
+The :class:`esp32.Partition` class implements a block device for partitions
+defined for the board. Like ESP8266, there is a global variable ``bdev`` which
+points to the default partition. This implements the extended interface.
+
+Custom block devices
+~~~~~~~~~~~~~~~~~~~~
+
+The following class implements a simple block device that stores its data in
+RAM using a ``bytearray``::
+
+    class RAMBlockDev:
+        def __init__(self, block_size, num_blocks):
+            self.block_size = block_size
+            self.data = bytearray(block_size * num_blocks)
+
+        def readblocks(self, block_num, buf):
+            for i in range(len(buf)):
+                buf[i] = self.data[block_num * self.block_size + i]
+
+        def writeblocks(self, block_num, buf):
+            for i in range(len(buf)):
+                self.data[block_num * self.block_size + i] = buf[i]
+
+        def ioctl(self, op, arg):
+            if op == 4: # get number of blocks
+                return len(self.data) // self.block_size
+            if op == 5: # get block size
+                return self.block_size
+
+It can be used as follows::
+
+    import os
+
+    bdev = RAMBlockDev(512, 50)
+    os.VfsFat.mkfs(bdev)
+    os.mount(bdev, '/ramdisk')
+
+An example of a block device that supports both the simple and extended
+interface (i.e. both signatures and behaviours of the
+:meth:`uos.AbstractBlockDev.readblocks` and
+:meth:`uos.AbstractBlockDev.writeblocks` methods) is::
+
+    class RAMBlockDev:
+        def __init__(self, block_size, num_blocks):
+            self.block_size = block_size
+            self.data = bytearray(block_size * num_blocks)
+
+        def readblocks(self, block_num, buf, offset=0):
+            addr = block_num * self.block_size + offset
+            for i in range(len(buf)):
+                buf[i] = self.data[addr + i]
+
+        def writeblocks(self, block_num, buf, offset=None):
+            if offset is None:
+                # do erase, then write
+                for i in range(len(buf) // self.block_size):
+                    self.ioctl(6, block_num + i)
+                offset = 0
+            addr = block_num * self.block_size + offset
+            for i in range(len(buf)):
+                self.data[addr + i] = buf[i]
+
+        def ioctl(self, op, arg):
+            if op == 4: # block count
+                return len(self.data) // self.block_size
+            if op == 5: # block size
+                return self.block_size
+            if op == 6: # block erase
+                return 0
+
+As it supports the extended interface, it can be used with :class:`littlefs
+<uos.VfsLfs2>`::
+
+    import os
+
+    bdev = RAMBlockDev(512, 50)
+    os.VfsLfs2.mkfs(bdev)
+    os.mount(bdev, '/ramdisk')
+
+Once mounted, the filesystem (regardless of its type) can be used as it
+normally would be used from Python code, for example::
+
+    with open('/ramdisk/hello.txt', 'w') as f:
+        f.write('Hello world')
+    print(open('/ramdisk/hello.txt').read())
+
+Filesystems
+-----------
+
+MicroPython ports can provide implementations of :class:`FAT <uos.VfsFat>`,
+:class:`littlefs v1 <uos.VfsLfs1>` and :class:`littlefs v2 <uos.VfsLfs2>`.
+
+The following table shows which filesystems are included in the firmware by
+default for given port/board combinations, however they can be optionally
+enabled in a custom firmware build.
+
+====================  =====  ===========  ===========
+Board                 FAT    littlefs v1  littlefs v2
+====================  =====  ===========  ===========
+pyboard 1.0, 1.1, D   Yes    No           Yes
+Other STM32           Yes    No           No
+ESP8266               Yes    No           No
+ESP32                 Yes    No           Yes
+====================  =====  ===========  ===========
+
+FAT
+~~~
+
+The main advantage of the FAT filesystem is that it can be accessed over USB MSC
+on supported boards (e.g. STM32) without any additional drivers required on the
+host PC.
+
+However, FAT is not tolerant to power failure during writes and this can lead to
+filesystem corruption. For applications that do not require USB MSC, it is
+recommended to use littlefs instead.
+
+To format the entire flash using FAT::
+
+    # ESP8266 and ESP32
+    import os
+    os.umount('/')
+    os.VfsFat.mkfs(bdev)
+    os.mount(bdev, '/')
+
+    # STM32
+    import os, pyb
+    os.umount('/flash')
+    os.VfsFat.mkfs(pyb.Flash(start=0))
+    os.mount(pyb.Flash(start=0), '/flash')
+    os.chdir('/flash')
+
+Littlefs
+~~~~~~~~
+
+Littlefs_ is a filesystem designed for flash-based devices, and is much more
+resistant to filesystem corruption.
+
+.. note:: There are reports of littlefs v1 and v2 failing in certain
+          situations, for details see `littlefs issue 347`_  and
+          `littlefs issue 295`_.
+
+Note: It can be still be accessed over USB MSC using the `littlefs FUSE
+driver`_. Note that you must use the ``-b=4096`` option to override the block
+size.
+
+.. _littlefs FUSE driver: https://github.com/ARMmbed/littlefs-fuse/tree/master/littlefs
+.. _Littlefs: https://github.com/ARMmbed/littlefs
+.. _littlefs issue 295: https://github.com/ARMmbed/littlefs/issues/295
+.. _littlefs issue 347: https://github.com/ARMmbed/littlefs/issues/347
+
+To format the entire flash using littlefs v2::
+
+    # ESP8266 and ESP32
+    import os
+    os.umount('/')
+    os.VfsLfs2.mkfs(bdev)
+    os.mount(bdev, '/')
+
+    # STM32
+    import os, pyb
+    os.umount('/flash')
+    os.VfsLfs2.mkfs(pyb.Flash(start=0))
+    os.mount(pyb.Flash(start=0), '/flash')
+    os.chdir('/flash')
+
+Hybrid (STM32)
+~~~~~~~~~~~~~~
+
+By using the ``start`` and ``len`` kwargs to :class:`pyb.Flash`, you can create
+block devices spanning a subset of the flash device.
+
+For example, to configure the first 256kiB as FAT (and available over USB MSC),
+and the remainder as littlefs::
+
+    import os, pyb
+    os.umount('/flash')
+    p1 = pyb.Flash(start=0, len=256*1024)
+    p2 = pyb.Flash(start=256*1024)
+    os.VfsFat.mkfs(p1)
+    os.VfsLfs2.mkfs(p2)
+    os.mount(p1, '/flash')
+    os.mount(p2, '/data')
+    os.chdir('/flash')
+
+This might be useful to make your Python files, configuration and other
+rarely-modified content available over USB MSC, but allowing for frequently
+changing application data to reside on littlefs with better resilience to power
+failure, etc.
+
+The partition at offset ``0`` will be mounted automatically (and the filesystem
+type automatically detected), but you can add::
+
+    import os, pyb
+    p2 = pyb.Flash(start=256*1024)
+    os.mount(p2, '/data')
+
+to ``boot.py`` to mount the data partition.
+
+Hybrid (ESP32)
+~~~~~~~~~~~~~~
+
+On ESP32, if you build custom firmware, you can modify ``partitions.csv`` to
+define an arbitrary partition layout.
+
+At boot, the partition named "vfs" will be mounted at ``/`` by default, but any
+additional partitions can be mounted in your ``boot.py`` using::
+
+    import esp32, os
+    p = esp32.Partition.find(esp32.Partition.TYPE_DATA, label='foo')
+    os.mount(p, '/foo')
+

docs/reference/index.rst

@@ -21,8 +21,11 @@ implementation and the best practices to use them.
 
    glossary.rst
    repl.rst
+   mpyfiles.rst
    isr_rules.rst
    speed_python.rst
    constrained.rst
    packages.rst
    asm_thumb2_index.rst
+   filesystem.rst
+   pyboard.py.rst

docs/reference/mpyfiles.rst

@@ -0,0 +1,178 @@
+.. _mpy_files:
+
+MicroPython .mpy files
+======================
+
+MicroPython defines the concept of an .mpy file which is a binary container
+file format that holds precompiled code, and which can be imported like a
+normal .py module.  The file ``foo.mpy`` can be imported via ``import foo``,
+as long as ``foo.mpy`` can be found in the usual way by the import machinery.
+Usually, each directory listed in ``sys.path`` is searched in order.  When
+searching a particular directory ``foo.py`` is looked for first and if that
+is not found then ``foo.mpy`` is looked for, then the search continues in the
+next directory if neither is found.  As such, ``foo.py`` will take precedence
+over ``foo.mpy``.
+
+These .mpy files can contain bytecode which is usually generated from Python
+source files (.py files) via the ``mpy-cross`` program.  For some architectures
+an .mpy file can also contain native machine code, which can be generated in
+a variety of ways, most notably from C source code.
+
+Versioning and compatibility of .mpy files
+------------------------------------------
+
+A given .mpy file may or may not be compatible with a given MicroPython system.
+Compatibility is based on the following:
+
+* Version of the .mpy file: the version of the file must match the version
+  supported by the system loading it.
+
+* Bytecode features used in the .mpy file: there are two bytecode features
+  which must match between the file and the system: unicode support and
+  inline caching of map lookups in the bytecode.
+
+* Small integer bits: the .mpy file will require a minimum number of bits in
+  a small integer and the system loading it must support at least this many
+  bits.
+
+* Qstr compression window size: the .mpy file will require a minimum window
+  size for qstr decompression and the system loading it must have a window
+  greater or equal to this size.
+
+* Native architecture: if the .mpy file contains native machine code then
+  it will specify the architecture of that machine code and the system
+  loading it must support execution of that architecture's code.
+
+If a MicroPython system supports importing .mpy files then the
+``sys.implementation.mpy`` field will exist and return an integer which
+encodes the version (lower 8 bits), features and native architecture.
+
+Trying to import an .mpy file that fails one of the first four tests will
+raise ``ValueError('incompatible .mpy file')``.  Trying to import an .mpy
+file that fails the native architecture test (if it contains native machine
+code) will raise ``ValueError('incompatible .mpy arch')``.
+
+If importing an .mpy file fails then try the following:
+
+* Determine the .mpy version and flags supported by your MicroPython system
+  by executing::
+
+    import sys
+    sys_mpy = sys.implementation.mpy
+    arch = [None, 'x86', 'x64',
+        'armv6', 'armv6m', 'armv7m', 'armv7em', 'armv7emsp', 'armv7emdp',
+        'xtensa', 'xtensawin'][sys_mpy >> 10]
+    print('mpy version:', sys_mpy & 0xff)
+    print('mpy flags:', end='')
+    if arch:
+        print(' -march=' + arch, end='')
+    if sys_mpy & 0x100:
+        print(' -mcache-lookup-bc', end='')
+    if not sys_mpy & 0x200:
+        print(' -mno-unicode', end='')
+    print()
+
+* Check the validity of the .mpy file by inspecting the first two bytes of
+  the file.  The first byte should be an uppercase 'M' and the second byte
+  will be the version number, which should match the system version from above.
+  If it doesn't match then rebuild the .mpy file.
+
+* Check if the system .mpy version matches the version emitted by ``mpy-cross``
+  that was used to build the .mpy file, found by ``mpy-cross --version``.
+  If it doesn't match then recompile ``mpy-cross`` from the Git repository
+  checked out at the tag (or hash) reported by ``mpy-cross --version``.
+
+* Make sure you are using the correct ``mpy-cross`` flags, found by the code
+  above, or by inspecting the ``MPY_CROSS_FLAGS`` Makefile variable for the
+  port that you are using.
+
+The following table shows the correspondence between MicroPython release
+and .mpy version.
+
+=================== ============
+MicroPython release .mpy version
+=================== ============
+v1.12 and up        5
+v1.11               4
+v1.9.3 - v1.10      3
+v1.9 - v1.9.2       2
+v1.5.1 - v1.8.7     0
+=================== ============
+
+For completeness, the next table shows the Git commit of the main
+MicroPython repository at which the .mpy version was changed.
+
+=================== ========================================
+.mpy version change Git commit
+=================== ========================================
+4 to 5              5716c5cf65e9b2cb46c2906f40302401bdd27517
+3 to 4              9a5f92ea72754c01cc03e5efcdfe94021120531e
+2 to 3              ff93fd4f50321c6190e1659b19e64fef3045a484
+1 to 2              dd11af209d226b7d18d5148b239662e30ed60bad
+0 to 1              6a11048af1d01c78bdacddadd1b72dc7ba7c6478
+initial version 0   d8c834c95d506db979ec871417de90b7951edc30
+=================== ========================================
+
+Binary encoding of .mpy files
+-----------------------------
+
+MicroPython .mpy files are a binary container format with code objects
+stored internally in a nested hierarchy.  To keep files small while still
+providing a large range of possible values it uses the concept of a
+variably-encoded-unsigned-integer (vuint) in many places.  Similar to utf-8
+encoding, this encoding stores 7 bits per byte with the 8th bit (MSB) set
+if one or more bytes follow.  The bits of the unsigned integer are stored
+in the vuint in LSB form.
+
+The top-level of an .mpy file consists of two parts:
+
+* The header.
+
+* The raw-code for the outer scope of the module.
+  This outer scope is executed when the .mpy file is imported.
+
+The header
+~~~~~~~~~~
+
+The .mpy header is:
+
+======  ================================
+size    field
+======  ================================
+byte    value 0x4d (ASCII 'M')
+byte    .mpy version number
+byte    feature flags
+byte    number of bits in a small int
+vuint   size of qstr window
+======  ================================
+
+Raw code elements
+~~~~~~~~~~~~~~~~~
+
+A raw-code element contains code, either bytecode or native machine code.  Its
+contents are:
+
+======  ================================
+size    field
+======  ================================
+vuint   type and size
+...     code (bytecode or machine code)
+vuint   number of constant objects
+vuint   number of sub-raw-code elements
+...     constant objects
+...     sub-raw-code elements
+======  ================================
+
+The first vuint in a raw-code element encodes the type of code stored in this
+element (the two least-significant bits), and the decompressed length of the code
+(the amount of RAM to allocate for it).
+
+Following the vuint comes the code itself.  In the case of bytecode it also contains
+compressed qstr values.
+
+Following the code comes a vuint counting the number of constant objects, and
+another vuint counting the number of sub-raw-code elements.
+
+The constant objects are then stored next.
+
+Finally any sub-raw-code elements are stored, recursively.

docs/reference/pyboard.py.rst

@@ -0,0 +1,133 @@
+.. _pyboard_py:
+
+The pyboard.py tool
+===================
+
+This is a standalone Python tool that runs on your PC that provides a way to:
+
+* Quickly run a Python script or command on a MicroPython device. This is useful
+  while developing MicroPython programs to quickly test code without needing to
+  copy files to/from the device.
+
+* Access the filesystem on a device. This allows you to deploy your code to the
+  device (even if the board doesn't support USB MSC).
+
+Despite the name, ``pyboard.py`` works on all MicroPython ports that support the
+raw REPL (including STM32, ESP32, ESP8266, NRF).
+
+You can download the latest version from `GitHub
+<https://github.com/micropython/micropython/blob/master/tools/pyboard.py>`_. The
+only dependency is the ``pyserial`` library which can be installed from PiPy or
+your system package manager.
+
+Running ``pyboard.py --help`` gives the following output:
+
+.. code-block:: text
+
+    usage: pyboard [-h] [--device DEVICE] [-b BAUDRATE] [-u USER]
+                   [-p PASSWORD] [-c COMMAND] [-w WAIT] [--follow] [-f]
+                   [files [files ...]]
+
+    Run scripts on the pyboard.
+
+    positional arguments:
+      files                 input files
+
+    optional arguments:
+      -h, --help            show this help message and exit
+      --device DEVICE       the serial device or the IP address of the
+                            pyboard
+      -b BAUDRATE, --baudrate BAUDRATE
+                            the baud rate of the serial device
+      -u USER, --user USER  the telnet login username
+      -p PASSWORD, --password PASSWORD
+                            the telnet login password
+      -c COMMAND, --command COMMAND
+                            program passed in as string
+      -w WAIT, --wait WAIT  seconds to wait for USB connected board to become
+                            available
+      --follow              follow the output after running the scripts
+                            [default if no scripts given]
+      -f, --filesystem      perform a filesystem action
+
+Running a command on the device
+-------------------------------
+
+This is useful for testing short snippets of code, or to script an interaction
+with the device.::
+
+    $ pyboard.py --device /dev/ttyACM0 -c 'print(1+1)'
+    2
+
+Running a script on the device
+------------------------------
+
+If you have a script, ``app.py`` that you want to run on a device, then use::
+
+    $ pyboard.py --device /dev/ttyACM0 app.py
+
+Note that this doesn't actually copy app.py to the device's filesystem, it just
+loads the code into RAM and executes it. Any output generated by the program
+will be displayed.
+
+If the program app.py does not finish then you'll need to stop ``pyboard.py``,
+eg with Ctrl-C. The program ``app.py`` will still continue to run on the
+MicroPython device.
+
+Filesystem access
+-----------------
+
+Using the ``-f`` flag, the following filesystem operations are supported:
+
+* ``cp src [src...] dest`` Copy files to/from the device.
+* ``cat path`` Print the contents of a file on the device.
+* ``ls [path]`` List contents of a directory (defaults to current working directory).
+* ``rm path`` Remove a file.
+* ``mkdir path`` Create a directory.
+* ``rmdir path`` Remove a directory.
+
+The ``cp`` command uses a ``ssh``-like convention for referring to local and
+remote files. Any path starting with a ``:`` will be interpreted as on the
+device, otherwise it will be local. So::
+
+    $ pyboard.py --device /dev/ttyACM0 -f cp main.py :main.py
+
+will copy main.py from the current directory on the PC to a file named main.py
+on the device. The filename can be omitted, e.g.::
+
+    $ pyboard.py --device /dev/ttyACM0 -f cp main.py :
+
+is equivalent to the above.
+
+Some more examples::
+
+    # Copy main.py from the device to the local PC.
+    $ pyboard.py --device /dev/ttyACM0 -f cp :main.py main.py
+    # Same, but using . instead.
+    $ pyboard.py --device /dev/ttyACM0 -f cp :main.py .
+
+    # Copy three files to the device, keeping their names
+    # and paths (note: `lib` must exist on the device)
+    $ pyboard.py --device /dev/ttyACM0 -f cp main.py app.py lib/foo.py :
+
+    # Remove a file from the device.
+    $ pyboard.py --device /dev/ttyACM0 -f rm util.py
+
+    # Print the contents of a file on the device.
+    $ pyboard.py --device /dev/ttyACM0 -f cat boot.py
+    ...contents of boot.py...
+
+Using the pyboard library
+-------------------------
+
+You can also use ``pyboard.py`` as a library for scripting interactions with a
+MicroPython board.
+
+.. code-block:: python
+
+    import pyboard
+    pyb = pyboard.Pyboard('/dev/ttyACM0', 115200)
+    pyb.enter_raw_repl()
+    ret = pyb.exec('print(1+1)')
+    print(ret)
+    pyb.exit_raw_repl()

docs/wipy/quickref.rst

@@ -62,7 +62,7 @@ Timer ``id``'s take values from 0 to 3.::
     tim = Timer(0, mode=Timer.PERIODIC)
     tim_a = tim.channel(Timer.A, freq=1000)
     tim_a.freq(5) # 5 Hz
-    
+
     p_out = Pin('GP2', mode=Pin.OUT)
     tim_a.irq(trigger=Timer.TIMEOUT, handler=lambda t: p_out.toggle())
 
@@ -75,7 +75,7 @@ See :ref:`machine.Pin <machine.Pin>` and :ref:`machine.Timer <machine.Timer>`. :
 
     # timer 1 in PWM mode and width must be 16 buts
     tim = Timer(1, mode=Timer.PWM, width=16)
-    
+
     # enable channel A @1KHz with a 50.55% duty cycle
     tim_a = tim.channel(Timer.A, freq=1000, duty_cycle=5055)
 

docs/wipy/tutorial/repl.rst

@@ -51,7 +51,7 @@ Open a terminal and run::
 or::
 
     $ screen /dev/tty.usbmodem* 115200
-    
+
 When you are finished and want to exit ``screen``, type CTRL-A CTRL-\\. If your keyboard does not have a \\-key (i.e. you need an obscure combination for \\ like ALT-SHIFT-7) you can remap the ``quit`` command:
 
 - create ``~/.screenrc``

drivers/wiznet5k/ethernet/w5200/w5200.c

@@ -52,10 +52,19 @@
 
 #include "w5200.h"
 
+#if WIZCHIP_USE_MAX_BUFFER
+// This option is intended to be used when MACRAW mode is enabled, to allow
+// the single raw socket to use all the available buffer space.
+#define SMASK (16 * 1024 - 1) /* tx buffer mask */
+#define RMASK (16 * 1024 - 1) /* rx buffer mask */
+#define SSIZE (16 * 1024) /* max tx buffer size */
+#define RSIZE (16 * 1024) /* max rx buffer size */
+#else
 #define SMASK (0x7ff) /* tx buffer mask */
 #define RMASK (0x7ff) /* rx buffer mask */
 #define SSIZE (2048) /* max tx buffer size */
 #define RSIZE (2048) /* max rx buffer size */
+#endif
 
 #define TXBUF_BASE (0x8000)
 #define RXBUF_BASE (0xc000)

examples/natmod/btree/Makefile

@@ -0,0 +1,37 @@
+# Location of top-level MicroPython directory
+MPY_DIR = ../../..
+
+# Name of module (different to built-in btree so it can coexist)
+MOD = btree_$(ARCH)
+
+# Source files (.c or .py)
+SRC = btree_c.c btree_py.py
+
+# Architecture to build for (x86, x64, armv7m, xtensa, xtensawin)
+ARCH = x64
+
+BTREE_DIR = $(MPY_DIR)/lib/berkeley-db-1.xx
+BTREE_DEFS = -D__DBINTERFACE_PRIVATE=1 -Dmpool_error="(void)" -Dabort=abort_ "-Dvirt_fd_t=void*" $(BTREE_DEFS_EXTRA)
+CFLAGS += -I$(BTREE_DIR)/PORT/include
+CFLAGS += -Wno-old-style-definition -Wno-sign-compare -Wno-unused-parameter $(BTREE_DEFS)
+
+SRC += $(addprefix $(realpath $(BTREE_DIR))/,\
+	btree/bt_close.c \
+	btree/bt_conv.c \
+	btree/bt_delete.c \
+	btree/bt_get.c \
+	btree/bt_open.c \
+	btree/bt_overflow.c \
+	btree/bt_page.c \
+	btree/bt_put.c \
+	btree/bt_search.c \
+	btree/bt_seq.c \
+	btree/bt_split.c \
+	btree/bt_utils.c \
+	mpool/mpool.c \
+	)
+
+include $(MPY_DIR)/py/dynruntime.mk
+
+# btree needs gnu99 defined
+CFLAGS += -std=gnu99

examples/natmod/btree/btree_c.c

@@ -0,0 +1,148 @@
+#define MICROPY_ENABLE_DYNRUNTIME (1)
+#define MICROPY_PY_BTREE (1)
+
+#include "py/dynruntime.h"
+
+#include <unistd.h>
+
+#if !defined(__linux__)
+void *memcpy(void *dst, const void *src, size_t n) {
+    return mp_fun_table.memmove_(dst, src, n);
+}
+void *memset(void *s, int c, size_t n) {
+    return mp_fun_table.memset_(s, c, n);
+}
+#endif
+
+void *memmove(void *dest, const void *src, size_t n) {
+    return mp_fun_table.memmove_(dest, src, n);
+}
+
+void *malloc(size_t n) {
+    void *ptr = m_malloc(n);
+    return ptr;
+}
+void *realloc(void *ptr, size_t n) {
+    mp_printf(&mp_plat_print, "UNDEF %d\n", __LINE__);
+    return NULL;
+}
+void *calloc(size_t n, size_t m) {
+    void *ptr = m_malloc(n * m);
+    // memory already cleared by conservative GC
+    return ptr;
+}
+
+void free(void *ptr) {
+    m_free(ptr);
+}
+
+void abort_(void) {
+    nlr_raise(mp_obj_new_exception(mp_load_global(MP_QSTR_RuntimeError)));
+}
+
+int native_errno;
+#if defined(__linux__)
+int *__errno_location (void)
+#else
+int *__errno (void)
+#endif
+{
+    return &native_errno;
+}
+
+ssize_t mp_stream_posix_write(void *stream, const void *buf, size_t len) {
+    mp_obj_base_t* o = stream;
+    const mp_stream_p_t *stream_p = o->type->protocol;
+    mp_uint_t out_sz = stream_p->write(MP_OBJ_FROM_PTR(stream), buf, len, &native_errno);
+    if (out_sz == MP_STREAM_ERROR) {
+        return -1;
+    } else {
+        return out_sz;
+    }
+}
+
+ssize_t mp_stream_posix_read(void *stream, void *buf, size_t len) {
+    mp_obj_base_t* o = stream;
+    const mp_stream_p_t *stream_p = o->type->protocol;
+    mp_uint_t out_sz = stream_p->read(MP_OBJ_FROM_PTR(stream), buf, len, &native_errno);
+    if (out_sz == MP_STREAM_ERROR) {
+        return -1;
+    } else {
+        return out_sz;
+    }
+}
+
+off_t mp_stream_posix_lseek(void *stream, off_t offset, int whence) {
+    const mp_obj_base_t* o = stream;
+    const mp_stream_p_t *stream_p = o->type->protocol;
+    struct mp_stream_seek_t seek_s;
+    seek_s.offset = offset;
+    seek_s.whence = whence;
+    mp_uint_t res = stream_p->ioctl(MP_OBJ_FROM_PTR(stream), MP_STREAM_SEEK, (mp_uint_t)(uintptr_t)&seek_s, &native_errno);
+    if (res == MP_STREAM_ERROR) {
+        return -1;
+    }
+    return seek_s.offset;
+}
+
+int mp_stream_posix_fsync(void *stream) {
+    mp_obj_base_t* o = stream;
+    const mp_stream_p_t *stream_p = o->type->protocol;
+    mp_uint_t res = stream_p->ioctl(MP_OBJ_FROM_PTR(stream), MP_STREAM_FLUSH, 0, &native_errno);
+    if (res == MP_STREAM_ERROR) {
+        return -1;
+    }
+    return res;
+}
+
+mp_obj_type_t btree_type;
+
+#include "extmod/modbtree.c"
+
+mp_map_elem_t btree_locals_dict_table[8];
+STATIC MP_DEFINE_CONST_DICT(btree_locals_dict, btree_locals_dict_table);
+
+STATIC mp_obj_t btree_open(size_t n_args, const mp_obj_t *args) {
+    // Make sure we got a stream object
+    mp_get_stream_raise(args[0], MP_STREAM_OP_READ | MP_STREAM_OP_WRITE | MP_STREAM_OP_IOCTL);
+
+    BTREEINFO openinfo = {0};
+    openinfo.flags = mp_obj_get_int(args[1]);
+    openinfo.cachesize = mp_obj_get_int(args[2]);
+    openinfo.psize = mp_obj_get_int(args[3]);
+    openinfo.minkeypage = mp_obj_get_int(args[4]);
+    DB *db = __bt_open(MP_OBJ_TO_PTR(args[0]), &btree_stream_fvtable, &openinfo, 0);
+    if (db == NULL) {
+        mp_raise_OSError(native_errno);
+    }
+
+    return MP_OBJ_FROM_PTR(btree_new(db));
+}
+STATIC MP_DEFINE_CONST_FUN_OBJ_VAR_BETWEEN(btree_open_obj, 5, 5, btree_open);
+
+mp_obj_t mpy_init(mp_obj_fun_bc_t *self, size_t n_args, size_t n_kw, mp_obj_t *args) {
+    MP_DYNRUNTIME_INIT_ENTRY
+
+    btree_type.base.type = (void*)&mp_fun_table.type_type;
+    btree_type.name = MP_QSTR_btree;
+    btree_type.print = btree_print;
+    btree_type.getiter = btree_getiter;
+    btree_type.iternext = btree_iternext;
+    btree_type.binary_op = btree_binary_op;
+    btree_type.subscr = btree_subscr;
+    btree_locals_dict_table[0] = (mp_map_elem_t){ MP_OBJ_NEW_QSTR(MP_QSTR_close), MP_OBJ_FROM_PTR(&btree_close_obj) };
+    btree_locals_dict_table[1] = (mp_map_elem_t){ MP_OBJ_NEW_QSTR(MP_QSTR_flush), MP_OBJ_FROM_PTR(&btree_flush_obj) };
+    btree_locals_dict_table[2] = (mp_map_elem_t){ MP_OBJ_NEW_QSTR(MP_QSTR_get), MP_OBJ_FROM_PTR(&btree_get_obj) };
+    btree_locals_dict_table[3] = (mp_map_elem_t){ MP_OBJ_NEW_QSTR(MP_QSTR_put), MP_OBJ_FROM_PTR(&btree_put_obj) };
+    btree_locals_dict_table[4] = (mp_map_elem_t){ MP_OBJ_NEW_QSTR(MP_QSTR_seq), MP_OBJ_FROM_PTR(&btree_seq_obj) };
+    btree_locals_dict_table[5] = (mp_map_elem_t){ MP_OBJ_NEW_QSTR(MP_QSTR_keys), MP_OBJ_FROM_PTR(&btree_keys_obj) };
+    btree_locals_dict_table[6] = (mp_map_elem_t){ MP_OBJ_NEW_QSTR(MP_QSTR_values), MP_OBJ_FROM_PTR(&btree_values_obj) };
+    btree_locals_dict_table[7] = (mp_map_elem_t){ MP_OBJ_NEW_QSTR(MP_QSTR_items), MP_OBJ_FROM_PTR(&btree_items_obj) };
+    btree_type.locals_dict = (void*)&btree_locals_dict;
+
+    mp_store_global(MP_QSTR__open, MP_OBJ_FROM_PTR(&btree_open_obj));
+    mp_store_global(MP_QSTR_INCL, MP_OBJ_NEW_SMALL_INT(FLAG_END_KEY_INCL));
+    mp_store_global(MP_QSTR_DESC, MP_OBJ_NEW_SMALL_INT(FLAG_DESC));
+
+    MP_DYNRUNTIME_INIT_EXIT
+}

examples/natmod/btree/btree_py.py

@@ -0,0 +1,3 @@
+# Implemented in Python to support keyword arguments
+def open(stream, *, flags=0, cachesize=0, pagesize=0, minkeypage=0):
+    return _open(stream, flags, cachesize, pagesize, minkeypage)

examples/natmod/features0/Makefile

@@ -0,0 +1,14 @@
+# Location of top-level MicroPython directory
+MPY_DIR = ../../..
+
+# Name of module
+MOD = features0
+
+# Source files (.c or .py)
+SRC = features0.c
+
+# Architecture to build for (x86, x64, armv7m, xtensa, xtensawin)
+ARCH = x64
+
+# Include to get the rules for compiling and linking the module
+include $(MPY_DIR)/py/dynruntime.mk

examples/natmod/features0/features0.c

@@ -0,0 +1,40 @@
+/* This example demonstrates the following features in a native module:
+    - defining a simple function exposed to Python
+    - defining a local, helper C function
+    - getting and creating integer objects
+*/
+
+// Include the header file to get access to the MicroPython API
+#include "py/dynruntime.h"
+
+// Helper function to compute factorial
+STATIC mp_int_t factorial_helper(mp_int_t x) {
+    if (x == 0) {
+        return 1;
+    }
+    return x * factorial_helper(x - 1);
+}
+
+// This is the function which will be called from Python, as factorial(x)
+STATIC mp_obj_t factorial(mp_obj_t x_obj) {
+    // Extract the integer from the MicroPython input object
+    mp_int_t x = mp_obj_get_int(x_obj);
+    // Calculate the factorial
+    mp_int_t result = factorial_helper(x);
+    // Convert the result to a MicroPython integer object and return it
+    return mp_obj_new_int(result);
+}
+// Define a Python reference to the function above
+STATIC MP_DEFINE_CONST_FUN_OBJ_1(factorial_obj, factorial);
+
+// This is the entry point and is called when the module is imported
+mp_obj_t mpy_init(mp_obj_fun_bc_t *self, size_t n_args, size_t n_kw, mp_obj_t *args) {
+    // This must be first, it sets up the globals dict and other things
+    MP_DYNRUNTIME_INIT_ENTRY
+
+    // Make the function available in the module's namespace
+    mp_store_global(MP_QSTR_factorial, MP_OBJ_FROM_PTR(&factorial_obj));
+
+    // This must be last, it restores the globals dict
+    MP_DYNRUNTIME_INIT_EXIT
+}

examples/natmod/features1/Makefile

@@ -0,0 +1,14 @@
+# Location of top-level MicroPython directory
+MPY_DIR = ../../..
+
+# Name of module
+MOD = features1
+
+# Source files (.c or .py)
+SRC = features1.c
+
+# Architecture to build for (x86, x64, armv7m, xtensa, xtensawin)
+ARCH = x64
+
+# Include to get the rules for compiling and linking the module
+include $(MPY_DIR)/py/dynruntime.mk

examples/natmod/features1/features1.c

@@ -0,0 +1,106 @@
+/* This example demonstrates the following features in a native module:
+    - defining simple functions exposed to Python
+    - defining local, helper C functions
+    - defining constant integers and strings exposed to Python
+    - getting and creating integer objects
+    - creating Python lists
+    - raising exceptions
+    - allocating memory
+    - BSS and constant data (rodata)
+    - relocated pointers in rodata
+*/
+
+// Include the header file to get access to the MicroPython API
+#include "py/dynruntime.h"
+
+// BSS (zero) data
+uint16_t data16[4];
+
+// Constant data (rodata)
+const uint8_t table8[] = { 0, 1, 1, 2, 3, 5, 8, 13 };
+const uint16_t table16[] = { 0x1000, 0x2000 };
+
+// Constant data pointing to BSS/constant data
+uint16_t *const table_ptr16a[] = { &data16[0], &data16[1], &data16[2], &data16[3] };
+const uint16_t *const table_ptr16b[] = { &table16[0], &table16[1] };
+
+// A simple function that adds its 2 arguments (must be integers)
+STATIC mp_obj_t add(mp_obj_t x_in, mp_obj_t y_in) {
+    mp_int_t x = mp_obj_get_int(x_in);
+    mp_int_t y = mp_obj_get_int(y_in);
+    return mp_obj_new_int(x + y);
+}
+STATIC MP_DEFINE_CONST_FUN_OBJ_2(add_obj, add);
+
+// A local helper function (not exposed to Python)
+STATIC mp_int_t fibonacci_helper(mp_int_t x) {
+    if (x < MP_ARRAY_SIZE(table8)) {
+        return table8[x];
+    } else {
+        return fibonacci_helper(x - 1) + fibonacci_helper(x - 2);
+    }
+}
+
+// A function which computes Fibonacci numbers
+STATIC mp_obj_t fibonacci(mp_obj_t x_in) {
+    mp_int_t x = mp_obj_get_int(x_in);
+    if (x < 0) {
+        mp_raise_ValueError("can't compute negative Fibonacci number");
+    }
+    return mp_obj_new_int(fibonacci_helper(x));
+}
+STATIC MP_DEFINE_CONST_FUN_OBJ_1(fibonacci_obj, fibonacci);
+
+// A function that accesses the BSS data
+STATIC mp_obj_t access(size_t n_args, const mp_obj_t *args) {
+    if (n_args == 0) {
+        // Create a list holding all items from data16
+        mp_obj_list_t *lst = MP_OBJ_TO_PTR(mp_obj_new_list(MP_ARRAY_SIZE(data16), NULL));
+        for (int i = 0; i < MP_ARRAY_SIZE(data16); ++i) {
+            lst->items[i] = mp_obj_new_int(data16[i]);
+        }
+        return MP_OBJ_FROM_PTR(lst);
+    } else if (n_args == 1) {
+        // Get one item from data16
+        mp_int_t idx = mp_obj_get_int(args[0]) & 3;
+        return mp_obj_new_int(data16[idx]);
+    } else {
+        // Set one item in data16 (via table_ptr16a)
+        mp_int_t idx = mp_obj_get_int(args[0]) & 3;
+        *table_ptr16a[idx] = mp_obj_get_int(args[1]);
+        return mp_const_none;
+    }
+}
+STATIC MP_DEFINE_CONST_FUN_OBJ_VAR_BETWEEN(access_obj, 0, 2, access);
+
+// A function that allocates memory and creates a bytearray
+STATIC mp_obj_t make_array(void) {
+    uint16_t *ptr = m_new(uint16_t, MP_ARRAY_SIZE(table_ptr16b));
+    for (int i = 0; i < MP_ARRAY_SIZE(table_ptr16b); ++i) {
+        ptr[i] = *table_ptr16b[i];
+    }
+    return mp_obj_new_bytearray_by_ref(sizeof(uint16_t) * MP_ARRAY_SIZE(table_ptr16b), ptr);
+}
+STATIC MP_DEFINE_CONST_FUN_OBJ_0(make_array_obj, make_array);
+
+// This is the entry point and is called when the module is imported
+mp_obj_t mpy_init(mp_obj_fun_bc_t *self, size_t n_args, size_t n_kw, mp_obj_t *args) {
+    // This must be first, it sets up the globals dict and other things
+    MP_DYNRUNTIME_INIT_ENTRY
+
+    // Messages can be printed as usualy
+    mp_printf(&mp_plat_print, "initialising module self=%p\n", self);
+
+    // Make the functions available in the module's namespace
+    mp_store_global(MP_QSTR_add, MP_OBJ_FROM_PTR(&add_obj));
+    mp_store_global(MP_QSTR_fibonacci, MP_OBJ_FROM_PTR(&fibonacci_obj));
+    mp_store_global(MP_QSTR_access, MP_OBJ_FROM_PTR(&access_obj));
+    mp_store_global(MP_QSTR_make_array, MP_OBJ_FROM_PTR(&make_array_obj));
+
+    // Add some constants to the module's namespace
+    mp_store_global(MP_QSTR_VAL, MP_OBJ_NEW_SMALL_INT(42));
+    mp_store_global(MP_QSTR_MSG, MP_OBJ_NEW_QSTR(MP_QSTR_HELLO_MICROPYTHON));
+
+    // This must be last, it restores the globals dict
+    MP_DYNRUNTIME_INIT_EXIT
+}

examples/natmod/features2/Makefile

@@ -0,0 +1,14 @@
+# Location of top-level MicroPython directory
+MPY_DIR = ../../..
+
+# Name of module
+MOD = features2
+
+# Source files (.c or .py)
+SRC = main.c prod.c test.py
+
+# Architecture to build for (x86, x64, armv7m, xtensa, xtensawin)
+ARCH = x64
+
+# Include to get the rules for compiling and linking the module
+include $(MPY_DIR)/py/dynruntime.mk

examples/natmod/features2/main.c

@@ -0,0 +1,83 @@
+/* This example demonstrates the following features in a native module:
+    - using floats
+    - defining additional code in Python (see test.py)
+    - have extra C code in a separate file (see prod.c)
+*/
+
+// Include the header file to get access to the MicroPython API
+#include "py/dynruntime.h"
+
+// Include the header for auxiliary C code for this module
+#include "prod.h"
+
+// Automatically detect if this module should include double-precision code.
+// If double precision is supported by the target architecture then it can
+// be used in native module regardless of what float setting the target
+// MicroPython runtime uses (being none, float or double).
+#if defined(__i386__) || defined(__x86_64__) || (defined(__ARM_FP) && (__ARM_FP & 8))
+#define USE_DOUBLE 1
+#else
+#define USE_DOUBLE 0
+#endif
+
+// A function that uses the default float type configured for the current target
+// This default can be overridden by specifying MICROPY_FLOAT_IMPL at the make level
+STATIC mp_obj_t add(mp_obj_t x, mp_obj_t y) {
+    return mp_obj_new_float(mp_obj_get_float(x) + mp_obj_get_float(y));
+}
+STATIC MP_DEFINE_CONST_FUN_OBJ_2(add_obj, add);
+
+// A function that explicitly uses single precision floats
+STATIC mp_obj_t add_f(mp_obj_t x, mp_obj_t y) {
+    return mp_obj_new_float_from_f(mp_obj_get_float_to_f(x) + mp_obj_get_float_to_f(y));
+}
+STATIC MP_DEFINE_CONST_FUN_OBJ_2(add_f_obj, add_f);
+
+#if USE_DOUBLE
+// A function that explicitly uses double precision floats
+STATIC mp_obj_t add_d(mp_obj_t x, mp_obj_t y) {
+    return mp_obj_new_float_from_d(mp_obj_get_float_to_d(x) + mp_obj_get_float_to_d(y));
+}
+STATIC MP_DEFINE_CONST_FUN_OBJ_2(add_d_obj, add_d);
+#endif
+
+// A function that computes the product of floats in an array.
+// This function uses the most general C argument interface, which is more difficult
+// to use but has access to the globals dict of the module via self->globals.
+STATIC mp_obj_t productf(mp_obj_fun_bc_t *self, size_t n_args, size_t n_kw, mp_obj_t *args) {
+    // Check number of arguments is valid
+    mp_arg_check_num(n_args, n_kw, 1, 1, false);
+
+    // Extract buffer pointer and verify typecode
+    mp_buffer_info_t bufinfo;
+    mp_get_buffer_raise(args[0], &bufinfo, MP_BUFFER_RW);
+    if (bufinfo.typecode != 'f') {
+        mp_raise_ValueError("expecting float array");
+    }
+
+    // Compute product, store result back in first element of array
+    float *ptr = bufinfo.buf;
+    float prod = prod_array(bufinfo.len / sizeof(*ptr), ptr);
+    ptr[0] = prod;
+
+    return mp_const_none;
+}
+
+// This is the entry point and is called when the module is imported
+mp_obj_t mpy_init(mp_obj_fun_bc_t *self, size_t n_args, size_t n_kw, mp_obj_t *args) {
+    // This must be first, it sets up the globals dict and other things
+    MP_DYNRUNTIME_INIT_ENTRY
+
+    // Make the functions available in the module's namespace
+    mp_store_global(MP_QSTR_add, MP_OBJ_FROM_PTR(&add_obj));
+    mp_store_global(MP_QSTR_add_f, MP_OBJ_FROM_PTR(&add_f_obj));
+    #if USE_DOUBLE
+    mp_store_global(MP_QSTR_add_d, MP_OBJ_FROM_PTR(&add_d_obj));
+    #endif
+
+    // The productf function uses the most general C argument interface
+    mp_store_global(MP_QSTR_productf, MP_DYNRUNTIME_MAKE_FUNCTION(productf));
+
+    // This must be last, it restores the globals dict
+    MP_DYNRUNTIME_INIT_EXIT
+}

examples/natmod/features2/prod.c

@@ -0,0 +1,9 @@
+#include "prod.h"
+
+float prod_array(int n, float *ar) {
+    float ans = 1;
+    for (int i = 0; i < n; ++i) {
+        ans *= ar[i];
+    }
+    return ans;
+}

examples/natmod/features2/prod.h

@@ -0,0 +1 @@
+float prod_array(int n, float *ar);

examples/natmod/features2/test.py

@@ -0,0 +1,26 @@
+# This Python code will be merged with the C code in main.c
+
+import array
+
+def isclose(a, b):
+    return abs(a - b) < 1e-3
+
+def test():
+    tests = [
+        isclose(add(0.1, 0.2), 0.3),
+        isclose(add_f(0.1, 0.2), 0.3),
+    ]
+
+    ar = array.array('f', [1, 2, 3.5])
+    productf(ar)
+    tests.append(isclose(ar[0], 7))
+
+    if 'add_d' in globals():
+        tests.append(isclose(add_d(0.1, 0.2), 0.3))
+
+    print(tests)
+
+    if not all(tests):
+        raise SystemExit(1)
+
+test()

examples/natmod/framebuf/Makefile

@@ -0,0 +1,13 @@
+# Location of top-level MicroPython directory
+MPY_DIR = ../../..
+
+# Name of module (different to built-in framebuf so it can coexist)
+MOD = framebuf_$(ARCH)
+
+# Source files (.c or .py)
+SRC = framebuf.c
+
+# Architecture to build for (x86, x64, armv7m, xtensa, xtensawin)
+ARCH = x64
+
+include $(MPY_DIR)/py/dynruntime.mk

examples/natmod/framebuf/framebuf.c

@@ -0,0 +1,50 @@
+#define MICROPY_ENABLE_DYNRUNTIME (1)
+#define MICROPY_PY_FRAMEBUF (1)
+
+#include "py/dynruntime.h"
+
+#if !defined(__linux__)
+void *memset(void *s, int c, size_t n) {
+    return mp_fun_table.memset_(s, c, n);
+}
+#endif
+
+mp_obj_type_t mp_type_framebuf;
+
+#include "extmod/modframebuf.c"
+
+mp_map_elem_t framebuf_locals_dict_table[10];
+STATIC MP_DEFINE_CONST_DICT(framebuf_locals_dict, framebuf_locals_dict_table);
+
+mp_obj_t mpy_init(mp_obj_fun_bc_t *self, size_t n_args, size_t n_kw, mp_obj_t *args) {
+    MP_DYNRUNTIME_INIT_ENTRY
+
+    mp_type_framebuf.base.type = (void*)&mp_type_type;
+    mp_type_framebuf.name = MP_QSTR_FrameBuffer;
+    mp_type_framebuf.make_new = framebuf_make_new;
+    mp_type_framebuf.buffer_p.get_buffer = framebuf_get_buffer;
+    framebuf_locals_dict_table[0] = (mp_map_elem_t){ MP_OBJ_NEW_QSTR(MP_QSTR_fill), MP_OBJ_FROM_PTR(&framebuf_fill_obj) };
+    framebuf_locals_dict_table[1] = (mp_map_elem_t){ MP_OBJ_NEW_QSTR(MP_QSTR_fill_rect), MP_OBJ_FROM_PTR(&framebuf_fill_rect_obj) };
+    framebuf_locals_dict_table[2] = (mp_map_elem_t){ MP_OBJ_NEW_QSTR(MP_QSTR_pixel), MP_OBJ_FROM_PTR(&framebuf_pixel_obj) };
+    framebuf_locals_dict_table[3] = (mp_map_elem_t){ MP_OBJ_NEW_QSTR(MP_QSTR_hline), MP_OBJ_FROM_PTR(&framebuf_hline_obj) };
+    framebuf_locals_dict_table[4] = (mp_map_elem_t){ MP_OBJ_NEW_QSTR(MP_QSTR_vline), MP_OBJ_FROM_PTR(&framebuf_vline_obj) };
+    framebuf_locals_dict_table[5] = (mp_map_elem_t){ MP_OBJ_NEW_QSTR(MP_QSTR_rect), MP_OBJ_FROM_PTR(&framebuf_rect_obj) };
+    framebuf_locals_dict_table[6] = (mp_map_elem_t){ MP_OBJ_NEW_QSTR(MP_QSTR_line), MP_OBJ_FROM_PTR(&framebuf_line_obj) };
+    framebuf_locals_dict_table[7] = (mp_map_elem_t){ MP_OBJ_NEW_QSTR(MP_QSTR_blit), MP_OBJ_FROM_PTR(&framebuf_blit_obj) };
+    framebuf_locals_dict_table[8] = (mp_map_elem_t){ MP_OBJ_NEW_QSTR(MP_QSTR_scroll), MP_OBJ_FROM_PTR(&framebuf_scroll_obj) };
+    framebuf_locals_dict_table[9] = (mp_map_elem_t){ MP_OBJ_NEW_QSTR(MP_QSTR_text), MP_OBJ_FROM_PTR(&framebuf_text_obj) };
+    mp_type_framebuf.locals_dict = (void*)&framebuf_locals_dict;
+
+    mp_store_global(MP_QSTR_FrameBuffer, MP_OBJ_FROM_PTR(&mp_type_framebuf));
+    mp_store_global(MP_QSTR_FrameBuffer1, MP_OBJ_FROM_PTR(&legacy_framebuffer1_obj));
+    mp_store_global(MP_QSTR_MVLSB, MP_OBJ_NEW_SMALL_INT(FRAMEBUF_MVLSB));
+    mp_store_global(MP_QSTR_MONO_VLSB, MP_OBJ_NEW_SMALL_INT(FRAMEBUF_MVLSB));
+    mp_store_global(MP_QSTR_RGB565, MP_OBJ_NEW_SMALL_INT(FRAMEBUF_RGB565));
+    mp_store_global(MP_QSTR_GS2_HMSB, MP_OBJ_NEW_SMALL_INT(FRAMEBUF_GS2_HMSB));
+    mp_store_global(MP_QSTR_GS4_HMSB, MP_OBJ_NEW_SMALL_INT(FRAMEBUF_GS4_HMSB));
+    mp_store_global(MP_QSTR_GS8, MP_OBJ_NEW_SMALL_INT(FRAMEBUF_GS8));
+    mp_store_global(MP_QSTR_MONO_HLSB, MP_OBJ_NEW_SMALL_INT(FRAMEBUF_MHLSB));
+    mp_store_global(MP_QSTR_MONO_HMSB, MP_OBJ_NEW_SMALL_INT(FRAMEBUF_MHMSB));
+
+    MP_DYNRUNTIME_INIT_EXIT
+}

examples/natmod/uheapq/Makefile

@@ -0,0 +1,13 @@
+# Location of top-level MicroPython directory
+MPY_DIR = ../../..
+
+# Name of module (different to built-in uheapq so it can coexist)
+MOD = uheapq_$(ARCH)
+
+# Source files (.c or .py)
+SRC = uheapq.c
+
+# Architecture to build for (x86, x64, armv7m, xtensa, xtensawin)
+ARCH = x64
+
+include $(MPY_DIR)/py/dynruntime.mk

examples/natmod/uheapq/uheapq.c

@@ -0,0 +1,17 @@
+#define MICROPY_ENABLE_DYNRUNTIME (1)
+#define MICROPY_PY_UHEAPQ (1)
+
+#include "py/dynruntime.h"
+
+#include "extmod/moduheapq.c"
+
+mp_obj_t mpy_init(mp_obj_fun_bc_t *self, size_t n_args, size_t n_kw, mp_obj_t *args) {
+    MP_DYNRUNTIME_INIT_ENTRY
+
+    mp_store_global(MP_QSTR___name__, MP_OBJ_NEW_QSTR(MP_QSTR_uheapq));
+    mp_store_global(MP_QSTR_heappush, MP_OBJ_FROM_PTR(&mod_uheapq_heappush_obj));
+    mp_store_global(MP_QSTR_heappop, MP_OBJ_FROM_PTR(&mod_uheapq_heappop_obj));
+    mp_store_global(MP_QSTR_heapify, MP_OBJ_FROM_PTR(&mod_uheapq_heapify_obj));
+
+    MP_DYNRUNTIME_INIT_EXIT
+}

examples/natmod/urandom/Makefile

@@ -0,0 +1,13 @@
+# Location of top-level MicroPython directory
+MPY_DIR = ../../..
+
+# Name of module (different to built-in urandom so it can coexist)
+MOD = urandom_$(ARCH)
+
+# Source files (.c or .py)
+SRC = urandom.c
+
+# Architecture to build for (x86, x64, armv7m, xtensa, xtensawin)
+ARCH = x64
+
+include $(MPY_DIR)/py/dynruntime.mk

examples/natmod/urandom/urandom.c

@@ -0,0 +1,34 @@
+#define MICROPY_ENABLE_DYNRUNTIME (1)
+#define MICROPY_PY_URANDOM (1)
+#define MICROPY_PY_URANDOM_EXTRA_FUNCS (1)
+
+#include "py/dynruntime.h"
+
+// Dynamic native modules don't support a data section so these must go in the BSS
+uint32_t yasmarang_pad, yasmarang_n, yasmarang_d;
+uint8_t yasmarang_dat;
+
+#include "extmod/modurandom.c"
+
+mp_obj_t mpy_init(mp_obj_fun_bc_t *self, size_t n_args, size_t n_kw, mp_obj_t *args) {
+    MP_DYNRUNTIME_INIT_ENTRY
+
+    yasmarang_pad = 0xeda4baba;
+    yasmarang_n = 69;
+    yasmarang_d = 233;
+
+    mp_store_global(MP_QSTR___name__, MP_OBJ_NEW_QSTR(MP_QSTR_urandom));
+    mp_store_global(MP_QSTR_getrandbits, MP_OBJ_FROM_PTR(&mod_urandom_getrandbits_obj));
+    mp_store_global(MP_QSTR_seed, MP_OBJ_FROM_PTR(&mod_urandom_seed_obj));
+    #if MICROPY_PY_URANDOM_EXTRA_FUNCS
+    mp_store_global(MP_QSTR_randrange, MP_OBJ_FROM_PTR(&mod_urandom_randrange_obj));
+    mp_store_global(MP_QSTR_randint, MP_OBJ_FROM_PTR(&mod_urandom_randint_obj));
+    mp_store_global(MP_QSTR_choice, MP_OBJ_FROM_PTR(&mod_urandom_choice_obj));
+    #if MICROPY_PY_BUILTINS_FLOAT
+    mp_store_global(MP_QSTR_random, MP_OBJ_FROM_PTR(&mod_urandom_random_obj));
+    mp_store_global(MP_QSTR_uniform, MP_OBJ_FROM_PTR(&mod_urandom_uniform_obj));
+    #endif
+    #endif
+
+    MP_DYNRUNTIME_INIT_EXIT
+}

examples/natmod/ure/Makefile

@@ -0,0 +1,13 @@
+# Location of top-level MicroPython directory
+MPY_DIR = ../../..
+
+# Name of module (different to built-in ure so it can coexist)
+MOD = ure_$(ARCH)
+
+# Source files (.c or .py)
+SRC = ure.c
+
+# Architecture to build for (x86, x64, armv7m, xtensa, xtensawin)
+ARCH = x64
+
+include $(MPY_DIR)/py/dynruntime.mk

examples/natmod/ure/ure.c

@@ -0,0 +1,79 @@
+#define MICROPY_ENABLE_DYNRUNTIME (1)
+#define MICROPY_STACK_CHECK (1)
+#define MICROPY_PY_URE (1)
+#define MICROPY_PY_URE_MATCH_GROUPS (1)
+#define MICROPY_PY_URE_MATCH_SPAN_START_END (1)
+#define MICROPY_PY_URE_SUB (0) // requires vstr interface
+
+#include <alloca.h>
+#include "py/dynruntime.h"
+
+#define STACK_LIMIT (2048)
+
+const char *stack_top;
+
+void mp_stack_check(void) {
+    // Assumes descending stack on target
+    volatile char dummy;
+    if (stack_top - &dummy >= STACK_LIMIT) {
+        mp_raise_msg(&mp_type_RuntimeError, "maximum recursion depth exceeded");
+    }
+}
+
+#if !defined(__linux__)
+void *memcpy(void *dst, const void *src, size_t n) {
+    return mp_fun_table.memmove_(dst, src, n);
+}
+void *memset(void *s, int c, size_t n) {
+    return mp_fun_table.memset_(s, c, n);
+}
+#endif
+
+void *memmove(void *dest, const void *src, size_t n) {
+    return mp_fun_table.memmove_(dest, src, n);
+}
+
+mp_obj_type_t match_type;
+mp_obj_type_t re_type;
+
+#include "extmod/modure.c"
+
+mp_map_elem_t match_locals_dict_table[5];
+STATIC MP_DEFINE_CONST_DICT(match_locals_dict, match_locals_dict_table);
+
+mp_map_elem_t re_locals_dict_table[3];
+STATIC MP_DEFINE_CONST_DICT(re_locals_dict, re_locals_dict_table);
+
+mp_obj_t mpy_init(mp_obj_fun_bc_t *self, size_t n_args, size_t n_kw, mp_obj_t *args) {
+    MP_DYNRUNTIME_INIT_ENTRY
+
+    char dummy;
+    stack_top = &dummy;
+
+    // Because MP_QSTR_start/end/split are static, xtensa and xtensawin will make a small data section
+    // to copy in this key/value pair if they are specified as a struct, so assign them separately.
+
+    match_type.base.type = (void*)&mp_fun_table.type_type;
+    match_type.name = MP_QSTR_match;
+    match_type.print = match_print;
+    match_locals_dict_table[0] = (mp_map_elem_t){ MP_OBJ_NEW_QSTR(MP_QSTR_group), MP_OBJ_FROM_PTR(&match_group_obj) };
+    match_locals_dict_table[1] = (mp_map_elem_t){ MP_OBJ_NEW_QSTR(MP_QSTR_groups), MP_OBJ_FROM_PTR(&match_groups_obj) };
+    match_locals_dict_table[2] = (mp_map_elem_t){ MP_OBJ_NEW_QSTR(MP_QSTR_span), MP_OBJ_FROM_PTR(&match_span_obj) };
+    match_locals_dict_table[3] = (mp_map_elem_t){ MP_OBJ_NEW_QSTR(MP_QSTR_start), MP_OBJ_FROM_PTR(&match_start_obj) };
+    match_locals_dict_table[4] = (mp_map_elem_t){ MP_OBJ_NEW_QSTR(MP_QSTR_end), MP_OBJ_FROM_PTR(&match_end_obj) };
+    match_type.locals_dict = (void*)&match_locals_dict;
+
+    re_type.base.type = (void*)&mp_fun_table.type_type;
+    re_type.name = MP_QSTR_ure;
+    re_type.print = re_print;
+    re_locals_dict_table[0] = (mp_map_elem_t){ MP_OBJ_NEW_QSTR(MP_QSTR_match), MP_OBJ_FROM_PTR(&re_match_obj) };
+    re_locals_dict_table[1] = (mp_map_elem_t){ MP_OBJ_NEW_QSTR(MP_QSTR_search), MP_OBJ_FROM_PTR(&re_search_obj) };
+    re_locals_dict_table[2] = (mp_map_elem_t){ MP_OBJ_NEW_QSTR(MP_QSTR_split), MP_OBJ_FROM_PTR(&re_split_obj) };
+    re_type.locals_dict = (void*)&re_locals_dict;
+
+    mp_store_global(MP_QSTR_compile, MP_OBJ_FROM_PTR(&mod_re_compile_obj));
+    mp_store_global(MP_QSTR_match, MP_OBJ_FROM_PTR(&mod_re_match_obj));
+    mp_store_global(MP_QSTR_search, MP_OBJ_FROM_PTR(&mod_re_search_obj));
+
+    MP_DYNRUNTIME_INIT_EXIT
+}

examples/natmod/uzlib/Makefile

@@ -0,0 +1,13 @@
+# Location of top-level MicroPython directory
+MPY_DIR = ../../..
+
+# Name of module (different to built-in uzlib so it can coexist)
+MOD = uzlib_$(ARCH)
+
+# Source files (.c or .py)
+SRC = uzlib.c
+
+# Architecture to build for (x86, x64, armv7m, xtensa, xtensawin)
+ARCH = x64
+
+include $(MPY_DIR)/py/dynruntime.mk

examples/natmod/uzlib/uzlib.c

@@ -0,0 +1,36 @@
+#define MICROPY_ENABLE_DYNRUNTIME (1)
+#define MICROPY_PY_UZLIB (1)
+
+#include "py/dynruntime.h"
+
+#if !defined(__linux__)
+void *memset(void *s, int c, size_t n) {
+    return mp_fun_table.memset_(s, c, n);
+}
+#endif
+
+mp_obj_type_t decompio_type;
+
+#include "extmod/moduzlib.c"
+
+mp_map_elem_t decompio_locals_dict_table[3];
+STATIC MP_DEFINE_CONST_DICT(decompio_locals_dict, decompio_locals_dict_table);
+
+mp_obj_t mpy_init(mp_obj_fun_bc_t *self, size_t n_args, size_t n_kw, mp_obj_t *args) {
+    MP_DYNRUNTIME_INIT_ENTRY
+
+    decompio_type.base.type = mp_fun_table.type_type;
+    decompio_type.name = MP_QSTR_DecompIO;
+    decompio_type.make_new = decompio_make_new;
+    decompio_type.protocol = &decompio_stream_p;
+    decompio_locals_dict_table[0] = (mp_map_elem_t){ MP_OBJ_NEW_QSTR(MP_QSTR_read), MP_OBJ_FROM_PTR(&mp_stream_read_obj) };
+    decompio_locals_dict_table[1] = (mp_map_elem_t){ MP_OBJ_NEW_QSTR(MP_QSTR_readinto), MP_OBJ_FROM_PTR(&mp_stream_readinto_obj) };
+    decompio_locals_dict_table[2] = (mp_map_elem_t){ MP_OBJ_NEW_QSTR(MP_QSTR_readline), MP_OBJ_FROM_PTR(&mp_stream_unbuffered_readline_obj) };
+    decompio_type.locals_dict = (void*)&decompio_locals_dict;
+
+    mp_store_global(MP_QSTR___name__, MP_OBJ_NEW_QSTR(MP_QSTR_uzlib));
+    mp_store_global(MP_QSTR_decompress, MP_OBJ_FROM_PTR(&mod_uzlib_decompress_obj));
+    mp_store_global(MP_QSTR_DecompIO, MP_OBJ_FROM_PTR(&decompio_type));
+
+    MP_DYNRUNTIME_INIT_EXIT
+}

examples/network/http_server_ssl.py

@@ -1,3 +1,4 @@
+import ubinascii as binascii
 try:
     import usocket as socket
 except:
@@ -5,6 +6,35 @@ except:
 import ussl as ssl
 
 
+# This self-signed key/cert pair is randomly generated and to be used for
+# testing/demonstration only.  You should always generate your own key/cert.
+key = binascii.unhexlify(
+    b'3082013b020100024100cc20643fd3d9c21a0acba4f48f61aadd675f52175a9dcf07fbef'
+    b'610a6a6ba14abb891745cd18a1d4c056580d8ff1a639460f867013c8391cdc9f2e573b0f'
+    b'872d0203010001024100bb17a54aeb3dd7ae4edec05e775ca9632cf02d29c2a089b563b0'
+    b'd05cdf95aeca507de674553f28b4eadaca82d5549a86058f9996b07768686a5b02cb240d'
+    b'd9f1022100f4a63f5549e817547dca97b5c658038e8593cb78c5aba3c4642cc4cd031d86'
+    b'8f022100d598d870ffe4a34df8de57047a50b97b71f4d23e323f527837c9edae88c79483'
+    b'02210098560c89a70385c36eb07fd7083235c4c1184e525d838aedf7128958bedfdbb102'
+    b'2051c0dab7057a8176ca966f3feb81123d4974a733df0f958525f547dfd1c271f9022044'
+    b'6c2cafad455a671a8cf398e642e1be3b18a3d3aec2e67a9478f83c964c4f1f')
+cert = binascii.unhexlify(
+    b'308201d53082017f020203e8300d06092a864886f70d01010505003075310b3009060355'
+    b'0406130258583114301206035504080c0b54686550726f76696e63653110300e06035504'
+    b'070c075468654369747931133011060355040a0c0a436f6d70616e7958595a3113301106'
+    b'0355040b0c0a436f6d70616e7958595a3114301206035504030c0b546865486f73744e61'
+    b'6d65301e170d3139313231383033333935355a170d3239313231353033333935355a3075'
+    b'310b30090603550406130258583114301206035504080c0b54686550726f76696e636531'
+    b'10300e06035504070c075468654369747931133011060355040a0c0a436f6d70616e7958'
+    b'595a31133011060355040b0c0a436f6d70616e7958595a3114301206035504030c0b5468'
+    b'65486f73744e616d65305c300d06092a864886f70d0101010500034b003048024100cc20'
+    b'643fd3d9c21a0acba4f48f61aadd675f52175a9dcf07fbef610a6a6ba14abb891745cd18'
+    b'a1d4c056580d8ff1a639460f867013c8391cdc9f2e573b0f872d0203010001300d06092a'
+    b'864886f70d0101050500034100b0513fe2829e9ecbe55b6dd14c0ede7502bde5d46153c8'
+    b'e960ae3ebc247371b525caeb41bbcf34686015a44c50d226e66aef0a97a63874ca5944ef'
+    b'979b57f0b3')
+
+
 CONTENT = b"""\
 HTTP/1.0 200 OK
 
@@ -31,7 +61,8 @@ def main(use_stream=True):
         client_addr = res[1]
         print("Client address:", client_addr)
         print("Client socket:", client_s)
-        client_s = ssl.wrap_socket(client_s, server_side=True)
+        # CPython uses key keyfile/certfile arguments, but MicroPython uses key/cert
+        client_s = ssl.wrap_socket(client_s, server_side=True, key=key, cert=cert)
         print(client_s)
         print("Request:")
         if use_stream:

extmod/modbluetooth.c

@@ -33,6 +33,7 @@
 #include "py/objarray.h"
 #include "py/qstr.h"
 #include "py/runtime.h"
+#include "py/mphal.h"
 #include "extmod/modbluetooth.h"
 #include <string.h>
 
@@ -42,16 +43,13 @@
 #error modbluetooth requires MICROPY_ENABLE_SCHEDULER
 #endif
 
-// This is used to protect the ringbuffer.
-#ifndef MICROPY_PY_BLUETOOTH_ENTER
-#define MICROPY_PY_BLUETOOTH_ENTER mp_uint_t atomic_state = MICROPY_BEGIN_ATOMIC_SECTION();
-#define MICROPY_PY_BLUETOOTH_EXIT MICROPY_END_ATOMIC_SECTION(atomic_state);
-#endif
-
 #define MP_BLUETOOTH_CONNECT_DEFAULT_SCAN_DURATION_MS 2000
 
 #define MICROPY_PY_BLUETOOTH_MAX_EVENT_DATA_TUPLE_LEN 5
-#define MICROPY_PY_BLUETOOTH_MAX_EVENT_DATA_BYTES_LEN (MICROPY_PY_BLUETOOTH_RINGBUF_SIZE / 2)
+
+// This formula is intended to allow queuing the data of a large characteristic
+// while still leaving room for a couple of normal (small, fixed size) events.
+#define MICROPY_PY_BLUETOOTH_MAX_EVENT_DATA_BYTES_LEN(ringbuf_size) (MAX((int)((ringbuf_size) / 2), (int)(ringbuf_size) - 64))
 
 STATIC const mp_obj_type_t bluetooth_ble_type;
 STATIC const mp_obj_type_t bluetooth_uuid_type;
@@ -60,9 +58,11 @@ typedef struct {
     mp_obj_base_t base;
     mp_obj_t irq_handler;
     uint16_t irq_trigger;
+    bool irq_scheduled;
     mp_obj_t irq_data_tuple;
     uint8_t irq_data_addr_bytes[6];
-    uint8_t irq_data_data_bytes[MICROPY_PY_BLUETOOTH_MAX_EVENT_DATA_BYTES_LEN];
+    uint16_t irq_data_data_alloc;
+    uint8_t *irq_data_data_bytes;
     mp_obj_str_t irq_data_addr;
     mp_obj_str_t irq_data_data;
     mp_obj_bluetooth_uuid_t irq_data_uuid;
@@ -248,11 +248,12 @@ STATIC mp_obj_t bluetooth_ble_make_new(const mp_obj_type_t *type, size_t n_args,
         // Pre-allocated buffers for address, payload and uuid.
         o->irq_data_addr.base.type = &mp_type_bytes;
         o->irq_data_addr.data = o->irq_data_addr_bytes;
+        o->irq_data_data_alloc = MICROPY_PY_BLUETOOTH_MAX_EVENT_DATA_BYTES_LEN(MICROPY_PY_BLUETOOTH_RINGBUF_SIZE);
         o->irq_data_data.base.type = &mp_type_bytes;
-        o->irq_data_data.data = o->irq_data_data_bytes;
+        o->irq_data_data.data = m_new(uint8_t, o->irq_data_data_alloc);
         o->irq_data_uuid.base.type = &bluetooth_uuid_type;
 
-        // Allocate the ringbuf. TODO: Consider making the size user-specified.
+        // Allocate the default ringbuf.
         ringbuf_alloc(&o->ringbuf, MICROPY_PY_BLUETOOTH_RINGBUF_SIZE);
 
         MP_STATE_VM(bluetooth) = MP_OBJ_FROM_PTR(o);
@@ -277,16 +278,77 @@ STATIC mp_obj_t bluetooth_ble_active(size_t n_args, const mp_obj_t *args) {
 }
 STATIC MP_DEFINE_CONST_FUN_OBJ_VAR_BETWEEN(bluetooth_ble_active_obj, 1, 2, bluetooth_ble_active);
 
-STATIC mp_obj_t bluetooth_ble_config(mp_obj_t self_in, mp_obj_t param) {
-    if (param == MP_OBJ_NEW_QSTR(MP_QSTR_mac)) {
-        uint8_t addr[6];
-        mp_bluetooth_get_device_addr(addr);
-        return mp_obj_new_bytes(addr, MP_ARRAY_SIZE(addr));
+STATIC mp_obj_t bluetooth_ble_config(size_t n_args, const mp_obj_t *args, mp_map_t *kwargs) {
+    mp_obj_bluetooth_ble_t *self = MP_OBJ_TO_PTR(args[0]);
+
+    if (kwargs->used == 0) {
+        // Get config value
+        if (n_args != 2) {
+            mp_raise_TypeError("must query one param");
+        }
+
+        switch (mp_obj_str_get_qstr(args[1])) {
+            case MP_QSTR_mac: {
+                uint8_t addr[6];
+                mp_bluetooth_get_device_addr(addr);
+                return mp_obj_new_bytes(addr, MP_ARRAY_SIZE(addr));
+            }
+            default:
+                mp_raise_ValueError("unknown config param");
+        }
     } else {
-        mp_raise_ValueError("unknown config param");
+        // Set config value(s)
+        if (n_args != 1) {
+            mp_raise_TypeError("can't specify pos and kw args");
+        }
+
+        for (size_t i = 0; i < kwargs->alloc; ++i) {
+            if (MP_MAP_SLOT_IS_FILLED(kwargs, i)) {
+                mp_map_elem_t *e = &kwargs->table[i];
+                switch (mp_obj_str_get_qstr(e->key)) {
+                    case MP_QSTR_rxbuf: {
+                        // Determine new buffer sizes
+                        mp_int_t ringbuf_alloc = mp_obj_get_int(e->value);
+                        if (ringbuf_alloc < 16 || ringbuf_alloc > 0xffff) {
+                            mp_raise_ValueError(NULL);
+                        }
+                        size_t irq_data_alloc = MICROPY_PY_BLUETOOTH_MAX_EVENT_DATA_BYTES_LEN(ringbuf_alloc);
+
+                        // Allocate new buffers
+                        uint8_t *ringbuf = m_new(uint8_t, ringbuf_alloc);
+                        uint8_t *irq_data = m_new(uint8_t, irq_data_alloc);
+
+                        // Get old buffer sizes and pointers
+                        uint8_t *old_ringbuf_buf = self->ringbuf.buf;
+                        size_t old_ringbuf_alloc = self->ringbuf.size;
+                        uint8_t *old_irq_data_buf = (uint8_t*)self->irq_data_data.data;
+                        size_t old_irq_data_alloc = self->irq_data_data_alloc;
+
+                        // Atomically update the ringbuf and irq data
+                        MICROPY_PY_BLUETOOTH_ENTER
+                        self->ringbuf.size = ringbuf_alloc;
+                        self->ringbuf.buf = ringbuf;
+                        self->ringbuf.iget = 0;
+                        self->ringbuf.iput = 0;
+                        self->irq_data_data_alloc = irq_data_alloc;
+                        self->irq_data_data.data = irq_data;
+                        MICROPY_PY_BLUETOOTH_EXIT
+
+                        // Free old buffers
+                        m_del(uint8_t, old_ringbuf_buf, old_ringbuf_alloc);
+                        m_del(uint8_t, old_irq_data_buf, old_irq_data_alloc);
+                        break;
+                    }
+                    default:
+                        mp_raise_ValueError("unknown config param");
+                }
+            }
+        }
+
+        return mp_const_none;
     }
 }
-STATIC MP_DEFINE_CONST_FUN_OBJ_2(bluetooth_ble_config_obj, bluetooth_ble_config);
+STATIC MP_DEFINE_CONST_FUN_OBJ_KW(bluetooth_ble_config_obj, 1, bluetooth_ble_config);
 
 STATIC mp_obj_t bluetooth_ble_irq(size_t n_args, const mp_obj_t *pos_args, mp_map_t *kw_args) {
     enum { ARG_handler, ARG_trigger };
@@ -645,9 +707,13 @@ STATIC mp_obj_t bluetooth_ble_gattc_write(size_t n_args, const mp_obj_t *args) {
     mp_buffer_info_t bufinfo = {0};
     mp_get_buffer_raise(data, &bufinfo, MP_BUFFER_READ);
     size_t len = bufinfo.len;
-    return bluetooth_handle_errno(mp_bluetooth_gattc_write(conn_handle, value_handle, bufinfo.buf, &len));
+    unsigned int mode = MP_BLUETOOTH_WRITE_MODE_NO_RESPONSE;
+    if (n_args == 5) {
+        mode = mp_obj_get_int(args[4]);
+    }
+    return bluetooth_handle_errno(mp_bluetooth_gattc_write(conn_handle, value_handle, bufinfo.buf, &len, mode));
 }
-STATIC MP_DEFINE_CONST_FUN_OBJ_VAR_BETWEEN(bluetooth_ble_gattc_write_obj, 4, 4, bluetooth_ble_gattc_write);
+STATIC MP_DEFINE_CONST_FUN_OBJ_VAR_BETWEEN(bluetooth_ble_gattc_write_obj, 4, 5, bluetooth_ble_gattc_write);
 
 #endif // MICROPY_PY_BLUETOOTH_ENABLE_CENTRAL_MODE
 
@@ -741,8 +807,8 @@ STATIC void ringbuf_extract(ringbuf_t* ringbuf, mp_obj_tuple_t *data_tuple, size
         data_tuple->items[j++] = MP_OBJ_FROM_PTR(uuid);
     }
     // The code that enqueues into the ringbuf should ensure that it doesn't
-    // put more than MICROPY_PY_BLUETOOTH_MAX_EVENT_DATA_BYTES_LEN bytes into
-    // the ringbuf.
+    // put more than bt->irq_data_data_alloc bytes into the ringbuf, because
+    // that's what's available here in bt->irq_data_bytes.
     if (bytes_data) {
         bytes_data->len = ringbuf_get(ringbuf);
         for (int i = 0; i < bytes_data->len; ++i) {
@@ -757,9 +823,12 @@ STATIC void ringbuf_extract(ringbuf_t* ringbuf, mp_obj_tuple_t *data_tuple, size
 
 STATIC mp_obj_t bluetooth_ble_invoke_irq(mp_obj_t none_in) {
     // This is always executing in schedule context.
+
+    mp_obj_bluetooth_ble_t *o = MP_OBJ_TO_PTR(MP_STATE_VM(bluetooth));
+    o->irq_scheduled = false;
+
     for (;;) {
         MICROPY_PY_BLUETOOTH_ENTER
-        mp_obj_bluetooth_ble_t *o = MP_OBJ_TO_PTR(MP_STATE_VM(bluetooth));
 
         mp_int_t event = event = ringbuf_get16(&o->ringbuf);
         if (event < 0) {
@@ -822,9 +891,7 @@ STATIC MP_DEFINE_CONST_FUN_OBJ_1(bluetooth_ble_invoke_irq_obj, bluetooth_ble_inv
 // Callbacks are called in interrupt context (i.e. can't allocate), so we need to push the data
 // into the ringbuf and schedule the callback via mp_sched_schedule.
 
-STATIC bool enqueue_irq(mp_obj_bluetooth_ble_t *o, size_t len, uint16_t event, bool *sched) {
-    *sched = false;
-
+STATIC bool enqueue_irq(mp_obj_bluetooth_ble_t *o, size_t len, uint16_t event) {
     if (!o || !(o->irq_trigger & event) || o->irq_handler == mp_const_none) {
         return false;
     }
@@ -849,11 +916,6 @@ STATIC bool enqueue_irq(mp_obj_bluetooth_ble_t *o, size_t len, uint16_t event, b
         for (int i = 0; i < n; ++i) {
             ringbuf_get(&o->ringbuf);
         }
-
-        // No need to schedule the handler, as the ringbuffer was non-empty.
-    } else {
-        // Schedule the handler only if this is the first thing in the ringbuffer.
-        *sched = ringbuf_avail(&o->ringbuf) == 0;
     }
 
     // Append this event, the caller will then append the arguments.
@@ -861,8 +923,10 @@ STATIC bool enqueue_irq(mp_obj_bluetooth_ble_t *o, size_t len, uint16_t event, b
     return true;
 }
 
-STATIC void schedule_ringbuf(bool sched) {
-    if (sched) {
+STATIC void schedule_ringbuf(void) {
+    mp_obj_bluetooth_ble_t *o = MP_OBJ_TO_PTR(MP_STATE_VM(bluetooth));
+    if (!o->irq_scheduled) {
+        o->irq_scheduled = true;
         mp_sched_schedule(MP_OBJ_FROM_PTR(MP_ROM_PTR(&bluetooth_ble_invoke_irq_obj)), mp_const_none);
     }
 }
@@ -870,47 +934,43 @@ STATIC void schedule_ringbuf(bool sched) {
 void mp_bluetooth_gap_on_connected_disconnected(uint16_t event, uint16_t conn_handle, uint8_t addr_type, const uint8_t *addr) {
     MICROPY_PY_BLUETOOTH_ENTER
     mp_obj_bluetooth_ble_t *o = MP_OBJ_TO_PTR(MP_STATE_VM(bluetooth));
-    bool sched;
-    if (enqueue_irq(o, 2 + 1 + 6, event, &sched)) {
+    if (enqueue_irq(o, 2 + 1 + 6, event)) {
         ringbuf_put16(&o->ringbuf, conn_handle);
         ringbuf_put(&o->ringbuf, addr_type);
         for (int i = 0; i < 6; ++i) {
             ringbuf_put(&o->ringbuf, addr[i]);
         }
     }
+    schedule_ringbuf();
     MICROPY_PY_BLUETOOTH_EXIT
-    schedule_ringbuf(sched);
 }
 
 void mp_bluetooth_gatts_on_write(uint16_t conn_handle, uint16_t value_handle) {
     MICROPY_PY_BLUETOOTH_ENTER
     mp_obj_bluetooth_ble_t *o = MP_OBJ_TO_PTR(MP_STATE_VM(bluetooth));
-    bool sched;
-    if (enqueue_irq(o, 2 + 2, MP_BLUETOOTH_IRQ_GATTS_WRITE, &sched)) {
+    if (enqueue_irq(o, 2 + 2, MP_BLUETOOTH_IRQ_GATTS_WRITE)) {
         ringbuf_put16(&o->ringbuf, conn_handle);
         ringbuf_put16(&o->ringbuf, value_handle);
     }
+    schedule_ringbuf();
     MICROPY_PY_BLUETOOTH_EXIT
-    schedule_ringbuf(sched);
 }
 
 #if MICROPY_PY_BLUETOOTH_ENABLE_CENTRAL_MODE
 void mp_bluetooth_gap_on_scan_complete(void) {
     MICROPY_PY_BLUETOOTH_ENTER
     mp_obj_bluetooth_ble_t *o = MP_OBJ_TO_PTR(MP_STATE_VM(bluetooth));
-    bool sched;
-    if (enqueue_irq(o, 0, MP_BLUETOOTH_IRQ_SCAN_COMPLETE, &sched)) {
+    if (enqueue_irq(o, 0, MP_BLUETOOTH_IRQ_SCAN_COMPLETE)) {
     }
+    schedule_ringbuf();
     MICROPY_PY_BLUETOOTH_EXIT
-    schedule_ringbuf(sched);
 }
 
 void mp_bluetooth_gap_on_scan_result(uint8_t addr_type, const uint8_t *addr, bool connectable, const int8_t rssi, const uint8_t *data, size_t data_len) {
     MICROPY_PY_BLUETOOTH_ENTER
     mp_obj_bluetooth_ble_t *o = MP_OBJ_TO_PTR(MP_STATE_VM(bluetooth));
-    bool sched;
-    data_len = MIN(MICROPY_PY_BLUETOOTH_MAX_EVENT_DATA_BYTES_LEN, data_len);
-    if (enqueue_irq(o, 1 + 6 + 1 + 1 + 1 + data_len, MP_BLUETOOTH_IRQ_SCAN_RESULT, &sched)) {
+    data_len = MIN(o->irq_data_data_alloc, data_len);
+    if (enqueue_irq(o, 1 + 6 + 1 + 1 + 1 + data_len, MP_BLUETOOTH_IRQ_SCAN_RESULT)) {
         ringbuf_put(&o->ringbuf, addr_type);
         for (int i = 0; i < 6; ++i) {
             ringbuf_put(&o->ringbuf, addr[i]);
@@ -923,80 +983,83 @@ void mp_bluetooth_gap_on_scan_result(uint8_t addr_type, const uint8_t *addr, boo
             ringbuf_put(&o->ringbuf, data[i]);
         }
     }
+    schedule_ringbuf();
     MICROPY_PY_BLUETOOTH_EXIT
-    schedule_ringbuf(sched);
 }
 
 void mp_bluetooth_gattc_on_primary_service_result(uint16_t conn_handle, uint16_t start_handle, uint16_t end_handle, mp_obj_bluetooth_uuid_t *service_uuid) {
     MICROPY_PY_BLUETOOTH_ENTER
     mp_obj_bluetooth_ble_t *o = MP_OBJ_TO_PTR(MP_STATE_VM(bluetooth));
-    bool sched;
-    if (enqueue_irq(o, 2 + 2 + 2 + 1 + service_uuid->type, MP_BLUETOOTH_IRQ_GATTC_SERVICE_RESULT, &sched)) {
+    if (enqueue_irq(o, 2 + 2 + 2 + 1 + service_uuid->type, MP_BLUETOOTH_IRQ_GATTC_SERVICE_RESULT)) {
         ringbuf_put16(&o->ringbuf, conn_handle);
         ringbuf_put16(&o->ringbuf, start_handle);
         ringbuf_put16(&o->ringbuf, end_handle);
         ringbuf_put_uuid(&o->ringbuf, service_uuid);
     }
+    schedule_ringbuf();
     MICROPY_PY_BLUETOOTH_EXIT
-    schedule_ringbuf(sched);
 }
 
 void mp_bluetooth_gattc_on_characteristic_result(uint16_t conn_handle, uint16_t def_handle, uint16_t value_handle, uint8_t properties, mp_obj_bluetooth_uuid_t *characteristic_uuid) {
     MICROPY_PY_BLUETOOTH_ENTER
     mp_obj_bluetooth_ble_t *o = MP_OBJ_TO_PTR(MP_STATE_VM(bluetooth));
-    bool sched;
-    if (enqueue_irq(o, 2 + 2 + 2 + 1 + characteristic_uuid->type, MP_BLUETOOTH_IRQ_GATTC_CHARACTERISTIC_RESULT, &sched)) {
+    if (enqueue_irq(o, 2 + 2 + 2 + 1 + characteristic_uuid->type, MP_BLUETOOTH_IRQ_GATTC_CHARACTERISTIC_RESULT)) {
         ringbuf_put16(&o->ringbuf, conn_handle);
         ringbuf_put16(&o->ringbuf, def_handle);
         ringbuf_put16(&o->ringbuf, value_handle);
         ringbuf_put(&o->ringbuf, properties);
         ringbuf_put_uuid(&o->ringbuf, characteristic_uuid);
     }
+    schedule_ringbuf();
     MICROPY_PY_BLUETOOTH_EXIT
-    schedule_ringbuf(sched);
 }
 
 void mp_bluetooth_gattc_on_descriptor_result(uint16_t conn_handle, uint16_t handle, mp_obj_bluetooth_uuid_t *descriptor_uuid) {
     MICROPY_PY_BLUETOOTH_ENTER
     mp_obj_bluetooth_ble_t *o = MP_OBJ_TO_PTR(MP_STATE_VM(bluetooth));
-    bool sched;
-    if (enqueue_irq(o, 2 + 2 + 1 + descriptor_uuid->type, MP_BLUETOOTH_IRQ_GATTC_DESCRIPTOR_RESULT, &sched)) {
+    if (enqueue_irq(o, 2 + 2 + 1 + descriptor_uuid->type, MP_BLUETOOTH_IRQ_GATTC_DESCRIPTOR_RESULT)) {
         ringbuf_put16(&o->ringbuf, conn_handle);
         ringbuf_put16(&o->ringbuf, handle);
         ringbuf_put_uuid(&o->ringbuf, descriptor_uuid);
     }
+    schedule_ringbuf();
     MICROPY_PY_BLUETOOTH_EXIT
-    schedule_ringbuf(sched);
 }
 
-void mp_bluetooth_gattc_on_data_available(uint16_t event, uint16_t conn_handle, uint16_t value_handle, const uint8_t *data, size_t data_len) {
-    MICROPY_PY_BLUETOOTH_ENTER
+size_t mp_bluetooth_gattc_on_data_available_start(uint16_t event, uint16_t conn_handle, uint16_t value_handle, size_t data_len) {
     mp_obj_bluetooth_ble_t *o = MP_OBJ_TO_PTR(MP_STATE_VM(bluetooth));
-    bool sched;
-    data_len = MIN(MICROPY_PY_BLUETOOTH_MAX_EVENT_DATA_BYTES_LEN, data_len);
-    if (enqueue_irq(o, 2 + 2 + 1 + data_len, event, &sched)) {
+    data_len = MIN(o->irq_data_data_alloc, data_len);
+    if (enqueue_irq(o, 2 + 2 + 1 + data_len, event)) {
         ringbuf_put16(&o->ringbuf, conn_handle);
         ringbuf_put16(&o->ringbuf, value_handle);
         ringbuf_put(&o->ringbuf, data_len);
-        for (int i = 0; i < data_len; ++i) {
-            ringbuf_put(&o->ringbuf, data[i]);
-        }
+        return data_len;
+    } else {
+        return 0;
     }
-    MICROPY_PY_BLUETOOTH_EXIT
-    schedule_ringbuf(sched);
+}
+
+void mp_bluetooth_gattc_on_data_available_chunk(const uint8_t *data, size_t data_len) {
+    mp_obj_bluetooth_ble_t *o = MP_OBJ_TO_PTR(MP_STATE_VM(bluetooth));
+    for (int i = 0; i < data_len; ++i) {
+        ringbuf_put(&o->ringbuf, data[i]);
+    }
+}
+
+void mp_bluetooth_gattc_on_data_available_end(void) {
+    schedule_ringbuf();
 }
 
 void mp_bluetooth_gattc_on_write_status(uint16_t conn_handle, uint16_t value_handle, uint16_t status) {
     MICROPY_PY_BLUETOOTH_ENTER
     mp_obj_bluetooth_ble_t *o = MP_OBJ_TO_PTR(MP_STATE_VM(bluetooth));
-    bool sched;
-    if (enqueue_irq(o, 2 + 2 + 2, MP_BLUETOOTH_IRQ_GATTC_WRITE_STATUS, &sched)) {
+    if (enqueue_irq(o, 2 + 2 + 2, MP_BLUETOOTH_IRQ_GATTC_WRITE_STATUS)) {
         ringbuf_put16(&o->ringbuf, conn_handle);
         ringbuf_put16(&o->ringbuf, value_handle);
         ringbuf_put16(&o->ringbuf, status);
     }
+    schedule_ringbuf();
     MICROPY_PY_BLUETOOTH_EXIT
-    schedule_ringbuf(sched);
 }
 
 #endif // MICROPY_PY_BLUETOOTH_ENABLE_CENTRAL_MODE

extmod/modbluetooth.h

@@ -47,6 +47,12 @@
 #define MICROPY_PY_BLUETOOTH_GATTS_ON_READ_CALLBACK (0)
 #endif
 
+// This is used to protect the ringbuffer.
+#ifndef MICROPY_PY_BLUETOOTH_ENTER
+#define MICROPY_PY_BLUETOOTH_ENTER mp_uint_t atomic_state = MICROPY_BEGIN_ATOMIC_SECTION();
+#define MICROPY_PY_BLUETOOTH_EXIT MICROPY_END_ATOMIC_SECTION(atomic_state);
+#endif
+
 // Common constants.
 #ifndef MP_BLUETOOTH_MAX_ATTR_SIZE
 #define MP_BLUETOOTH_MAX_ATTR_SIZE (20)
@@ -59,6 +65,10 @@
 #define MP_BLUETOOTH_CHARACTERISTIC_FLAG_WRITE    (1 << 3)
 #define MP_BLUETOOTH_CHARACTERISTIC_FLAG_NOTIFY   (1 << 4)
 
+// For mp_bluetooth_gattc_write, the mode parameter
+#define MP_BLUETOOTH_WRITE_MODE_NO_RESPONSE     (0)
+#define MP_BLUETOOTH_WRITE_MODE_WITH_RESPONSE   (1)
+
 // Type value also doubles as length.
 #define MP_BLUETOOTH_UUID_TYPE_16  (2)
 #define MP_BLUETOOTH_UUID_TYPE_32  (4)
@@ -213,7 +223,7 @@ int mp_bluetooth_gattc_discover_descriptors(uint16_t conn_handle, uint16_t start
 int mp_bluetooth_gattc_read(uint16_t conn_handle, uint16_t value_handle);
 
 // Write the value to the remote peripheral.
-int mp_bluetooth_gattc_write(uint16_t conn_handle, uint16_t value_handle, const uint8_t *value, size_t *value_len);
+int mp_bluetooth_gattc_write(uint16_t conn_handle, uint16_t value_handle, const uint8_t *value, size_t *value_len, unsigned int mode);
 #endif
 
 /////////////////////////////////////////////////////////////////////////////
@@ -247,7 +257,11 @@ void mp_bluetooth_gattc_on_characteristic_result(uint16_t conn_handle, uint16_t
 void mp_bluetooth_gattc_on_descriptor_result(uint16_t conn_handle, uint16_t handle, mp_obj_bluetooth_uuid_t *descriptor_uuid);
 
 // Notify modbluetooth that a read has completed with data (or notify/indicate data available, use `event` to disambiguate).
-void mp_bluetooth_gattc_on_data_available(uint16_t event, uint16_t conn_handle, uint16_t value_handle, const uint8_t *data, size_t data_len);
+// Note: these functions are to be called in a group protected by MICROPY_PY_BLUETOOTH_ENTER/EXIT.
+// _start returns the number of bytes to submit to the calls to _chunk, followed by a call to _end.
+size_t mp_bluetooth_gattc_on_data_available_start(uint16_t event, uint16_t conn_handle, uint16_t value_handle, size_t data_len);
+void mp_bluetooth_gattc_on_data_available_chunk(const uint8_t *data, size_t data_len);
+void mp_bluetooth_gattc_on_data_available_end(void);
 
 // Notify modbluetooth that a write has completed.
 void mp_bluetooth_gattc_on_write_status(uint16_t conn_handle, uint16_t value_handle, uint16_t status);

extmod/modbluetooth_nimble.c

@@ -616,6 +616,20 @@ int mp_bluetooth_gatts_set_buffer(uint16_t value_handle, size_t len, bool append
 
 #if MICROPY_PY_BLUETOOTH_ENABLE_CENTRAL_MODE
 
+STATIC void gattc_on_data_available(uint16_t event, uint16_t conn_handle, uint16_t value_handle, const struct os_mbuf *om) {
+    MICROPY_PY_BLUETOOTH_ENTER
+    size_t len = OS_MBUF_PKTLEN(om);
+    len = mp_bluetooth_gattc_on_data_available_start(event, conn_handle, value_handle, len);
+    while (len > 0 && om != NULL) {
+        size_t n = MIN(om->om_len, len);
+        mp_bluetooth_gattc_on_data_available_chunk(OS_MBUF_DATA(om, const uint8_t*), n);
+        len -= n;
+        om = SLIST_NEXT(om, om_next);
+    }
+    mp_bluetooth_gattc_on_data_available_end();
+    MICROPY_PY_BLUETOOTH_EXIT
+}
+
 STATIC int gap_scan_cb(struct ble_gap_event *event, void *arg) {
     DEBUG_EVENT_printf("gap_scan_cb: event=%d type=%d\n", event->type, event->type == BLE_GAP_EVENT_DISC ? event->disc.event_type : -1);
 
@@ -674,8 +688,6 @@ int mp_bluetooth_gap_scan_stop(void) {
 STATIC int peripheral_gap_event_cb(struct ble_gap_event *event, void *arg) {
     DEBUG_EVENT_printf("peripheral_gap_event_cb: event=%d\n", event->type);
     struct ble_gap_conn_desc desc;
-    uint8_t buf[MP_BLUETOOTH_MAX_ATTR_SIZE];
-    size_t len;
     uint8_t addr[6] = {0};
 
     switch (event->type) {
@@ -698,15 +710,11 @@ STATIC int peripheral_gap_event_cb(struct ble_gap_event *event, void *arg) {
 
             break;
 
-        case BLE_GAP_EVENT_NOTIFY_RX:
-            len = MIN(MP_BLUETOOTH_MAX_ATTR_SIZE, OS_MBUF_PKTLEN(event->notify_rx.om));
-            os_mbuf_copydata(event->notify_rx.om, 0, len, buf);
-            if (event->notify_rx.indication == 0) {
-                mp_bluetooth_gattc_on_data_available(MP_BLUETOOTH_IRQ_GATTC_NOTIFY, event->notify_rx.conn_handle, event->notify_rx.attr_handle, buf, len);
-            } else {
-                mp_bluetooth_gattc_on_data_available(MP_BLUETOOTH_IRQ_GATTC_INDICATE, event->notify_rx.conn_handle, event->notify_rx.attr_handle, buf, len);
-            }
+        case BLE_GAP_EVENT_NOTIFY_RX: {
+            uint16_t ev = event->notify_rx.indication == 0 ? MP_BLUETOOTH_IRQ_GATTC_NOTIFY : MP_BLUETOOTH_IRQ_GATTC_INDICATE;
+            gattc_on_data_available(ev, event->notify_rx.conn_handle, event->notify_rx.attr_handle, event->notify_rx.om);
             break;
+        }
 
         case BLE_GAP_EVENT_CONN_UPDATE:
             // TODO
@@ -790,10 +798,7 @@ STATIC int ble_gatt_attr_read_cb(uint16_t conn_handle, const struct ble_gatt_err
     DEBUG_EVENT_printf("ble_gatt_attr_read_cb: conn_handle=%d status=%d handle=%d\n", conn_handle, error->status, attr ? attr->handle : -1);
     // TODO: Maybe send NULL if error->status non-zero.
     if (error->status == 0) {
-        uint8_t buf[MP_BLUETOOTH_MAX_ATTR_SIZE];
-        size_t len = MIN(MP_BLUETOOTH_MAX_ATTR_SIZE, OS_MBUF_PKTLEN(attr->om));
-        os_mbuf_copydata(attr->om, 0, len, buf);
-        mp_bluetooth_gattc_on_data_available(MP_BLUETOOTH_IRQ_GATTC_READ_RESULT, conn_handle, attr->handle, buf, len);
+        gattc_on_data_available(MP_BLUETOOTH_IRQ_GATTC_READ_RESULT, conn_handle, attr->handle, attr->om);
     }
     return 0;
 }
@@ -811,8 +816,15 @@ STATIC int ble_gatt_attr_write_cb(uint16_t conn_handle, const struct ble_gatt_er
 }
 
 // Write the value to the remote peripheral.
-int mp_bluetooth_gattc_write(uint16_t conn_handle, uint16_t value_handle, const uint8_t *value, size_t *value_len) {
-    int err = ble_gattc_write_flat(conn_handle, value_handle, value, *value_len, &ble_gatt_attr_write_cb, NULL);
+int mp_bluetooth_gattc_write(uint16_t conn_handle, uint16_t value_handle, const uint8_t *value, size_t *value_len, unsigned int mode) {
+    int err;
+    if (mode == MP_BLUETOOTH_WRITE_MODE_NO_RESPONSE) {
+        err = ble_gattc_write_no_rsp_flat(conn_handle, value_handle, value, *value_len);
+    } else if (mode == MP_BLUETOOTH_WRITE_MODE_WITH_RESPONSE) {
+        err = ble_gattc_write_flat(conn_handle, value_handle, value, *value_len, &ble_gatt_attr_write_cb, NULL);
+    } else {
+        err = BLE_HS_EINVAL;
+    }
     return ble_hs_err_to_errno(err);
 }
 

extmod/modbtree.c

@@ -52,7 +52,9 @@ typedef struct _mp_obj_btree_t {
     byte next_flags;
 } mp_obj_btree_t;
 
+#if !MICROPY_ENABLE_DYNRUNTIME
 STATIC const mp_obj_type_t btree_type;
+#endif
 
 #define CHECK_ERROR(res) \
         if (res == RET_ERROR) { \
@@ -60,7 +62,7 @@ STATIC const mp_obj_type_t btree_type;
         }
 
 void __dbpanic(DB *db) {
-    printf("__dbpanic(%p)\n", db);
+    mp_printf(&mp_plat_print, "__dbpanic(%p)\n", db);
 }
 
 STATIC mp_obj_btree_t *btree_new(DB *db) {
@@ -295,6 +297,7 @@ STATIC mp_obj_t btree_binary_op(mp_binary_op_t op, mp_obj_t lhs_in, mp_obj_t rhs
     }
 }
 
+#if !MICROPY_ENABLE_DYNRUNTIME
 STATIC const mp_rom_map_elem_t btree_locals_dict_table[] = {
     { MP_ROM_QSTR(MP_QSTR_close), MP_ROM_PTR(&btree_close_obj) },
     { MP_ROM_QSTR(MP_QSTR_flush), MP_ROM_PTR(&btree_flush_obj) },
@@ -319,6 +322,7 @@ STATIC const mp_obj_type_t btree_type = {
     .subscr = btree_subscr,
     .locals_dict = (void*)&btree_locals_dict,
 };
+#endif
 
 STATIC const FILEVTABLE btree_stream_fvtable = {
     mp_stream_posix_read,
@@ -327,6 +331,7 @@ STATIC const FILEVTABLE btree_stream_fvtable = {
     mp_stream_posix_fsync
 };
 
+#if !MICROPY_ENABLE_DYNRUNTIME
 STATIC mp_obj_t mod_btree_open(size_t n_args, const mp_obj_t *pos_args, mp_map_t *kw_args) {
     static const mp_arg_t allowed_args[] = {
         { MP_QSTR_flags, MP_ARG_KW_ONLY | MP_ARG_INT, {.u_int = 0} },
@@ -373,5 +378,6 @@ const mp_obj_module_t mp_module_btree = {
     .base = { &mp_type_module },
     .globals = (mp_obj_dict_t*)&mp_module_btree_globals,
 };
+#endif
 
 #endif // MICROPY_PY_BTREE

extmod/modframebuf.c

@@ -580,6 +580,7 @@ STATIC mp_obj_t framebuf_text(size_t n_args, const mp_obj_t *args) {
 }
 STATIC MP_DEFINE_CONST_FUN_OBJ_VAR_BETWEEN(framebuf_text_obj, 4, 5, framebuf_text);
 
+#if !MICROPY_ENABLE_DYNRUNTIME
 STATIC const mp_rom_map_elem_t framebuf_locals_dict_table[] = {
     { MP_ROM_QSTR(MP_QSTR_fill), MP_ROM_PTR(&framebuf_fill_obj) },
     { MP_ROM_QSTR(MP_QSTR_fill_rect), MP_ROM_PTR(&framebuf_fill_rect_obj) },
@@ -601,6 +602,7 @@ STATIC const mp_obj_type_t mp_type_framebuf = {
     .buffer_p = { .get_buffer = framebuf_get_buffer },
     .locals_dict = (mp_obj_dict_t*)&framebuf_locals_dict,
 };
+#endif
 
 // this factory function is provided for backwards compatibility with old FrameBuffer1 class
 STATIC mp_obj_t legacy_framebuffer1(size_t n_args, const mp_obj_t *args) {
@@ -624,6 +626,7 @@ STATIC mp_obj_t legacy_framebuffer1(size_t n_args, const mp_obj_t *args) {
 }
 STATIC MP_DEFINE_CONST_FUN_OBJ_VAR_BETWEEN(legacy_framebuffer1_obj, 3, 4, legacy_framebuffer1);
 
+#if !MICROPY_ENABLE_DYNRUNTIME
 STATIC const mp_rom_map_elem_t framebuf_module_globals_table[] = {
     { MP_ROM_QSTR(MP_QSTR___name__), MP_ROM_QSTR(MP_QSTR_framebuf) },
     { MP_ROM_QSTR(MP_QSTR_FrameBuffer), MP_ROM_PTR(&mp_type_framebuf) },
@@ -644,5 +647,6 @@ const mp_obj_module_t mp_module_framebuf = {
     .base = { &mp_type_module },
     .globals = (mp_obj_dict_t*)&framebuf_module_globals,
 };
+#endif
 
 #endif // MICROPY_PY_FRAMEBUF

extmod/moduheapq.c

@@ -103,6 +103,7 @@ STATIC mp_obj_t mod_uheapq_heapify(mp_obj_t heap_in) {
 }
 STATIC MP_DEFINE_CONST_FUN_OBJ_1(mod_uheapq_heapify_obj, mod_uheapq_heapify);
 
+#if !MICROPY_ENABLE_DYNRUNTIME
 STATIC const mp_rom_map_elem_t mp_module_uheapq_globals_table[] = {
     { MP_ROM_QSTR(MP_QSTR___name__), MP_ROM_QSTR(MP_QSTR_uheapq) },
     { MP_ROM_QSTR(MP_QSTR_heappush), MP_ROM_PTR(&mod_uheapq_heappush_obj) },
@@ -116,5 +117,6 @@ const mp_obj_module_t mp_module_uheapq = {
     .base = { &mp_type_module },
     .globals = (mp_obj_dict_t*)&mp_module_uheapq_globals,
 };
+#endif
 
 #endif //MICROPY_PY_UHEAPQ

extmod/modurandom.c

@@ -36,8 +36,10 @@
 // http://www.literatecode.com/yasmarang
 // Public Domain
 
+#if !MICROPY_ENABLE_DYNRUNTIME
 STATIC uint32_t yasmarang_pad = 0xeda4baba, yasmarang_n = 69, yasmarang_d = 233;
 STATIC uint8_t yasmarang_dat = 0;
+#endif
 
 STATIC uint32_t yasmarang(void)
 {
@@ -208,6 +210,7 @@ STATIC mp_obj_t mod_urandom___init__() {
 STATIC MP_DEFINE_CONST_FUN_OBJ_0(mod_urandom___init___obj, mod_urandom___init__);
 #endif
 
+#if !MICROPY_ENABLE_DYNRUNTIME
 STATIC const mp_rom_map_elem_t mp_module_urandom_globals_table[] = {
     { MP_ROM_QSTR(MP_QSTR___name__), MP_ROM_QSTR(MP_QSTR_urandom) },
     #ifdef MICROPY_PY_URANDOM_SEED_INIT_FUNC
@@ -232,5 +235,6 @@ const mp_obj_module_t mp_module_urandom = {
     .base = { &mp_type_module },
     .globals = (mp_obj_dict_t*)&mp_module_urandom_globals,
 };
+#endif
 
 #endif //MICROPY_PY_URANDOM

extmod/modure.c

@@ -144,6 +144,7 @@ MP_DEFINE_CONST_FUN_OBJ_VAR_BETWEEN(match_end_obj, 1, 2, match_end);
 
 #endif
 
+#if !MICROPY_ENABLE_DYNRUNTIME
 STATIC const mp_rom_map_elem_t match_locals_dict_table[] = {
     { MP_ROM_QSTR(MP_QSTR_group), MP_ROM_PTR(&match_group_obj) },
     #if MICROPY_PY_URE_MATCH_GROUPS
@@ -164,6 +165,7 @@ STATIC const mp_obj_type_t match_type = {
     .print = match_print,
     .locals_dict = (void*)&match_locals_dict,
 };
+#endif
 
 STATIC void re_print(const mp_print_t *print, mp_obj_t self_in, mp_print_kind_t kind) {
     (void)kind;
@@ -363,6 +365,7 @@ MP_DEFINE_CONST_FUN_OBJ_VAR_BETWEEN(re_sub_obj, 3, 5, re_sub);
 
 #endif
 
+#if !MICROPY_ENABLE_DYNRUNTIME
 STATIC const mp_rom_map_elem_t re_locals_dict_table[] = {
     { MP_ROM_QSTR(MP_QSTR_match), MP_ROM_PTR(&re_match_obj) },
     { MP_ROM_QSTR(MP_QSTR_search), MP_ROM_PTR(&re_search_obj) },
@@ -380,6 +383,7 @@ STATIC const mp_obj_type_t re_type = {
     .print = re_print,
     .locals_dict = (void*)&re_locals_dict,
 };
+#endif
 
 STATIC mp_obj_t mod_re_compile(size_t n_args, const mp_obj_t *args) {
     (void)n_args;
@@ -437,6 +441,7 @@ STATIC mp_obj_t mod_re_sub(size_t n_args, const mp_obj_t *args) {
 MP_DEFINE_CONST_FUN_OBJ_VAR_BETWEEN(mod_re_sub_obj, 3, 5, mod_re_sub);
 #endif
 
+#if !MICROPY_ENABLE_DYNRUNTIME
 STATIC const mp_rom_map_elem_t mp_module_re_globals_table[] = {
     { MP_ROM_QSTR(MP_QSTR___name__), MP_ROM_QSTR(MP_QSTR_ure) },
     { MP_ROM_QSTR(MP_QSTR_compile), MP_ROM_PTR(&mod_re_compile_obj) },
@@ -456,6 +461,7 @@ const mp_obj_module_t mp_module_ure = {
     .base = { &mp_type_module },
     .globals = (mp_obj_dict_t*)&mp_module_re_globals,
 };
+#endif
 
 // Source files #include'd here to make sure they're compiled in
 // only if module is enabled by config setting.

extmod/moduzlib.c

@@ -122,6 +122,7 @@ STATIC mp_uint_t decompio_read(mp_obj_t o_in, void *buf, mp_uint_t size, int *er
     return o->decomp.dest - (byte*)buf;
 }
 
+#if !MICROPY_ENABLE_DYNRUNTIME
 STATIC const mp_rom_map_elem_t decompio_locals_dict_table[] = {
     { MP_ROM_QSTR(MP_QSTR_read), MP_ROM_PTR(&mp_stream_read_obj) },
     { MP_ROM_QSTR(MP_QSTR_readinto), MP_ROM_PTR(&mp_stream_readinto_obj) },
@@ -129,11 +130,13 @@ STATIC const mp_rom_map_elem_t decompio_locals_dict_table[] = {
 };
 
 STATIC MP_DEFINE_CONST_DICT(decompio_locals_dict, decompio_locals_dict_table);
+#endif
 
 STATIC const mp_stream_p_t decompio_stream_p = {
     .read = decompio_read,
 };
 
+#if !MICROPY_ENABLE_DYNRUNTIME
 STATIC const mp_obj_type_t decompio_type = {
     { &mp_type_type },
     .name = MP_QSTR_DecompIO,
@@ -141,6 +144,7 @@ STATIC const mp_obj_type_t decompio_type = {
     .protocol = &decompio_stream_p,
     .locals_dict = (void*)&decompio_locals_dict,
 };
+#endif
 
 STATIC mp_obj_t mod_uzlib_decompress(size_t n_args, const mp_obj_t *args) {
     mp_obj_t data = args[0];
@@ -201,6 +205,7 @@ error:
 }
 STATIC MP_DEFINE_CONST_FUN_OBJ_VAR_BETWEEN(mod_uzlib_decompress_obj, 1, 3, mod_uzlib_decompress);
 
+#if !MICROPY_ENABLE_DYNRUNTIME
 STATIC const mp_rom_map_elem_t mp_module_uzlib_globals_table[] = {
     { MP_ROM_QSTR(MP_QSTR___name__), MP_ROM_QSTR(MP_QSTR_uzlib) },
     { MP_ROM_QSTR(MP_QSTR_decompress), MP_ROM_PTR(&mod_uzlib_decompress_obj) },
@@ -213,6 +218,7 @@ const mp_obj_module_t mp_module_uzlib = {
     .base = { &mp_type_module },
     .globals = (mp_obj_dict_t*)&mp_module_uzlib_globals,
 };
+#endif
 
 // Source files #include'd here to make sure they're compiled in
 // only if module is enabled by config setting.

extmod/webrepl/manifest.py

@@ -0,0 +1 @@
+freeze('.', ('webrepl.py', 'webrepl_setup.py', 'websocket_helper.py',))

extmod/webrepl/webrepl.py

@@ -43,8 +43,9 @@ def accept_conn(listen_sock):
     ws = uwebsocket.websocket(cl, True)
     ws = _webrepl._webrepl(ws)
     cl.setblocking(False)
-    # notify REPL on socket incoming data
-    cl.setsockopt(socket.SOL_SOCKET, 20, uos.dupterm_notify)
+    # notify REPL on socket incoming data (ESP32/ESP8266-only)
+    if hasattr(uos, 'dupterm_notify'):
+        cl.setsockopt(socket.SOL_SOCKET, 20, uos.dupterm_notify)
     uos.dupterm(ws)
 
 

lib/lv_bindings

@@ -1 +1 @@
-Subproject commit 29ef44705de7ac2ba64fc278e0aaa341b4a65554
+Subproject commit 33a4cf02601e6959255465a3f34450e6788058e8

lib/tinytest/tinytest.c

@@ -234,9 +234,8 @@ testcase_run_one(const struct testgroup_t *group,
 		return SKIP;
 	}
 
-	printf("# starting %s%s\n", group->prefix, testcase->name);
 	if (opt_verbosity>0 && !opt_forked) {
-		//printf("%s%s: ", group->prefix, testcase->name);
+		printf("%s%s: ", group->prefix, testcase->name);
 	} else {
 		if (opt_verbosity==0) printf(".");
 		cur_test_prefix = group->prefix;
@@ -253,7 +252,6 @@ testcase_run_one(const struct testgroup_t *group,
 		outcome = testcase_run_bare_(testcase);
 	}
 
-	printf("%s%s: ", group->prefix, testcase->name);
 	if (outcome == OK) {
 		++n_ok;
 		if (opt_verbosity>0 && !opt_forked)
@@ -265,8 +263,7 @@ testcase_run_one(const struct testgroup_t *group,
 	} else {
 		++n_bad;
 		if (!opt_forked)
-			//printf("\n  [%s FAILED]\n", testcase->name);
-			puts("FAILED");
+			printf("\n  [%s FAILED]\n", testcase->name);
 	}
 
 	if (opt_forked) {

mpy-cross/README.md

@@ -22,4 +22,10 @@ the unix port of MicroPython requires the following:
 
     $ ./mpy-cross -mcache-lookup-bc foo.py
 
+If the Python code contains `@native` or `@viper` annotations, then you must
+specify `-march` to match the target architecture.
+
 Run `./mpy-cross -h` to get a full list of options.
+
+The optimisation level is 0 by default. Optimisation levels are detailed in
+https://docs.micropython.org/en/latest/library/micropython.html#micropython.opt_level

mpy-cross/main.c

@@ -109,7 +109,7 @@ STATIC int usage(char **argv) {
 "-msmall-int-bits=number : set the maximum bits used to encode a small-int\n"
 "-mno-unicode : don't support unicode in compiled strings\n"
 "-mcache-lookup-bc : cache map lookups in the bytecode\n"
-"-march=<arch> : set architecture for native emitter; x86, x64, armv6, armv7m, xtensa, xtensawin\n"
+"-march=<arch> : set architecture for native emitter; x86, x64, armv6, armv7m, armv7em, armv7emsp, armv7emdp, xtensa, xtensawin\n"
 "\n"
 "Implementation specific options:\n", argv[0]
 );
@@ -285,6 +285,15 @@ MP_NOINLINE int main_(int argc, char **argv) {
                 } else if (strcmp(arch, "armv7m") == 0) {
                     mp_dynamic_compiler.native_arch = MP_NATIVE_ARCH_ARMV7M;
                     mp_dynamic_compiler.nlr_buf_num_regs = MICROPY_NLR_NUM_REGS_ARM_THUMB_FP;
+                } else if (strcmp(arch, "armv7em") == 0) {
+                    mp_dynamic_compiler.native_arch = MP_NATIVE_ARCH_ARMV7EM;
+                    mp_dynamic_compiler.nlr_buf_num_regs = MICROPY_NLR_NUM_REGS_ARM_THUMB_FP;
+                } else if (strcmp(arch, "armv7emsp") == 0) {
+                    mp_dynamic_compiler.native_arch = MP_NATIVE_ARCH_ARMV7EMSP;
+                    mp_dynamic_compiler.nlr_buf_num_regs = MICROPY_NLR_NUM_REGS_ARM_THUMB_FP;
+                } else if (strcmp(arch, "armv7emdp") == 0) {
+                    mp_dynamic_compiler.native_arch = MP_NATIVE_ARCH_ARMV7EMDP;
+                    mp_dynamic_compiler.nlr_buf_num_regs = MICROPY_NLR_NUM_REGS_ARM_THUMB_FP;
                 } else if (strcmp(arch, "xtensa") == 0) {
                     mp_dynamic_compiler.native_arch = MP_NATIVE_ARCH_XTENSA;
                     mp_dynamic_compiler.nlr_buf_num_regs = MICROPY_NLR_NUM_REGS_XTENSA;

ports/esp32/Makefile

@@ -323,6 +323,7 @@ SRC_C = \
 	modsocket.c \
 	modesp.c \
 	esp32_partition.c \
+	esp32_rmt.c \
 	esp32_ulp.c \
 	modesp32.c \
 	espneopixel.c \

ports/esp32/boards/TINYPICO/manifest.py

@@ -0,0 +1,2 @@
+include('$(PORT_DIR)/boards/manifest.py')
+freeze("modules")

ports/esp32/boards/TINYPICO/modules/dotstar.py

@@ -0,0 +1,228 @@
+# DotStar strip driver for MicroPython
+#
+# The MIT License (MIT)
+#
+# Copyright (c) 2016 Damien P. George (original Neopixel object)
+# Copyright (c) 2017 Ladyada
+# Copyright (c) 2017 Scott Shawcroft for Adafruit Industries
+# Copyright (c) 2019 Matt Trentini (porting back to MicroPython)
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included in
+# all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+# THE SOFTWARE.
+
+START_HEADER_SIZE = 4
+LED_START = 0b11100000  # Three "1" bits, followed by 5 brightness bits
+
+# Pixel color order constants
+RGB = (0, 1, 2)
+RBG = (0, 2, 1)
+GRB = (1, 0, 2)
+GBR = (1, 2, 0)
+BRG = (2, 0, 1)
+BGR = (2, 1, 0)
+
+
+class DotStar:
+    """
+    A sequence of dotstars.
+
+    :param SPI spi: The SPI object to write output to.
+    :param int n: The number of dotstars in the chain
+    :param float brightness: Brightness of the pixels between 0.0 and 1.0
+    :param bool auto_write: True if the dotstars should immediately change when
+        set. If False, `show` must be called explicitly.
+    :param tuple pixel_order: Set the pixel order on the strip - different
+         strips implement this differently. If you send red, and it looks blue
+         or green on the strip, modify this! It should be one of the values above
+
+
+    Example for TinyPICO:
+
+    .. code-block:: python
+
+        from dotstar import DotStar
+        from machine import Pin, SPI
+
+        spi = SPI(sck=Pin(12), mosi=Pin(13), miso=Pin(18)) # Configure SPI - note: miso is unused
+        dotstar = DotStar(spi, 1)
+        dotstar[0] = (128, 0, 0) # Red
+    """
+
+    def __init__(self, spi, n, *, brightness=1.0, auto_write=True,
+                 pixel_order=BGR):
+        self._spi = spi
+        self._n = n
+        # Supply one extra clock cycle for each two pixels in the strip.
+        self.end_header_size = n // 16
+        if n % 16 != 0:
+            self.end_header_size += 1
+        self._buf = bytearray(n * 4 + START_HEADER_SIZE + self.end_header_size)
+        self.end_header_index = len(self._buf) - self.end_header_size
+        self.pixel_order = pixel_order
+        # Four empty bytes to start.
+        for i in range(START_HEADER_SIZE):
+            self._buf[i] = 0x00
+        # Mark the beginnings of each pixel.
+        for i in range(START_HEADER_SIZE, self.end_header_index, 4):
+            self._buf[i] = 0xff
+        # 0xff bytes at the end.
+        for i in range(self.end_header_index, len(self._buf)):
+            self._buf[i] = 0xff
+        self._brightness = 1.0
+        # Set auto_write to False temporarily so brightness setter does _not_
+        # call show() while in __init__.
+        self.auto_write = False
+        self.brightness = brightness
+        self.auto_write = auto_write
+
+    def deinit(self):
+        """Blank out the DotStars and release the resources."""
+        self.auto_write = False
+        for i in range(START_HEADER_SIZE, self.end_header_index):
+            if i % 4 != 0:
+                self._buf[i] = 0
+        self.show()
+        if self._spi:
+            self._spi.deinit()
+
+    def __enter__(self):
+        return self
+
+    def __exit__(self, exception_type, exception_value, traceback):
+        self.deinit()
+
+    def __repr__(self):
+        return "[" + ", ".join([str(x) for x in self]) + "]"
+
+    def _set_item(self, index, value):
+        """
+        value can be one of three things:
+                a (r,g,b) list/tuple
+                a (r,g,b, brightness) list/tuple
+                a single, longer int that contains RGB values, like 0xFFFFFF
+            brightness, if specified should be a float 0-1
+
+        Set a pixel value. You can set per-pixel brightness here, if it's not passed it
+        will use the max value for pixel brightness value, which is a good default.
+
+        Important notes about the per-pixel brightness - it's accomplished by
+        PWMing the entire output of the LED, and that PWM is at a much
+        slower clock than the rest of the LEDs. This can cause problems in
+        Persistence of Vision Applications
+        """
+
+        offset = index * 4 + START_HEADER_SIZE
+        rgb = value
+        if isinstance(value, int):
+            rgb = (value >> 16, (value >> 8) & 0xff, value & 0xff)
+
+        if len(rgb) == 4:
+            brightness = value[3]
+            # Ignore value[3] below.
+        else:
+            brightness = 1
+
+        # LED startframe is three "1" bits, followed by 5 brightness bits
+        # then 8 bits for each of R, G, and B. The order of those 3 are configurable and
+        # vary based on hardware
+        # same as math.ceil(brightness * 31) & 0b00011111
+        # Idea from https://www.codeproject.com/Tips/700780/Fast-floor-ceiling-functions
+        brightness_byte = 32 - int(32 - brightness * 31) & 0b00011111
+        self._buf[offset] = brightness_byte | LED_START
+        self._buf[offset + 1] = rgb[self.pixel_order[0]]
+        self._buf[offset + 2] = rgb[self.pixel_order[1]]
+        self._buf[offset + 3] = rgb[self.pixel_order[2]]
+
+    def __setitem__(self, index, val):
+        if isinstance(index, slice):
+            start, stop, step = index.indices(self._n)
+            length = stop - start
+            if step != 0:
+                # same as math.ceil(length / step)
+                # Idea from https://fizzbuzzer.com/implement-a-ceil-function/
+                length = (length + step - 1) // step
+            if len(val) != length:
+                raise ValueError("Slice and input sequence size do not match.")
+            for val_i, in_i in enumerate(range(start, stop, step)):
+                self._set_item(in_i, val[val_i])
+        else:
+            self._set_item(index, val)
+
+        if self.auto_write:
+            self.show()
+
+    def __getitem__(self, index):
+        if isinstance(index, slice):
+            out = []
+            for in_i in range(*index.indices(self._n)):
+                out.append(
+                    tuple(self._buf[in_i * 4 + (3 - i) + START_HEADER_SIZE] for i in range(3)))
+            return out
+        if index < 0:
+            index += len(self)
+        if index >= self._n or index < 0:
+            raise IndexError
+        offset = index * 4
+        return tuple(self._buf[offset + (3 - i) + START_HEADER_SIZE]
+                     for i in range(3))
+
+    def __len__(self):
+        return self._n
+
+    @property
+    def brightness(self):
+        """Overall brightness of the pixel"""
+        return self._brightness
+
+    @brightness.setter
+    def brightness(self, brightness):
+        self._brightness = min(max(brightness, 0.0), 1.0)
+        if self.auto_write:
+            self.show()
+
+    def fill(self, color):
+        """Colors all pixels the given ***color***."""
+        auto_write = self.auto_write
+        self.auto_write = False
+        for i in range(self._n):
+            self[i] = color
+        if auto_write:
+            self.show()
+        self.auto_write = auto_write
+
+    def show(self):
+        """Shows the new colors on the pixels themselves if they haven't already
+        been autowritten.
+
+        The colors may or may not be showing after this function returns because
+        it may be done asynchronously."""
+        # Create a second output buffer if we need to compute brightness
+        buf = self._buf
+        if self.brightness < 1.0:
+            buf = bytearray(self._buf)
+            # Four empty bytes to start.
+            for i in range(START_HEADER_SIZE):
+                buf[i] = 0x00
+            for i in range(START_HEADER_SIZE, self.end_header_index):
+                buf[i] = self._buf[i] if i % 4 == 0 else int(self._buf[i] * self._brightness)
+            # Four 0xff bytes at the end.
+            for i in range(self.end_header_index, len(buf)):
+                buf[i] = 0xff
+
+        if self._spi:
+            self._spi.write(buf)

ports/esp32/boards/TINYPICO/modules/tinypico.py

@@ -0,0 +1,113 @@
+# TinyPICO MicroPython Helper Library
+# 2019 Seon Rozenblum, Matt Trentini
+#
+# Project home:
+#   https://github.com/TinyPICO
+#
+# 2019-Mar-12 - v0.1 - Initial implementation
+# 2019-May-20 - v1.0 - Initial Release
+# 2019-Oct-23 - v1.1 - Removed temp sensor code, prep for frozen modules
+
+# Import required libraries
+from micropython import const
+from machine import Pin, SPI, ADC
+import machine, time, esp32
+
+# TinyPICO Hardware Pin Assignments
+
+# Battery
+BAT_VOLTAGE = const(35)
+BAT_CHARGE = const(34)
+
+# APA102 Dotstar pins for production boards
+DOTSTAR_CLK = const(12)
+DOTSTAR_DATA = const(2)
+DOTSTAR_PWR = const(13)
+
+# SPI
+SPI_MOSI = const(23)
+SPI_CLK = const(18)
+SPI_MISO = const(19)
+
+#I2C
+I2C_SDA = const(21)
+I2C_SCL = const(22)
+
+#DAC
+DAC1 = const(25)
+DAC2 = const(26)
+
+# Helper functions
+
+# Get a *rough* estimate of the current battery voltage
+# If the battery is not present, the charge IC will still report it's trying to charge at X voltage
+# so it will still show a voltage.
+def get_battery_voltage():
+    """
+    Returns the current battery voltage. If no battery is connected, returns 3.7V
+    This is an approximation only, but useful to detect of the charge state of the battery is getting low.
+    """
+    adc = ADC(Pin(BAT_VOLTAGE))             # Assign the ADC pin to read
+    measuredvbat = adc.read()               # Read the value
+    measuredvbat /= 4095                    # divide by 4095 as we are using the default ADC voltage range of 0-1V
+    measuredvbat *= 3.7                     # Multiply by 3.7V, our reference voltage
+    return measuredvbat
+
+# Return the current charge state of the battery - we need to read the value multiple times
+# to eliminate false negatives due to the charge IC not knowing the difference between no battery
+# and a full battery not charging - This is why the charge LED flashes
+def get_battery_charging():
+    """
+    Returns the current battery charging state.
+    This can trigger false positives as the charge IC can't tell the difference between a full battery or no battery connected.
+    """
+    measuredVal = 0                         # start our reading at 0
+    io = Pin(BAT_CHARGE, Pin.IN)            # Assign the pin to read
+
+    for y in range(0, 10):                  # loop through 10 times adding the read values together to ensure no false positives
+        measuredVal += io.value()
+
+    return measuredVal == 0                 # return True if the value is 0
+
+
+# Power to the on-board Dotstar is controlled by a PNP transistor, so low is ON and high is OFF
+# We also need to set the Dotstar clock and data pins to be inputs to prevent power leakage when power is off
+# This might be improved at a future date
+# The reason we have power control for the Dotstar is that it has a quiescent current of around 1mA, so we
+# need to be able to cut power to it to minimise power consumption during deep sleep or with general battery powered use
+# to minimise unneeded battery drain
+def set_dotstar_power(state):
+    """Set the power for the on-board Dostar to allow no current draw when not needed."""
+    # Set the power pin to the inverse of state
+    if state:
+        Pin(DOTSTAR_PWR, Pin.OUT, None)     # Break the PULL_HOLD on the pin
+        Pin(DOTSTAR_PWR).value(False)       # Set the pin to LOW to enable the Transistor
+    else:
+        Pin(13, Pin.IN, Pin.PULL_HOLD)      # Set PULL_HOLD on the pin to allow the 3V3 pull-up to work
+
+    Pin(DOTSTAR_CLK, Pin.OUT if state else Pin.IN)  # If power is on, set CLK to be output, otherwise input
+    Pin(DOTSTAR_DATA, Pin.OUT if state else Pin.IN) # If power is on, set DATA to be output, otherwise input
+
+    # A small delay to let the IO change state
+    time.sleep(.035)
+
+# Dotstar rainbow colour wheel
+def dotstar_color_wheel(wheel_pos):
+    """Color wheel to allow for cycling through the rainbow of RGB colors."""
+    wheel_pos = wheel_pos % 255
+
+    if wheel_pos < 85:
+        return 255 - wheel_pos * 3, 0, wheel_pos * 3
+    elif wheel_pos < 170:
+        wheel_pos -= 85
+        return 0, wheel_pos * 3, 255 - wheel_pos * 3
+    else:
+        wheel_pos -= 170
+        return wheel_pos * 3, 255 - wheel_pos * 3, 0
+
+# Go into deep sleep but shut down the APA first to save power
+# Use this if you want lowest deep  sleep current
+def go_deepsleep(t):
+    """Deep sleep helper that also powers down the on-board Dotstar."""
+    set_dotstar_power(False)
+    machine.deepsleep(t)

ports/esp32/boards/TINYPICO/mpconfigboard.mk

@@ -4,3 +4,5 @@ SDKCONFIG += boards/sdkconfig.base
 SDKCONFIG += boards/sdkconfig.240mhz
 SDKCONFIG += boards/sdkconfig.spiram
 SDKCONFIG += boards/TINYPICO/sdkconfig.board
+
+FROZEN_MANIFEST ?= $(BOARD_DIR)/manifest.py

ports/esp32/boards/manifest.py

@@ -1,6 +1,6 @@
 freeze('$(PORT_DIR)/modules')
 freeze('$(MPY_DIR)/tools', ('upip.py', 'upip_utarfile.py'))
 freeze('$(MPY_DIR)/ports/esp8266/modules', 'ntptime.py')
-freeze('$(MPY_DIR)/ports/esp8266/modules', ('webrepl.py', 'webrepl_setup.py', 'websocket_helper.py',))
 freeze('$(MPY_DIR)/drivers/dht', 'dht.py')
 freeze('$(MPY_DIR)/drivers/onewire')
+include('$(MPY_DIR)/extmod/webrepl/manifest.py')

ports/esp32/boards/sdkconfig.base

@@ -34,6 +34,9 @@ CONFIG_LWIP_PPP_CHAP_SUPPORT=y
 # Use 4kiB output buffer instead of default 16kiB (because IDF heap is fragmented in 4.0)
 CONFIG_MBEDTLS_ASYMMETRIC_CONTENT_LEN=y
 
+# ULP coprocessor support
+CONFIG_ESP32_ULP_COPROC_ENABLED=y
+
 # v3.3-only (renamed in 4.0)
 CONFIG_LOG_BOOTLOADER_LEVEL_WARN=y
 CONFIG_TASK_WDT_CHECK_IDLE_TASK_CPU0=n
@@ -43,6 +46,7 @@ CONFIG_ENABLE_STATIC_TASK_CLEAN_UP_HOOK=y
 CONFIG_PPP_SUPPORT=y
 CONFIG_PPP_PAP_SUPPORT=y
 CONFIG_PPP_CHAP_SUPPORT=y
+CONFIG_ULP_COPROC_ENABLED=y
 
 # Logs
 CONFIG_LOG_DEFAULT_LEVEL_WARN=y

ports/esp32/esp32_rmt.c

@@ -0,0 +1,237 @@
+/*
+ * This file is part of the MicroPython project, http://micropython.org/
+ *
+ * The MIT License (MIT)
+ *
+ * Copyright (c) 2019 "Matt Trentini" <matt.trentini@gmail.com>
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+ * THE SOFTWARE.
+ */
+
+#include "py/runtime.h"
+#include "modmachine.h"
+#include "mphalport.h"
+#include "driver/rmt.h"
+
+// This exposes the ESP32's RMT module to MicroPython. RMT is provided by the Espressif ESP-IDF:
+//
+//    https://docs.espressif.com/projects/esp-idf/en/latest/api-reference/peripherals/rmt.html
+//
+// With some examples provided:
+//
+//    https://github.com/espressif/arduino-esp32/tree/master/libraries/ESP32/examples/RMT
+//
+// RMT allows accurate (down to 12.5ns resolution) transmit - and receive - of pulse signals.
+// Originally designed to generate infrared remote control signals, the module is very
+// flexible and quite easy-to-use.
+//
+// This current MicroPython implementation lacks some major features, notably receive pulses
+// and carrier output.
+
+// Forward declaration
+extern const mp_obj_type_t esp32_rmt_type;
+
+typedef struct _esp32_rmt_obj_t {
+    mp_obj_base_t base;
+    uint8_t channel_id;
+    gpio_num_t pin;
+    uint8_t clock_div;
+    mp_uint_t num_items;
+    rmt_item32_t* items;
+} esp32_rmt_obj_t;
+
+// Defined in machine_time.c; simply added the error message
+// Fixme: Should use this updated error hadline more widely in the ESP32 port.
+//        At least update the method in machine_time.c.
+STATIC esp_err_t check_esp_err(esp_err_t code) {
+    if (code) {
+        mp_raise_msg(&mp_type_OSError, esp_err_to_name(code));
+    }
+
+    return code;
+}
+
+STATIC mp_obj_t esp32_rmt_make_new(const mp_obj_type_t *type, size_t n_args, size_t n_kw, const mp_obj_t *all_args) {
+    static const mp_arg_t allowed_args[] = {
+        { MP_QSTR_id,        MP_ARG_REQUIRED |                  MP_ARG_INT, {.u_int = -1} },
+        { MP_QSTR_pin,       MP_ARG_REQUIRED | MP_ARG_KW_ONLY | MP_ARG_OBJ, {.u_obj = mp_const_none} },
+        { MP_QSTR_clock_div,                   MP_ARG_KW_ONLY | MP_ARG_INT, {.u_int = 8} }, // 100ns resolution
+    };
+    mp_arg_val_t args[MP_ARRAY_SIZE(allowed_args)];
+    mp_arg_parse_all_kw_array(n_args, n_kw, all_args, MP_ARRAY_SIZE(allowed_args), allowed_args, args);
+    mp_uint_t channel_id = args[0].u_int;
+    gpio_num_t pin_id = machine_pin_get_id(args[1].u_obj);
+    mp_uint_t clock_div = args[2].u_int;
+
+    if (clock_div < 1 || clock_div > 255) {
+        mp_raise_ValueError("clock_div must be between 1 and 255");
+    }
+
+    esp32_rmt_obj_t *self = m_new_obj_with_finaliser(esp32_rmt_obj_t);
+    self->base.type = &esp32_rmt_type;
+    self->channel_id = channel_id;
+    self->pin = pin_id;
+    self->clock_div = clock_div;
+
+    rmt_config_t config;
+    config.rmt_mode = RMT_MODE_TX;
+    config.channel = (rmt_channel_t) self->channel_id;
+    config.gpio_num = self->pin;
+    config.mem_block_num = 1;
+    config.tx_config.loop_en = 0;
+
+    config.tx_config.carrier_en = 0;
+    config.tx_config.idle_output_en = 1;
+    config.tx_config.idle_level = 0;
+    config.tx_config.carrier_duty_percent = 0;
+    config.tx_config.carrier_freq_hz = 0;
+    config.tx_config.carrier_level = 1;
+
+    config.clk_div = self->clock_div;
+
+    check_esp_err(rmt_config(&config));
+    check_esp_err(rmt_driver_install(config.channel, 0, 0));
+
+    return MP_OBJ_FROM_PTR(self);
+}
+
+STATIC void esp32_rmt_print(const mp_print_t *print, mp_obj_t self_in, mp_print_kind_t kind) {
+    esp32_rmt_obj_t *self = MP_OBJ_TO_PTR(self_in);
+    if (self->pin != -1) {
+        mp_printf(print, "RMT(channel=%u, pin=%u, source_freq=%u, clock_div=%u)",
+            self->channel_id, self->pin, APB_CLK_FREQ, self->clock_div);
+    } else {
+        mp_printf(print, "RMT()");
+    }
+}
+
+STATIC mp_obj_t esp32_rmt_deinit(mp_obj_t self_in) {
+    // fixme: check for valid channel. Return exception if error occurs.
+    esp32_rmt_obj_t *self = MP_OBJ_TO_PTR(self_in);
+    if (self->pin != -1) { // Check if channel has already been deinitialised.
+        rmt_driver_uninstall(self->channel_id);
+        self->pin = -1; // -1 to indicate RMT is unused
+        m_free(self->items);
+    }
+    return mp_const_none;
+}
+STATIC MP_DEFINE_CONST_FUN_OBJ_1(esp32_rmt_deinit_obj, esp32_rmt_deinit);
+
+// Return the source frequency.
+// Currently only the APB clock (80MHz) can be used but it is possible other
+// clock sources will added in the future.
+STATIC mp_obj_t esp32_rmt_source_freq(mp_obj_t self_in) {
+    return mp_obj_new_int(APB_CLK_FREQ);
+}
+STATIC MP_DEFINE_CONST_FUN_OBJ_1(esp32_rmt_source_freq_obj, esp32_rmt_source_freq);
+
+// Return the clock divider.
+STATIC mp_obj_t esp32_rmt_clock_div(mp_obj_t self_in) {
+    esp32_rmt_obj_t *self = MP_OBJ_TO_PTR(self_in);
+    return mp_obj_new_int(self->clock_div);
+}
+STATIC MP_DEFINE_CONST_FUN_OBJ_1(esp32_rmt_clock_div_obj, esp32_rmt_clock_div);
+
+// Query whether the channel has finished sending pulses. Takes an optional
+// timeout (in ticks of the 80MHz clock), returning true if the pulse stream has
+// completed or false if they are still transmitting (or timeout is reached).
+STATIC mp_obj_t esp32_rmt_wait_done(size_t n_args, const mp_obj_t *pos_args, mp_map_t *kw_args) {
+    static const mp_arg_t allowed_args[] = {
+        { MP_QSTR_self,    MP_ARG_REQUIRED | MP_ARG_OBJ, {.u_obj = mp_const_none} },
+        { MP_QSTR_timeout, MP_ARG_KW_ONLY  | MP_ARG_INT, {.u_int = 0} },
+    };
+
+    mp_arg_val_t args[MP_ARRAY_SIZE(allowed_args)];
+    mp_arg_parse_all(n_args, pos_args, kw_args, MP_ARRAY_SIZE(allowed_args), allowed_args, args);
+
+    esp32_rmt_obj_t *self = MP_OBJ_TO_PTR(args[0].u_obj);
+
+    esp_err_t err = rmt_wait_tx_done(self->channel_id, args[1].u_int);
+    return err == ESP_OK ? mp_const_true : mp_const_false;
+}
+STATIC MP_DEFINE_CONST_FUN_OBJ_KW(esp32_rmt_wait_done_obj, 1, esp32_rmt_wait_done);
+
+STATIC mp_obj_t esp32_rmt_loop(mp_obj_t self_in, mp_obj_t loop) {
+    esp32_rmt_obj_t *self = MP_OBJ_TO_PTR(self_in);
+    check_esp_err(rmt_set_tx_loop_mode(self->channel_id, mp_obj_get_int(loop)));
+    return mp_const_none;
+}
+STATIC MP_DEFINE_CONST_FUN_OBJ_2(esp32_rmt_loop_obj, esp32_rmt_loop);
+
+STATIC mp_obj_t esp32_rmt_write_pulses(size_t n_args, const mp_obj_t *pos_args, mp_map_t *kw_args) {
+    static const mp_arg_t allowed_args[] = {
+        { MP_QSTR_self,   MP_ARG_REQUIRED | MP_ARG_OBJ, {.u_obj = mp_const_none} },
+        { MP_QSTR_pulses, MP_ARG_REQUIRED | MP_ARG_OBJ, {.u_obj = mp_const_none} },
+        { MP_QSTR_start,  MP_ARG_KW_ONLY  | MP_ARG_INT, {.u_int = 1} },
+    };
+
+    mp_arg_val_t args[MP_ARRAY_SIZE(allowed_args)];
+    mp_arg_parse_all(n_args, pos_args, kw_args, MP_ARRAY_SIZE(allowed_args), allowed_args, args);
+
+    esp32_rmt_obj_t *self = MP_OBJ_TO_PTR(args[0].u_obj);
+    mp_obj_t pulses = args[1].u_obj;
+    mp_uint_t start = args[2].u_int;
+
+    if (start < 0 || start > 1) {
+        mp_raise_ValueError("start must be 0 or 1");
+    }
+
+    size_t pulses_length = 0;
+    mp_obj_t* pulses_ptr = NULL;
+    mp_obj_get_array(pulses, &pulses_length, &pulses_ptr);
+
+    mp_uint_t num_items = (pulses_length / 2) + (pulses_length % 2);
+    if (num_items > self->num_items) {
+        self->items = (rmt_item32_t*)m_realloc(self->items, num_items * sizeof(rmt_item32_t *));
+        self->num_items = num_items;
+    }
+
+    for (mp_uint_t item_index = 0; item_index < num_items; item_index++) {
+        mp_uint_t pulse_index = item_index * 2;
+        self->items[item_index].duration0 = mp_obj_get_int(pulses_ptr[pulse_index++]);
+        self->items[item_index].level0 = start++; // Note that start _could_ wrap.
+        if (pulse_index < pulses_length) {
+            self->items[item_index].duration1 = mp_obj_get_int(pulses_ptr[pulse_index]);
+            self->items[item_index].level1 = start++;
+        }
+    }
+    check_esp_err(rmt_write_items(self->channel_id, self->items, num_items, false /* non-blocking */));
+
+    return mp_const_none;
+}
+STATIC MP_DEFINE_CONST_FUN_OBJ_KW(esp32_rmt_write_pulses_obj, 2, esp32_rmt_write_pulses);
+
+STATIC const mp_rom_map_elem_t esp32_rmt_locals_dict_table[] = {
+    { MP_ROM_QSTR(MP_QSTR___del__), MP_ROM_PTR(&esp32_rmt_deinit_obj) },
+    { MP_ROM_QSTR(MP_QSTR_deinit), MP_ROM_PTR(&esp32_rmt_deinit_obj) },
+    { MP_ROM_QSTR(MP_QSTR_source_freq), MP_ROM_PTR(&esp32_rmt_source_freq_obj) },
+    { MP_ROM_QSTR(MP_QSTR_clock_div), MP_ROM_PTR(&esp32_rmt_clock_div_obj) },
+    { MP_ROM_QSTR(MP_QSTR_wait_done), MP_ROM_PTR(&esp32_rmt_wait_done_obj) },
+    { MP_ROM_QSTR(MP_QSTR_loop), MP_ROM_PTR(&esp32_rmt_loop_obj) },
+    { MP_ROM_QSTR(MP_QSTR_write_pulses), MP_ROM_PTR(&esp32_rmt_write_pulses_obj) },
+};
+STATIC MP_DEFINE_CONST_DICT(esp32_rmt_locals_dict, esp32_rmt_locals_dict_table);
+
+const mp_obj_type_t esp32_rmt_type = {
+    { &mp_type_type },
+    .name = MP_QSTR_RMT,
+    .print = esp32_rmt_print,
+    .make_new = esp32_rmt_make_new,
+    .locals_dict = (mp_obj_dict_t*)&esp32_rmt_locals_dict,
+};

ports/esp32/main.c

@@ -47,6 +47,7 @@
 #include "py/nlr.h"
 #include "py/compile.h"
 #include "py/runtime.h"
+#include "py/persistentcode.h"
 #include "py/repl.h"
 #include "py/gc.h"
 #include "py/mphal.h"
@@ -173,12 +174,15 @@ void mbedtls_debug_set_threshold(int threshold) {
     (void)threshold;
 }
 
-void *esp_native_code_commit(void *buf, size_t len) {
+void *esp_native_code_commit(void *buf, size_t len, void *reloc) {
     len = (len + 3) & ~3;
     uint32_t *p = heap_caps_malloc(len, MALLOC_CAP_EXEC);
     if (p == NULL) {
         m_malloc_fail(len);
     }
+    if (reloc) {
+        mp_native_relocate(reloc, buf, (uintptr_t)p);
+    }
     memcpy(p, buf, len);
     return p;
 }

ports/esp32/modesp32.c

@@ -155,6 +155,7 @@ STATIC const mp_rom_map_elem_t esp32_module_globals_table[] = {
     { MP_ROM_QSTR(MP_QSTR_hall_sensor), MP_ROM_PTR(&esp32_hall_sensor_obj) },
 
     { MP_ROM_QSTR(MP_QSTR_Partition), MP_ROM_PTR(&esp32_partition_type) },
+    { MP_ROM_QSTR(MP_QSTR_RMT), MP_ROM_PTR(&esp32_rmt_type) },
     { MP_ROM_QSTR(MP_QSTR_ULP), MP_ROM_PTR(&esp32_ulp_type) },
 
     { MP_ROM_QSTR(MP_QSTR_WAKEUP_ALL_LOW), MP_ROM_PTR(&mp_const_false_obj) },

ports/esp32/modesp32.h

@@ -27,6 +27,7 @@
 #define RTC_IS_VALID_EXT_PIN(pin_id) ((1ll << (pin_id)) & RTC_VALID_EXT_PINS)
 
 extern const mp_obj_type_t esp32_partition_type;
+extern const mp_obj_type_t esp32_rmt_type;
 extern const mp_obj_type_t esp32_ulp_type;
 
 #endif // MICROPY_INCLUDED_ESP32_MODESP32_H

ports/esp32/mpconfigport.h

@@ -21,8 +21,6 @@
 // emitters
 #define MICROPY_PERSISTENT_CODE_LOAD        (1)
 #define MICROPY_EMIT_XTENSAWIN              (1)
-void *esp_native_code_commit(void*, size_t);
-#define MP_PLAT_COMMIT_EXEC(buf, len) esp_native_code_commit(buf, len)
 
 // compiler configuration
 #define MICROPY_COMP_MODULE_CONST           (1)
@@ -284,6 +282,8 @@ struct mp_bluetooth_nimble_root_pointers_t;
 #define BYTES_PER_WORD (4)
 #define MICROPY_MAKE_POINTER_CALLABLE(p) ((void*)((mp_uint_t)(p)))
 #define MP_PLAT_PRINT_STRN(str, len) mp_hal_stdout_tx_strn_cooked(str, len)
+void *esp_native_code_commit(void*, size_t, void*);
+#define MP_PLAT_COMMIT_EXEC(buf, len, reloc) esp_native_code_commit(buf, len, reloc)
 #define MP_SSIZE_MAX (0x7fffffff)
 
 // Note: these "critical nested" macros do not ensure cross-CPU exclusion,

ports/esp8266/boards/manifest.py

@@ -2,3 +2,4 @@ freeze('$(PORT_DIR)/modules')
 freeze('$(MPY_DIR)/tools', ('upip.py', 'upip_utarfile.py'))
 freeze('$(MPY_DIR)/drivers/dht', 'dht.py')
 freeze('$(MPY_DIR)/drivers/onewire')
+include('$(MPY_DIR)/extmod/webrepl/manifest.py')

ports/esp8266/modesp.c

@@ -28,6 +28,7 @@
 
 #include "py/gc.h"
 #include "py/runtime.h"
+#include "py/persistentcode.h"
 #include "py/mperrno.h"
 #include "py/mphal.h"
 #include "drivers/dht/dht.h"
@@ -282,7 +283,7 @@ void esp_native_code_init(void) {
     esp_native_code_erased = 0;
 }
 
-void *esp_native_code_commit(void *buf, size_t len) {
+void *esp_native_code_commit(void *buf, size_t len, void *reloc) {
     //printf("COMMIT(buf=%p, len=%u, start=%08x, cur=%08x, end=%08x, erased=%08x)\n", buf, len, esp_native_code_start, esp_native_code_cur, esp_native_code_end, esp_native_code_erased);
 
     len = (len + 3) & ~3;
@@ -294,6 +295,14 @@ void *esp_native_code_commit(void *buf, size_t len) {
     void *dest;
     if (esp_native_code_location == ESP_NATIVE_CODE_IRAM1) {
         dest = (void*)esp_native_code_cur;
+    } else {
+        dest = (void*)(FLASH_START + esp_native_code_cur);
+    }
+    if (reloc) {
+        mp_native_relocate(reloc, buf, (uintptr_t)dest);
+    }
+
+    if (esp_native_code_location == ESP_NATIVE_CODE_IRAM1) {
         memcpy(dest, buf, len);
     } else {
         SpiFlashOpResult res;
@@ -313,7 +322,6 @@ void *esp_native_code_commit(void *buf, size_t len) {
         if (res != SPI_FLASH_RESULT_OK) {
             mp_raise_OSError(res == SPI_FLASH_RESULT_TIMEOUT ? MP_ETIMEDOUT : MP_EIO);
         }
-        dest = (void*)(FLASH_START + esp_native_code_cur);
     }
 
     esp_native_code_cur += len;

ports/esp8266/mpconfigport.h

@@ -131,8 +131,8 @@ typedef uint32_t sys_prot_t; // for modlwip
 #include <sys/types.h>
 
 #define MP_PLAT_PRINT_STRN(str, len) mp_hal_stdout_tx_strn_cooked(str, len)
-void *esp_native_code_commit(void*, size_t);
-#define MP_PLAT_COMMIT_EXEC(buf, len) esp_native_code_commit(buf, len)
+void *esp_native_code_commit(void*, size_t, void*);
+#define MP_PLAT_COMMIT_EXEC(buf, len, reloc) esp_native_code_commit(buf, len, reloc)
 
 // printer for debugging output, goes to UART only
 extern const struct _mp_print_t mp_debug_print;

ports/nrf/boards/particle_xenon/mpconfigboard.h

@@ -38,6 +38,8 @@
 #define MICROPY_PY_MACHINE_TEMP     (1)
 #define MICROPY_PY_RANDOM_HW_RNG    (1)
 
+#define MICROPY_HW_USB_CDC          (1)
+
 #define MICROPY_HW_HAS_LED          (1)
 #define MICROPY_HW_LED_TRICOLOR     (1)
 #define MICROPY_HW_LED_PULLUP       (1)

ports/nrf/main.c

@@ -230,7 +230,7 @@ pin_init0();
 
 led_state(1, 0);
 
-#if MICROPY_VFS || MICROPY_MBFS
+#if MICROPY_VFS || MICROPY_MBFS || MICROPY_MODULE_FROZEN
     // run boot.py and main.py if they exist.
     pyexec_file_if_exists("boot.py");
     pyexec_file_if_exists("main.py");

ports/qemu-arm/Makefile

@@ -31,14 +31,11 @@ LDSCRIPT = mps2.ld
 SRC_BOARD_O = lib/utils/gchelper_m3.o
 endif
 
-CROSS_COMPILE = arm-none-eabi-
-
-TINYTEST = $(TOP)/lib/tinytest
+CROSS_COMPILE ?= arm-none-eabi-
 
 INC += -I.
 INC += -I$(TOP)
 INC += -I$(BUILD)
-INC += -I$(TINYTEST)
 
 CFLAGS += $(INC) -Wall -Wpointer-arith -Werror -std=gnu99 $(COPT) \
 	 -ffunction-sections -fdata-sections
@@ -71,6 +68,7 @@ SRC_RUN_C = \
 
 SRC_TEST_C = \
 	test_main.c \
+	lib/tinytest/tinytest.c \
 
 LIB_SRC_C += $(addprefix lib/,\
 	libc/string0.c \
@@ -108,7 +106,6 @@ ALL_OBJ_RUN = $(OBJ_COMMON) $(OBJ_RUN)
 
 OBJ_TEST =
 OBJ_TEST += $(addprefix $(BUILD)/, $(SRC_TEST_C:.c=.o))
-OBJ_TEST += $(BUILD)/tinytest.o
 
 ALL_OBJ_TEST = $(OBJ_COMMON) $(OBJ_TEST)
 

ports/qemu-arm/Makefile.test

@@ -13,8 +13,7 @@ $(BUILD)/genhdr/tests.h:
 	(cd $(TOP)/tests; ./run-tests --target=qemu-arm --write-exp)
 	$(Q)echo "Generating $@";(cd $(TOP)/tests; ../tools/tinytest-codegen.py) > $@
 
-$(BUILD)/tinytest.o:
-	$(Q)$(CC) $(CFLAGS) -DNO_FORKING -o $@ -c $(TINYTEST)/tinytest.c
+$(BUILD)/lib/tinytest/tinytest.o: CFLAGS += -DNO_FORKING
 
 $(BUILD)/firmware-test.elf: $(LDSCRIPT) $(ALL_OBJ_TEST)
 	$(Q)$(LD) $(LDFLAGS) -o $@ $(ALL_OBJ_TEST) $(LIBS)

ports/qemu-arm/test_main.c

@@ -12,9 +12,8 @@
 #include "py/gc.h"
 #include "py/mperrno.h"
 #include "lib/utils/gchelper.h"
-
-#include "tinytest.h"
-#include "tinytest_macros.h"
+#include "lib/tinytest/tinytest.h"
+#include "lib/tinytest/tinytest_macros.h"
 
 #define HEAP_SIZE (100 * 1024)