From 404fc1e939fd7d7de1f5dd2b6988ce3ed74971ea Mon Sep 17 00:00:00 2001 From: Scott Shawcroft Date: Wed, 7 Oct 2026 10:34:14 -0700 Subject: [PATCH 1/3] Add initial video pipeline support Start with `mipicsi` and a `videocore` object for storing a framebuffer. With this you can capture camera input into a buffer or directly into another DSI display's framebuffer. Capturing into the buffer and shown with displayio will be much slower than copying straight into the framebuffer. It is more flexible with size and position though. --- locale/circuitpython.pot | 64 +- ports/espressif/CMakeLists.txt | 17 +- ports/espressif/Makefile | 23 +- .../espressif_esp32p4_function_ev/sdkconfig | 24 + ports/espressif/common-hal/mipicsi/Camera.c | 539 ++++++++++++ ports/espressif/common-hal/mipicsi/Camera.h | 95 ++ ports/espressif/common-hal/mipicsi/__init__.c | 7 + ports/espressif/common-hal/mipidsi/Bus.c | 45 +- ports/espressif/common-hal/mipidsi/Bus.h | 9 + ports/espressif/common-hal/mipidsi/Display.c | 13 +- ports/espressif/mpconfigport.mk | 7 + ports/espressif/supervisor/port.c | 6 +- py/circuitpy_defns.mk | 12 + py/circuitpy_mpconfig.mk | 9 + shared-bindings/adafruit_pixelbuf/PixelBuf.c | 8 +- shared-bindings/busdisplay/BusDisplay.c | 8 + .../framebufferio/FramebufferDisplay.c | 8 + shared-bindings/mipicsi/BayerOrder.c | 50 ++ shared-bindings/mipicsi/Camera.c | 244 ++++++ shared-bindings/mipicsi/Camera.h | 73 ++ shared-bindings/mipicsi/PixelFormat.c | 66 ++ shared-bindings/mipicsi/__init__.c | 46 + shared-bindings/mipicsi/__init__.h | 33 + shared-bindings/vectorio/__init__.h | 5 + shared-bindings/videocore/VideoBuffer.c | 238 +++++ shared-bindings/videocore/VideoBuffer.h | 11 + shared-bindings/videocore/__init__.c | 91 ++ shared-bindings/videocore/__init__.h | 9 + shared-module/displayio/Group.c | 53 +- shared-module/displayio/Group.h | 3 +- shared-module/displayio/display_core.c | 7 + .../framebufferio/FramebufferDisplay.c | 22 +- shared-module/videocore/VideoBuffer.c | 828 ++++++++++++++++++ shared-module/videocore/VideoBuffer.h | 96 ++ shared-module/videocore/__init__.c | 56 ++ shared-module/videocore/__init__.h | 120 +++ 36 files changed, 2910 insertions(+), 35 deletions(-) create mode 100644 ports/espressif/common-hal/mipicsi/Camera.c create mode 100644 ports/espressif/common-hal/mipicsi/Camera.h create mode 100644 ports/espressif/common-hal/mipicsi/__init__.c create mode 100644 shared-bindings/mipicsi/BayerOrder.c create mode 100644 shared-bindings/mipicsi/Camera.c create mode 100644 shared-bindings/mipicsi/Camera.h create mode 100644 shared-bindings/mipicsi/PixelFormat.c create mode 100644 shared-bindings/mipicsi/__init__.c create mode 100644 shared-bindings/mipicsi/__init__.h create mode 100644 shared-bindings/videocore/VideoBuffer.c create mode 100644 shared-bindings/videocore/VideoBuffer.h create mode 100644 shared-bindings/videocore/__init__.c create mode 100644 shared-bindings/videocore/__init__.h create mode 100644 shared-module/videocore/VideoBuffer.c create mode 100644 shared-module/videocore/VideoBuffer.h create mode 100644 shared-module/videocore/__init__.c create mode 100644 shared-module/videocore/__init__.h diff --git a/locale/circuitpython.pot b/locale/circuitpython.pot index c03d1bd0ae4..97b798f7f5f 100644 --- a/locale/circuitpython.pot +++ b/locale/circuitpython.pot @@ -974,7 +974,6 @@ msgstr "" #: ports/atmel-samd/common-hal/pulseio/PulseIn.c #: ports/atmel-samd/common-hal/rotaryio/IncrementalEncoder.c #: ports/cxd56/common-hal/pulseio/PulseIn.c -#: ports/espressif/common-hal/mii/RMII.c #: ports/raspberrypi/common-hal/picodvi/Framebuffer_RP2350.c #: ports/zephyr-cp/common-hal/busio/I2C.c #: ports/zephyr-cp/common-hal/busio/SPI.c @@ -1182,6 +1181,7 @@ msgid "Another send is already active" msgstr "" #: ports/atmel-samd/common-hal/sdioio/SDCard.c +#: ports/espressif/common-hal/mii/Ethernet.c #: ports/espressif/common-hal/qspibus/QSPIBus.c shared-bindings/emmcio/EMMC.c #: shared-bindings/socketpool/SocketPool.c msgid "%q failure: %d" @@ -1219,6 +1219,7 @@ msgstr "" #: shared-bindings/busdisplay/BusDisplay.c #: shared-bindings/framebufferio/FramebufferDisplay.c #: shared-bindings/struct/__init__.c shared-module/struct/__init__.c +#: shared-module/videocore/VideoBuffer.c msgid "Buffer too small" msgstr "" @@ -1517,7 +1518,8 @@ msgid "%q must be 8, 16, 24, or 32" msgstr "" #: ports/espressif/common-hal/audiobusio/__init__.c -#: ports/espressif/common-hal/audioi2sin/I2SIn.c shared-bindings/emmcio/EMMC.c +#: ports/espressif/common-hal/audioi2sin/I2SIn.c +#: ports/espressif/common-hal/mii/Ethernet.c shared-bindings/emmcio/EMMC.c msgid "Peripheral in use" msgstr "" @@ -1563,6 +1565,8 @@ msgid "Can't construct AudioOut because continuous channel already open" msgstr "" #: ports/espressif/common-hal/audioio/AudioOut.c +#: ports/espressif/common-hal/mipicsi/Camera.c +#: shared-module/videocore/VideoBuffer.c msgid "already playing" msgstr "" @@ -1751,6 +1755,18 @@ msgstr "" msgid "Address range not allowed" msgstr "" +#: ports/espressif/common-hal/mipicsi/Camera.c +msgid "pixel_format must be RAW8, RAW10 or RAW12 when output_format differs" +msgstr "" + +#: ports/espressif/common-hal/mipicsi/Camera.c +msgid "output_format must be RGB565 or RGB888" +msgstr "" + +#: ports/espressif/common-hal/mipicsi/Camera.c +msgid "ISP conversion not supported on this chip" +msgstr "" + #: ports/espressif/common-hal/nvm/ByteArray.c msgid "NVS Error" msgstr "" @@ -4771,6 +4787,50 @@ msgstr "" msgid "Polygon needs at least 3 points" msgstr "" +#: shared-module/videocore/VideoBuffer.c +msgid "framebuffer went away" +msgstr "" + +#: shared-module/videocore/VideoBuffer.c +msgid "video doesn't fit in framebuffer" +msgstr "" + +#: shared-module/videocore/VideoBuffer.c +msgid "framebuffer missing required functions" +msgstr "" + +#: shared-module/videocore/VideoBuffer.c +msgid "expected FrameBuffer or %q" +msgstr "" + +#: shared-module/videocore/VideoBuffer.c +msgid "video format must be byte aligned" +msgstr "" + +#: shared-module/videocore/VideoBuffer.c +msgid "video format doesn't match framebuffer" +msgstr "" + +#: shared-module/videocore/VideoBuffer.c +msgid "video source rejected the destination" +msgstr "" + +#: shared-module/videocore/VideoBuffer.c +msgid "video in a framebuffer must be at x == 0" +msgstr "" + +#: shared-module/videocore/VideoBuffer.c +msgid "video size changed since play; stop and play again" +msgstr "" + +#: shared-module/videocore/VideoBuffer.c +msgid "VideoBuffer does not support transforms" +msgstr "" + +#: shared-module/videocore/VideoBuffer.c +msgid "VideoBuffer is playing into a different framebuffer" +msgstr "" + #: supervisor/shared/bluetooth/bluetooth.c msgid "Reconnecting" msgstr "" diff --git a/ports/espressif/CMakeLists.txt b/ports/espressif/CMakeLists.txt index 608d4f4c7b3..979afdad548 100644 --- a/ports/espressif/CMakeLists.txt +++ b/ports/espressif/CMakeLists.txt @@ -5,7 +5,22 @@ cmake_minimum_required(VERSION 3.16) set(ENV{IDF_PATH} ${CMAKE_SOURCE_DIR}/esp-idf) # The component list here determines what options we get in menuconfig and what the ninja file can build. -set(COMPONENTS bt driver esp_driver_dac esp_driver_gpio esp_driver_gptimer esp_driver_i2c esp_driver_i2s esp_driver_ledc esp_driver_pcnt esp_driver_rmt esp_driver_sdmmc esp_driver_spi esp_driver_touch_sens esp_driver_tsens esp_driver_uart esp_eth esp-tls esp_adc esp_adc_cal esp_event esp_netif esp_psram esp_security esp_wifi esptool_py freertos log lwip main mbedtls mdns pthread soc ulp wpa_supplicant esp-camera esp_lcd vfs esp_vfs_console esp_stdio sdmmc usb) +# Keep this split into separate set()/list() calls so additions merge easily. +# Core components +set(COMPONENTS bt driver esptool_py freertos log main mbedtls pthread soc) +# Peripheral drivers +list(APPEND COMPONENTS esp_driver_dac esp_driver_gpio esp_driver_gptimer esp_driver_i2c esp_driver_i2s esp_driver_ledc esp_driver_pcnt esp_driver_rmt esp_driver_sdmmc esp_driver_spi esp_driver_touch_sens esp_driver_tsens esp_driver_uart) +# Networking +list(APPEND COMPONENTS esp_eth esp-tls esp_adc esp_adc_cal esp_event esp_netif esp_psram esp_security esp_wifi lwip mdns wpa_supplicant) +# Misc +list(APPEND COMPONENTS ulp vfs esp_vfs_console esp_stdio sdmmc usb) +# Display and camera +list(APPEND COMPONENTS esp-camera esp_lcd) + +if(CIRCUITPY_MIPICSI) + list(APPEND COMPONENTS esp_driver_cam esp_hal_cam) +endif() + set(EXTRA_COMPONENT_DIRS "esp-protocols/components/mdns" "esp-camera") include($ENV{IDF_PATH}/tools/cmake/project.cmake) diff --git a/ports/espressif/Makefile b/ports/espressif/Makefile index 1d713916168..415eeb0238c 100644 --- a/ports/espressif/Makefile +++ b/ports/espressif/Makefile @@ -226,6 +226,13 @@ REGISTRATION_FUNCTIONS = \ -u esp_security_init_include_impl \ -u mbedtls_psa_crypto_init_include_impl +# Video DMA destinations come from the Python heap (user bytearrays), and +# the cache line is 64 bytes on the P4, so make GC blocks 64 bytes as well. +# supervisor/port.c aligns the PSRAM heap areas to the same value. +ifeq ($(IDF_TARGET),esp32p4) +CFLAGS += -DMICROPY_BYTES_PER_GC_BLOCK=64 +endif + # RISC-V picolibc ships optimized memcpy/memset/etc. that need forcing # (gated by CONFIG_LIBC_OPTIMIZED_MISALIGNED_ACCESS in IDF's esp_libc CMake). ifeq ($(IDF_TARGET_ARCH),riscv) @@ -504,9 +511,12 @@ endif CHIP_COMPONENTS = \ + esp_driver_cam \ + esp_driver_isp \ esp_driver_touch_sens \ esp_driver_tsens \ esp_driver_usb_serial_jtag \ + esp_hal_cam \ esp_hal_dma \ esp_hal_emac \ esp_hal_i2s \ @@ -736,6 +746,14 @@ CFLAGS += \ -isystem esp-idf/components/esp_lcd/dsi/include endif +ifneq ($(CIRCUITPY_MIPICSI),0) +CFLAGS += \ + -isystem esp-idf/components/esp_driver_cam/include \ + -isystem esp-idf/components/esp_driver_cam/csi/include \ + -isystem esp-idf/components/esp_driver_isp/include \ + -isystem esp-idf/components/esp_hal_cam/include +endif + ifneq ($(CIRCUITPY_ESPCAMERA),0) SRC_CAMERA := \ $(wildcard common-hal/espcamera/*.c) \ @@ -936,7 +954,7 @@ do-sdkconfig: $(BUILD)/esp-idf/config/sdkconfig.h QSTR_GLOBAL_REQUIREMENTS += $(BUILD)/esp-idf/config/sdkconfig.h $(BUILD)/esp-idf/config/sdkconfig.h: boards/$(BOARD)/sdkconfig boards/$(BOARD)/mpconfigboard.mk CMakeLists.txt | $(BUILD)/esp-idf $(STEPECHO) "Create $@" - $(Q)env IDF_PATH=$(IDF_PATH) IDF_COMPONENT_MANAGER=0 cmake -S . -B $(BUILD)/esp-idf -DSDKCONFIG=$(BUILD)/esp-idf/sdkconfig -DSDKCONFIG_DEFAULTS="$(SDKCONFIGS)" -DCMAKE_TOOLCHAIN_FILE=$(IDF_PATH)/tools/cmake/toolchain-$(IDF_TARGET).cmake -DIDF_TARGET=$(IDF_TARGET) -GNinja + $(Q)env IDF_PATH=$(IDF_PATH) IDF_COMPONENT_MANAGER=0 cmake -S . -B $(BUILD)/esp-idf -DSDKCONFIG=$(BUILD)/esp-idf/sdkconfig -DSDKCONFIG_DEFAULTS="$(SDKCONFIGS)" -DCMAKE_TOOLCHAIN_FILE=$(IDF_PATH)/tools/cmake/toolchain-$(IDF_TARGET).cmake -DIDF_TARGET=$(IDF_TARGET) -DCIRCUITPY_MIPICSI=$(CIRCUITPY_MIPICSI) -GNinja $(Q)$(PYTHON) tools/check-sdkconfig.py \ CIRCUITPY_DUALBANK=$(CIRCUITPY_DUALBANK) \ CIRCUITPY_STORAGE_EXTEND=$(CIRCUITPY_STORAGE_EXTEND) \ @@ -1097,6 +1115,9 @@ endif ifneq ($(CIRCUITPY_MIPIDSI),0) ESP_IDF_COMPONENTS_LINK += esp_lcd endif +ifneq ($(CIRCUITPY_MIPICSI),0) + ESP_IDF_COMPONENTS_LINK += esp_driver_cam esp_driver_isp esp_hal_cam +endif ifneq ($(CIRCUITPY_PARALLELDISPLAYBUS),0) ESP_IDF_COMPONENTS_LINK += esp_lcd endif diff --git a/ports/espressif/boards/espressif_esp32p4_function_ev/sdkconfig b/ports/espressif/boards/espressif_esp32p4_function_ev/sdkconfig index fb8986613ff..8ad98a7b081 100644 --- a/ports/espressif/boards/espressif_esp32p4_function_ev/sdkconfig +++ b/ports/espressif/boards/espressif_esp32p4_function_ev/sdkconfig @@ -1,3 +1,27 @@ +# +# Espressif IoT Development Framework Configuration +# +# Bootloader +# +CONFIG_BOOTLOADER_LOG_LEVEL_WARN=y +CONFIG_BOOTLOADER_LOG_LEVEL=2 +# end of Bootloader + +# This board ships with pre-v3 (eco2) silicon: link the 2nd stage bootloader +# into the lower memory layout used by revisions 0.x and 1.x. CONFIG_ESP32P4_SELECTS_REV_LESS_V3=y +# This board's silicon is rev v1.3; match the working BSP-demo build (REV_MIN +# v1.0) so IDF compiles the same errata/clock handling as the reference stack. CONFIG_ESP32P4_REV_MIN_100=y CONFIG_ETH_USE_ESP32_EMAC=y + +# ESP-STDIO: send the ESP-IDF console to the USB-Serial-JTAG port so bootloader +# logs are visible on the board's serial connector. +# +CONFIG_ESP_CONSOLE_USB_SERIAL_JTAG=y +CONFIG_ESP_CONSOLE_USB_SERIAL_JTAG_ENABLED=y +CONFIG_ESP_CONSOLE_UART_NUM=-1 +CONFIG_ESP_CONSOLE_ROM_SERIAL_PORT_NUM=6 +# end of ESP-STDIO + +# CONFIG_ESP_SYSTEM_MEMPROT is not set diff --git a/ports/espressif/common-hal/mipicsi/Camera.c b/ports/espressif/common-hal/mipicsi/Camera.c new file mode 100644 index 00000000000..be2e234ae3c --- /dev/null +++ b/ports/espressif/common-hal/mipicsi/Camera.c @@ -0,0 +1,539 @@ +// This file is part of the CircuitPython project: https://circuitpython.org +// +// SPDX-FileCopyrightText: Copyright (c) 2025 Jeff Epler for Adafruit Industries +// +// SPDX-License-Identifier: MIT + +#include "py/mperrno.h" +#include "py/mphal.h" +#include "py/nlr.h" +#include "py/runtime.h" + +#include "common-hal/mipicsi/Camera.h" +#include "shared-bindings/mipicsi/Camera.h" +#include "shared-bindings/mipicsi/__init__.h" +#include "bindings/espidf/__init__.h" + +#include "common-hal/mipidsi/Bus.h" + +#include +#include "esp_attr.h" +#include "esp_cache.h" +#include "esp_cam_ctlr.h" +#include "esp_private/esp_cache_private.h" +#include "esp_cam_ctlr_csi.h" +#include "hal/color_hal.h" + +// MIPI CSI receives data from a fixed, dedicated set of pins, so no pin +// configuration or claiming is done here. + +#define ISP_CLK_HZ (80 * 1000 * 1000) + +static cam_ctlr_color_t mipicsi_color_for(mipicsi_pixel_format_t format) { + switch (format) { + case MIPICSI_FORMAT_RAW8: + return CAM_CTLR_COLOR_RAW8; + case MIPICSI_FORMAT_RAW10: + return CAM_CTLR_COLOR_RAW10; + case MIPICSI_FORMAT_RAW12: + return CAM_CTLR_COLOR_RAW12; + case MIPICSI_FORMAT_RGB888: + return CAM_CTLR_COLOR_RGB888; + case MIPICSI_FORMAT_YUV422: + return CAM_CTLR_COLOR_YUV422_YUYV; + case MIPICSI_FORMAT_GRAYSCALE: + return CAM_CTLR_COLOR_GRAY8; + case MIPICSI_FORMAT_RGB565: + default: + return CAM_CTLR_COLOR_RGB565; + } +} + +static mp_int_t mipicsi_bits_per_pixel_for(mipicsi_pixel_format_t format) { + switch (format) { + case MIPICSI_FORMAT_RAW10: + return 10; + case MIPICSI_FORMAT_RAW12: + return 12; + case MIPICSI_FORMAT_RGB888: + return 24; + case MIPICSI_FORMAT_RGB565: + case MIPICSI_FORMAT_YUV422: + return 16; + case MIPICSI_FORMAT_RAW8: + case MIPICSI_FORMAT_GRAYSCALE: + default: + return 8; + } +} + +#if SOC_ISP_SUPPORTED +// The ISP color types share their FOURCC values with the CAM_CTLR color +// types, except for GRAYSCALE (no ISP grey mode) and YUV422 (the ISP input is +// UYVY, the CAM_CTLR YUYV). Only sensor RAW formats convert here. +static isp_color_t mipicsi_isp_color_for(mipicsi_pixel_format_t format) { + switch (format) { + case MIPICSI_FORMAT_RAW8: + return ISP_COLOR_RAW8; + case MIPICSI_FORMAT_RAW10: + return ISP_COLOR_RAW10; + case MIPICSI_FORMAT_RAW12: + return ISP_COLOR_RAW12; + case MIPICSI_FORMAT_RGB565: + return ISP_COLOR_RGB565; + case MIPICSI_FORMAT_RGB888: + return ISP_COLOR_RGB888; + case MIPICSI_FORMAT_YUV422: + case MIPICSI_FORMAT_GRAYSCALE: + default: + return 0; + } +} +#endif + +// Called by the driver (in ISR context) when the CSI host detects an error +// (PHY, packet, frame or CRC problems). Only installed for diagnostics: the +// counts are printed to the ROM console so they show up on the serial console. +static IRAM_ATTR bool mipicsi_on_error( + esp_cam_ctlr_handle_t handle, const esp_cam_ctlr_error_event_data_t *edata, void *user_data) { + mipicsi_camera_obj_t *self = user_data; + self->error_count++; + self->error_events |= edata->csi_host_err_evts; + if (self->error_count <= 3) { + esp_rom_printf("mipicsi: csi error #%lu events 0x%lx\n", + (unsigned long)self->error_count, (unsigned long)edata->csi_host_err_evts); + } + return false; +} + +// Called by the driver (in ISR context) to get the next buffer to DMA into. +static IRAM_ATTR bool mipicsi_on_get_new_trans( + esp_cam_ctlr_handle_t handle, esp_cam_ctlr_trans_t *trans, void *user_data) { + mipicsi_camera_obj_t *self = user_data; + + // Direct streaming: DMA the whole frame into the current destination. + // The videobuffer can change the destination between frames; the change + // is picked up here on the next frame. + // + // The controller only runs while a destination is set, so there is + // always somewhere to DMA into here; set_destination(NULL) stops the + // controller before clearing the destination. With the driver's backup + // buffer disabled, no buffer to hand over is unrecoverable (the driver + // asserts, or DMAs to address 0 in release builds), so there is no + // "give NULL" path. + trans->buffer = (void *)self->dest_base; + trans->buflen = self->buffer_len; + return false; +} + +// Called by the driver (in ISR context) when a frame has been written. +static IRAM_ATTR bool mipicsi_on_trans_finished( + esp_cam_ctlr_handle_t handle, esp_cam_ctlr_trans_t *trans, void *user_data) { + mipicsi_camera_obj_t *self = user_data; + + // Report where the frame actually landed: after a destination change + // (or a pause) the frame that was in flight lands at the previous one. + if (self->dest_base != NULL && self->dest_on_frame != NULL) { + self->dest_on_frame(self->dest_ctx, trans->buffer); + } + return false; +} + +void common_hal_mipicsi_camera_construct( + mipicsi_camera_obj_t *self, + mipicsi_pixel_format_t output_format) { + + self->width = 1; + self->height = 1; + self->pixel_format = output_format; + self->output_format = output_format; + self->dest_base = NULL; + self->dest_on_frame = NULL; + self->dest_ctx = NULL; + self->ctlr_started = false; + self->paused = false; + self->error_count = 0; + self->error_events = 0; + self->sensor = mp_const_none; + self->phy_ldo = NULL; + self->isp_proc = NULL; + self->spinlock = (portMUX_TYPE)portMUX_INITIALIZER_UNLOCKED; + // No receiver hardware yet: it is created in the videosource play step, + // where the sensor's current mode is known. +} + +// Videosource protocol play step: create the receiver hardware for the +// given (freshly read) sensor parameters and start the sensor streaming. +// When paused is true the CSI controller is left stopped, so the camera is +// active but no frames are DMAd until resume(). Raises when the camera is +// already streaming for someone else. Not a user facing method: it is driven +// by the videosource protocol (from VideoBuffer.play), so it may raise. +void common_hal_mipicsi_camera_videosource_play( + mipicsi_camera_obj_t *self, + const mipicsi_sensor_params_t *params, + bool paused) { + + if (self->handle != NULL) { + mp_raise_RuntimeError(MP_ERROR_TEXT("already playing")); + } + self->paused = paused; + + // Adopt what we need from the sensor's current mode; it may have + // changed since the Camera was made (sensors are free to reconfigure + // while they aren't being played). + self->width = params->width; + self->height = params->height; + self->bayer_order = params->bayer_order; + self->data_lanes = params->data_lanes; + self->lane_bit_rate_mbps = params->lane_bit_rate_mbps; + + const cam_ctlr_color_t color = mipicsi_color_for(self->pixel_format); + + #if SOC_ISP_SUPPORTED + if (self->output_format != self->pixel_format) { + // The sensor transmits in a different format than the one requested. + // On gen-1 P4 silicon the CSI receiver cannot convert, so route it + // through the ISP processor instead, mirroring the IDF + // esp_driver_cam csi test. + const isp_color_t isp_in = mipicsi_isp_color_for(self->pixel_format); + const isp_color_t isp_out = mipicsi_isp_color_for(self->output_format); + if (isp_in == 0 || (isp_in != ISP_COLOR_RAW8 && isp_in != ISP_COLOR_RAW10 && isp_in != ISP_COLOR_RAW12)) { + mp_raise_ValueError(MP_ERROR_TEXT("pixel_format must be RAW8, RAW10 or RAW12 when output_format differs")); + } + if (isp_out != ISP_COLOR_RGB565 && isp_out != ISP_COLOR_RGB888) { + mp_raise_ValueError(MP_ERROR_TEXT("output_format must be RGB565 or RGB888")); + } + esp_isp_processor_cfg_t isp_config = { + .clk_hz = ISP_CLK_HZ, + .input_data_source = ISP_INPUT_DATA_SOURCE_CSI, + .input_data_color_type = isp_in, + .output_data_color_type = isp_out, + .has_line_start_packet = false, + .has_line_end_packet = false, + .h_res = self->width, + .v_res = self->height, + .bayer_order = (color_raw_element_order_t)self->bayer_order, + }; + CHECK_ESP_RESULT(esp_isp_new_processor(&isp_config, &self->isp_proc)); + } + #else + if (self->output_format != self->pixel_format) { + mp_raise_NotImplementedError(MP_ERROR_TEXT("ISP conversion not supported on this chip")); + } + #endif + + // Power the MIPI D-PHY. The CSI and DSI PHYs share LDO channel 3; the + // shared reference count (see mipidsi_phy_ldo_acquire) keeps the rail up + // while either the camera or a mipidsi.Bus is alive. The sensor's I2C + // control interface is on a different rail and works without this. + self->phy_ldo = mipidsi_phy_ldo_acquire(); + + // The received images are not converted by the CSI bridge (gen-1 P4 + // silicon can't convert RAW to RGB565), so the input and output color + // types are the same and bridge conversion is bypassed. The ISP converts + // to the output format instead, so tell the CSI driver the memory-side + // frame is the ISP output depth. Without this it sizes the transfer and + // buffer from the 10 bpp RAW input (384,000 bytes for 1024x300) instead of + // the 16 bpp RGB565 output (614,400 bytes), which lets the DMA run past + // the destination. bits_per_pixel also makes the driver bypass bridge + // conversion; keep its default CSI packet data-type range. + esp_cam_ctlr_csi_config_t csi_config = { + .ctlr_id = 0, + .h_res = self->width, + .v_res = self->height, + .data_lane_num = self->data_lanes, + .lane_bit_rate_mbps = self->lane_bit_rate_mbps, + .input_data_color_type = color, + .output_data_color_type = color, + .data_type = { + .bits_per_pixel = mipicsi_bits_per_pixel_for(self->output_format), + .data_type_min = 0x12, + .data_type_max = 0x2f, + }, + // One transaction queue item; frames go straight to the current + // destination. The controller is only started while a destination + // is set, so the driver's backup buffer is never needed and its + // frame-sized allocation is skipped. + .queue_items = 1, + .bk_buffer_dis = true, + }; + + esp_err_t result = esp_cam_new_csi_ctlr(&csi_config, &self->handle); + if (result != ESP_OK) { + mipidsi_phy_ldo_release(); + self->phy_ldo = NULL; + raise_esp_error(result); + } + + // Exact frame size. Destinations are written directly (no cache-line + // padding is ever DMA'd past the last row), so they only need to be wide + // enough to hold the whole frame; the DSI framebuffer always is. When + // the ISP is in the pipeline, the CSI receiver hands its converted + // output to us. + self->buffer_len = ((size_t)self->width * self->height * mipicsi_bits_per_pixel_for(self->output_format) + 7) / 8; + + esp_cam_ctlr_evt_cbs_t cbs = { + .on_get_new_trans = mipicsi_on_get_new_trans, + .on_trans_finished = mipicsi_on_trans_finished, + .on_error = mipicsi_on_error, + }; + result = esp_cam_ctlr_register_event_callbacks(self->handle, &cbs, self); + if (result != ESP_OK) { + common_hal_mipicsi_camera_deinit(self); + self->sensor = mp_const_none; + raise_esp_error(result); + } + + result = esp_cam_ctlr_enable(self->handle); + if (result != ESP_OK) { + common_hal_mipicsi_camera_deinit(self); + self->sensor = mp_const_none; + raise_esp_error(result); + } + #if SOC_ISP_SUPPORTED + if (self->isp_proc) { + result = esp_isp_enable(self->isp_proc); + if (result != ESP_OK) { + common_hal_mipicsi_camera_deinit(self); + self->sensor = mp_const_none; + raise_esp_error(result); + } + } + #endif + + // Start the sensor streaming so frames exist to capture once a + // destination is set. Sensors' _start() is expected to be idempotent. + // The camera never calls the sensor's stop(): when a stream ends (or on + // reload) only the CSI receiver and ISP are torn down; the sensor stops + // through its own finaliser (deinit/__del__). + if (self->sensor != mp_const_none) { + nlr_buf_t nlr; + if (nlr_push(&nlr) == 0) { + mp_obj_t start_method[2]; + mp_load_method(self->sensor, MP_QSTR__start, start_method); + mp_call_method_n_kw(0, 0, start_method); + nlr_pop(); + } else { + common_hal_mipicsi_camera_deinit(self); + self->sensor = mp_const_none; + nlr_raise(nlr.ret_val); + } + } + + // Don't start the controller yet: it only runs while a destination is + // set (see common_hal_mipicsi_camera_set_destination), so no frame is + // ever captured without a place to put it. +} + +// Tear down the receiver hardware quietly: this runs from Python (a +// VideoBuffer stopped) and from background contexts (destination dropped on +// reload), so it must not raise. The sensor is left alone: its own finaliser +// stops it. Only the CSI receiver and ISP turn off. +static void mipicsi_camera_teardown(mipicsi_camera_obj_t *self) { + // Always stop, even when paused: a pause's stop finishes in the + // background, and it must be complete before the controller is disabled. + // esp_cam_ctlr_stop waits for it (and is a no-op once stopped). + if (self->handle != NULL) { + esp_cam_ctlr_stop(self->handle); + } + self->ctlr_started = false; + portENTER_CRITICAL(&self->spinlock); + self->dest_base = NULL; + self->dest_on_frame = NULL; + self->dest_ctx = NULL; + self->paused = false; + portEXIT_CRITICAL(&self->spinlock); + + nlr_buf_t nlr; + if (nlr_push(&nlr) == 0) { + if (self->handle != NULL) { + esp_cam_ctlr_disable(self->handle); + esp_cam_ctlr_del(self->handle); + self->handle = NULL; + } + #if SOC_ISP_SUPPORTED + if (self->isp_proc) { + esp_isp_disable(self->isp_proc); + esp_isp_del_processor(self->isp_proc); + self->isp_proc = NULL; + } + #endif + mipidsi_phy_ldo_release(); + self->phy_ldo = NULL; + nlr_pop(); + } else { + // Nothing to do about a failed teardown, not even a partially + // released one; swallow the exception and continue. + } +} + +void common_hal_mipicsi_camera_set_sensor(mipicsi_camera_obj_t *self, mp_obj_t sensor) { + self->sensor = sensor; +} + +void common_hal_mipicsi_camera_deinit(mipicsi_camera_obj_t *self) { + if (self->handle == NULL) { + // Not currently playing: nothing to tear down. + return; + } + + // Only turn off the CSI receiver and ISP here (including from background + // contexts such as reload): the sensor stops itself through its own + // finaliser (deinit/__del__). + mipicsi_camera_teardown(self); + self->sensor = mp_const_none; +} +// Videosource protocol stop step: tear down the receiver hardware and stop +// the sensor so its settings can be changed. Not a user facing method: it is +// driven by the videosource protocol (from VideoBuffer.stop), so it may +// raise; exceptions from the sensor's stop() propagate. +void common_hal_mipicsi_camera_videosource_stop(mipicsi_camera_obj_t *self) { + if (self->handle == NULL) { + return; // Not playing. + } + + // The receiver first: after the CSI controller stops, no more frames are + // captured, whatever the sensor does. + mipicsi_camera_teardown(self); + + // Stop the sensor (e.g. the sc2336 stops streaming and unlocks its + // mode). Sensors' _stop() is expected to be idempotent. + if (self->sensor != mp_const_none) { + mp_obj_t stop_method[2]; + mp_load_method(self->sensor, MP_QSTR__stop, stop_method); + mp_call_method_n_kw(0, 0, stop_method); + } +} + +// Videosource protocol pause step: stop the CSI controller's DMA without +// tearing down the receiver, ISP or sensor. The sensor keeps streaming, so the +// camera stays active and resume is quick; no frames are written to memory +// while stopped. Best effort: this runs from displayio group operations, so it +// must not raise. Called when the target is hidden. The destination is kept so +// resume() can use the latest one. +void common_hal_mipicsi_camera_videosource_pause(mipicsi_camera_obj_t *self) { + if (self->handle == NULL) { + return; // Not playing. + } + + // Keep the ISP enabled too: disabling it would also stop its AE/AWB/etc. + // processing and require a pipeline restart. Stopping the CSI controller + // disables the CSI bridge and its GDMA channel, which is enough to keep + // frames out of memory. + // + // Don't wait for it: the controller stops at the end of the frame being + // captured (stopping mid-frame would leave a partial frame behind), up to + // a frame time from now. That frame still lands at the current + // destination and is reported through on_frame, so the target can + // repaint over it. + if (self->ctlr_started) { + // Only fails when not started, which ctlr_started rules out. + (void)esp_cam_ctlr_csi_stop_at_frame_end(self->handle); + self->ctlr_started = false; + } + self->paused = true; +} + +// Videosource protocol resume step: start the CSI controller again at the most +// recent destination. The receiver hardware, ISP and sensor were never torn +// down. Called from Python contexts, so it may raise; paused stays set if the +// controller can't start. +void common_hal_mipicsi_camera_videosource_resume(mipicsi_camera_obj_t *self) { + if (self->handle == NULL) { + return; // Not playing. + } + + if (!self->ctlr_started && self->dest_base != NULL) { + // Starting while the pause's stop is still pending cancels it; the + // frame in flight still lands at the destination from pause time and + // on_frame reports it there. + CHECK_ESP_RESULT(esp_cam_ctlr_start(self->handle)); + self->ctlr_started = true; + } + self->paused = false; +} + +bool common_hal_mipicsi_camera_deinited(mipicsi_camera_obj_t *self) { + return self->handle == NULL; +} + +bool common_hal_mipicsi_camera_set_destination( + mipicsi_camera_obj_t *self, + uint8_t *base, + size_t stride, + void (*on_frame)(void *ctx, const void *frame), + void *ctx) { + const size_t row_bytes = ((size_t)self->width * mipicsi_bits_per_pixel_for(self->output_format) + 7) / 8; + const size_t frame_bytes = (size_t)self->height * row_bytes; + + // There is no stop value on the destination: stopping is the videosource + // stop step's job. The protocol never passes NULL, + // so this guard is only about internal misuse. + assert(base != NULL); + + // DMA destinations must be cache-line aligned: the driver syncs the + // buffer on every frame with esp_cache_msync, which rejects anything + // else. PSRAM heap areas (and with 64-byte GC blocks, every GC + // allocation in them, e.g. bytearrays) are allocated 64-byte aligned on + // this chip, so this is a safety net. + size_t cache_line = 0; + esp_cache_get_alignment(MALLOC_CAP_SPIRAM, &cache_line); + if (cache_line == 0) { + cache_line = 64; + } + if ((uintptr_t)base % cache_line != 0) { + return false; + } + + // The destination is assumed to hold the frame rows with no stride + // padding: the CSI link-list DMA only does contiguous transfers in this + // IDF. The videobuffer validates the fit before getting here; the whole + // (padded) frame must lie within the destination: rows can't wrap past + // its end, so there is no cropping or overhang. + if (stride != row_bytes) { + return false; + } + + // Size handed to the driver: the frame, cache-line padded at the end so + // the driver's per-frame sync is happy. + const size_t buffer_len = (frame_bytes + cache_line - 1) / cache_line * cache_line; + + portENTER_CRITICAL(&self->spinlock); + self->dest_base = base; + self->dest_on_frame = on_frame; + self->dest_ctx = ctx; + self->buffer_len = buffer_len; + portEXIT_CRITICAL(&self->spinlock); + + // The sensor streaming and the receiver hardware exist from play(); + // here only the controller runs or doesn't. While paused, just remember + // the new destination: resume() starts the controller there. + if (!self->paused && !self->ctlr_started) { + esp_err_t result = esp_cam_ctlr_start(self->handle); + if (result != ESP_OK) { + return false; + } + self->ctlr_started = true; + } + return true; +} + +mipicsi_pixel_format_t common_hal_mipicsi_camera_get_pixel_format(mipicsi_camera_obj_t *self) { + return self->pixel_format; +} + +mipicsi_pixel_format_t common_hal_mipicsi_camera_get_output_format(mipicsi_camera_obj_t *self) { + return self->output_format; +} + +mp_int_t common_hal_mipicsi_camera_get_width(mipicsi_camera_obj_t *self) { + return self->width; +} + +mp_int_t common_hal_mipicsi_camera_get_height(mipicsi_camera_obj_t *self) { + return self->height; +} + +mp_int_t common_hal_mipicsi_camera_get_bits_per_pixel(mipicsi_camera_obj_t *self) { + return mipicsi_bits_per_pixel_for(self->output_format); +} diff --git a/ports/espressif/common-hal/mipicsi/Camera.h b/ports/espressif/common-hal/mipicsi/Camera.h new file mode 100644 index 00000000000..86cd86bcc2f --- /dev/null +++ b/ports/espressif/common-hal/mipicsi/Camera.h @@ -0,0 +1,95 @@ +// This file is part of the CircuitPython project: https://circuitpython.org +// +// SPDX-FileCopyrightText: Copyright (c) 2025 Jeff Epler for Adafruit Industries +// +// SPDX-License-Identifier: MIT + +#pragma once + +#include + +#include "py/obj.h" + +#include "freertos/FreeRTOS.h" +#include "freertos/portmacro.h" + +#include "esp_cam_ctlr.h" +#include "esp_ldo_regulator.h" + +#if SOC_ISP_SUPPORTED +#include "driver/isp.h" +#endif + +#include "shared-bindings/mipicsi/__init__.h" + +typedef struct mipicsi_camera_obj mipicsi_camera_obj_t; + +// Sensor parameters used to build the receiver. Read off the sensor object +// when a stream starts (the sensor is free to reconfigure between streams). +typedef struct { + mp_int_t width; + mp_int_t height; + mp_int_t bayer_order; + mp_int_t data_lanes; + mp_int_t lane_bit_rate_mbps; +} mipicsi_sensor_params_t; + +struct mipicsi_camera_obj { + mp_obj_base_t base; + esp_cam_ctlr_handle_t handle; + + // Power for the MIPI D-PHY, shared with the DSI transmitter. Acquired + // when a stream starts and released when it (or the camera) ends. + esp_ldo_channel_handle_t phy_ldo; + + // Camera sensor configured to transmit to this receiver, or None. + mp_obj_t sensor; + + // ISP processor used to convert sensor frames to output_format, only + // present when the CSI receiver itself cannot convert (e.g. gen-1 P4). + mipicsi_pixel_format_t output_format; + #if SOC_ISP_SUPPORTED + isp_proc_handle_t isp_proc; + #endif + + // The receiver hardware (LDO, ISP, CSI controller) is created when a + // stream starts (see play) and torn down when it ends. The sensor's + // parameters (width, height, data_lanes, lane_bit_rate_mbps, bayer_order) + // come from the sensor at play time as well; pixel_format only when it + // wasn't overridden at construction. + // Streaming destination, set by videocore.VideoBuffer. While base is + // NULL the CSI controller is stopped: no capture buffers are used at + // all. The destination is assumed to be a whole frame: DMA'd straight + // into base, stride equal to the row size. on_frame(ctx, frame) is called + // from ISR context each time a frame lands, with the base it landed at + // (a frame in flight when base changes lands at the previous base). + // The DMA destination: set_destination requires it to be cache-line + // aligned (the driver's per-frame cache sync needs that) and reports a + // rejected destination with a false return, rather than shifting frames + // elsewhere. The frame must be contiguous. + volatile void *dest_base; + void (*dest_on_frame)(void *ctx, const void *frame); + void *dest_ctx; + bool ctlr_started; + // True between pause and resume: the receiver hardware, ISP and sensor stay + // alive, but the CSI controller is stopped so no frame is DMAd. A + // destination can still be changed while paused; resume starts the + // controller again at the latest one. + bool paused; + + // Diagnostics from the CSI host error interrupt. + volatile uint32_t error_count; + volatile uint32_t error_events; + + portMUX_TYPE spinlock; + + mp_int_t width; + mp_int_t height; + mipicsi_pixel_format_t pixel_format; + mp_int_t data_lanes; + mp_int_t lane_bit_rate_mbps; + // Bayer element order of the sensor's RAW output, only used when the ISP + // converts from a RAW format. + mp_int_t bayer_order; + size_t buffer_len; // Length handed to the driver, includes cache-alignment padding. +}; diff --git a/ports/espressif/common-hal/mipicsi/__init__.c b/ports/espressif/common-hal/mipicsi/__init__.c new file mode 100644 index 00000000000..816fea93ae9 --- /dev/null +++ b/ports/espressif/common-hal/mipicsi/__init__.c @@ -0,0 +1,7 @@ +// This file is part of the CircuitPython project: https://circuitpython.org +// +// SPDX-FileCopyrightText: Copyright (c) 2025 Jeff Epler for Adafruit Industries +// +// SPDX-License-Identifier: MIT + +#include "shared-bindings/mipicsi/__init__.h" diff --git a/ports/espressif/common-hal/mipidsi/Bus.c b/ports/espressif/common-hal/mipidsi/Bus.c index dbb559ea30f..dab781f8cc9 100644 --- a/ports/espressif/common-hal/mipidsi/Bus.c +++ b/ports/espressif/common-hal/mipidsi/Bus.c @@ -9,6 +9,39 @@ #include "py/runtime.h" #include +// Shared MIPI PHY LDO rail. DSI buses and CSI receivers both need it; it is +// acquired once and released only when the last user goes away. +static esp_ldo_channel_handle_t mipidsi_phy_ldo = NULL; +static uint8_t mipidsi_phy_ldo_users = 0; + +esp_ldo_channel_handle_t mipidsi_phy_ldo_acquire(void) { + if (mipidsi_phy_ldo_users == 0) { + esp_ldo_channel_handle_t ldo = NULL; + esp_ldo_channel_config_t ldo_config = { + .chan_id = 3, + .voltage_mv = 2500, + // Fixed-voltage (non-adjustable) acquire so both users share the + // rail. An adjustable acquire would block the other user. + .flags.adjustable = false, + }; + ESP_ERROR_CHECK(esp_ldo_acquire_channel(&ldo_config, &ldo)); + mipidsi_phy_ldo = ldo; + } + mipidsi_phy_ldo_users++; + return mipidsi_phy_ldo; +} + +void mipidsi_phy_ldo_release(void) { + if (mipidsi_phy_ldo_users == 0) { + return; + } + mipidsi_phy_ldo_users--; + if (mipidsi_phy_ldo_users == 0 && mipidsi_phy_ldo != NULL) { + esp_ldo_release_channel(mipidsi_phy_ldo); + mipidsi_phy_ldo = NULL; + } +} + void common_hal_mipidsi_bus_construct(mipidsi_bus_obj_t *self, mp_uint_t frequency, uint8_t num_lanes) { self->frequency = frequency; self->num_data_lanes = num_lanes; @@ -18,12 +51,7 @@ void common_hal_mipidsi_bus_construct(mipidsi_bus_obj_t *self, mp_uint_t frequen mp_raise_ValueError_varg(MP_ERROR_TEXT("%q in use"), MP_QSTR_mipidsi); } - esp_ldo_channel_handle_t ldo_mipi_phy = NULL; - esp_ldo_channel_config_t ldo_mipi_phy_config = { - .chan_id = 3, - .voltage_mv = 2500, - }; - ESP_ERROR_CHECK(esp_ldo_acquire_channel(&ldo_mipi_phy_config, &ldo_mipi_phy)); + self->phy_ldo = mipidsi_phy_ldo_acquire(); // Create the DSI bus esp_lcd_dsi_bus_config_t bus_config = { @@ -50,6 +78,11 @@ void common_hal_mipidsi_bus_deinit(mipidsi_bus_obj_t *self) { self->bus_handle = NULL; } + // Release our hold on the shared MIPI PHY LDO; it stays powered as long + // as a CSI receiver (or other user) still holds it. + mipidsi_phy_ldo_release(); + self->phy_ldo = NULL; + self->frequency = 0; self->num_data_lanes = 0; } diff --git a/ports/espressif/common-hal/mipidsi/Bus.h b/ports/espressif/common-hal/mipidsi/Bus.h index b6257fd7223..1920275d430 100644 --- a/ports/espressif/common-hal/mipidsi/Bus.h +++ b/ports/espressif/common-hal/mipidsi/Bus.h @@ -8,14 +8,23 @@ #include "py/obj.h" #include +#include typedef struct { mp_obj_base_t base; mp_uint_t frequency; esp_lcd_dsi_bus_handle_t bus_handle; + esp_ldo_channel_handle_t phy_ldo; uint8_t num_data_lanes; uint8_t use_count; // Up to 4 displays } mipidsi_bus_obj_t; void mipidsi_bus_increment_use_count(mipidsi_bus_obj_t *self); void mipidsi_bus_decrement_use_count(mipidsi_bus_obj_t *self); + +// Acquire the shared MIPI PHY LDO rail (LDO channel 3, 2.5 V). DSI and CSI +// users share the rail; a shared reference count keeps it powered until the +// last user releases it, so the CSI receiver can stay active while a DSI bus +// comes and goes (and vice versa). +esp_ldo_channel_handle_t mipidsi_phy_ldo_acquire(void); +void mipidsi_phy_ldo_release(void); diff --git a/ports/espressif/common-hal/mipidsi/Display.c b/ports/espressif/common-hal/mipidsi/Display.c index ea204924fe6..9ebaec1bd35 100644 --- a/ports/espressif/common-hal/mipidsi/Display.c +++ b/ports/espressif/common-hal/mipidsi/Display.c @@ -13,6 +13,7 @@ #include "bindings/espidf/__init__.h" #include #include +#include #include "py/runtime.h" // Cache write-back function (should be from rom/cache.h but it's not always available) @@ -49,6 +50,7 @@ void common_hal_mipidsi_display_construct(mipidsi_display_obj_t *self, self->framebuffer = NULL; self->dbi_io_handle = NULL; self->dpi_panel_handle = NULL; + self->backlight_inout.base.type = &mp_type_NoneType; // Create the DBI interface for sending commands esp_lcd_dbi_io_config_t dbi_config = { @@ -111,6 +113,10 @@ void common_hal_mipidsi_display_construct(mipidsi_display_obj_t *self, self->framebuffer = (uint8_t *)fb; self->framebuffer_size = width * height * (color_depth / 8); + // The panel's reset line is not managed here, matching the other displayio + // display drivers: callers pulse/hold reset themselves (e.g. with + // digitalio) when a board needs it. + // Send initialization sequence (format matches busdisplay) #define DELAY 0x80 uint32_t i = 0; @@ -141,8 +147,12 @@ void common_hal_mipidsi_display_construct(mipidsi_display_obj_t *self, CHECK_ESP_RESULT(ret); } + // Send display-ON (0x29) after the DPI/video stream is running, matching + // the reference BSP flow order. + esp_lcd_panel_io_tx_param(self->dbi_io_handle, 0x29, NULL, 0); + common_hal_time_delay_ms(20); + // Setup backlight PWM if pin is provided - self->backlight_inout.base.type = &mp_type_NoneType; if (backlight_pin != NULL && common_hal_mcu_pin_is_free(backlight_pin)) { #if (CIRCUITPY_PWMIO) pwmout_result_t result = common_hal_pwmio_pwmout_construct(&self->backlight_pwm, backlight_pin, 0, 50000, false); @@ -192,6 +202,7 @@ void common_hal_mipidsi_display_deinit(mipidsi_display_obj_t *self) { common_hal_digitalio_digitalinout_deinit(&self->backlight_inout); } + // Delete the DPI panel if (self->dpi_panel_handle != NULL) { esp_lcd_panel_del(self->dpi_panel_handle); diff --git a/ports/espressif/mpconfigport.mk b/ports/espressif/mpconfigport.mk index 87ba43d1ebf..20c8cbf8b01 100644 --- a/ports/espressif/mpconfigport.mk +++ b/ports/espressif/mpconfigport.mk @@ -404,6 +404,13 @@ CIRCUITPY_ESPCAMERA = 0 # P4 has MIPI-DSI CIRCUITPY_MIPIDSI = 1 +# P4 has MIPI-CSI +CIRCUITPY_MIPICSI = 1 + +# P4 can play MIPI-CSI video into MIPI-DSI framebuffers +CIRCUITPY_VIDEOCORE = 1 + + #### esp32s31 ######################################################### else ifeq ($(IDF_TARGET),esp32s31) diff --git a/ports/espressif/supervisor/port.c b/ports/espressif/supervisor/port.c index 91fe8cf5a2e..81eb4714db6 100644 --- a/ports/espressif/supervisor/port.c +++ b/ports/espressif/supervisor/port.c @@ -343,9 +343,11 @@ void *port_malloc(size_t size, bool dma_capable) { } void *ptr = NULL; - // Try SPIRAM first if available. + // Try SPIRAM first if available. Aligned to the GC block size so heap + // areas (and with 64-byte GC blocks, every GC allocation in them) are + // cache-line aligned, which video DMA destinations require. #ifdef CONFIG_SPIRAM - ptr = heap_caps_malloc(size, MALLOC_CAP_8BIT | MALLOC_CAP_SPIRAM); + ptr = heap_caps_aligned_alloc(MICROPY_BYTES_PER_GC_BLOCK, size, MALLOC_CAP_8BIT | MALLOC_CAP_SPIRAM); #endif if (ptr == NULL) { ptr = heap_caps_malloc(size, MALLOC_CAP_8BIT); diff --git a/py/circuitpy_defns.mk b/py/circuitpy_defns.mk index ec84d3ec381..8a4b9c59a18 100755 --- a/py/circuitpy_defns.mk +++ b/py/circuitpy_defns.mk @@ -324,6 +324,9 @@ endif ifeq ($(CIRCUITPY_MICROCONTROLLER),1) SRC_PATTERNS += microcontroller/% endif +ifeq ($(CIRCUITPY_MIPICSI),1) +SRC_PATTERNS += mipicsi/% +endif ifeq ($(CIRCUITPY_MIPIDSI),1) SRC_PATTERNS += mipidsi/% endif @@ -492,6 +495,9 @@ endif ifeq ($(CIRCUITPY_VECTORIO),1) SRC_PATTERNS += vectorio/% endif +ifeq ($(CIRCUITPY_VIDEOCORE),1) +SRC_PATTERNS += videocore/% +endif ifeq ($(CIRCUITPY_WARNINGS),1) SRC_PATTERNS += warnings/% endif @@ -586,6 +592,8 @@ SRC_COMMON_HAL_ALL = \ mdns/__init__.c \ mdns/Server.c \ mdns/RemoteService.c \ + mipicsi/__init__.c \ + mipicsi/Camera.c \ mipidsi/Bus.c \ mipidsi/Display.c \ mipidsi/__init__.c \ @@ -690,6 +698,8 @@ $(filter $(SRC_PATTERNS), \ imagecapture/ParallelImageCapture.c \ locale/__init__.c \ math/__init__.c \ + mipicsi/BayerOrder.c \ + mipicsi/PixelFormat.c \ microcontroller/ResetReason.c \ microcontroller/RunMode.c \ msgpack/__init__.c \ @@ -895,6 +905,8 @@ SRC_SHARED_MODULE_ALL = \ vectorio/Rectangle.c \ vectorio/VectorShape.c \ vectorio/__init__.c \ + videocore/VideoBuffer.c \ + videocore/__init__.c \ warnings/__init__.c \ watchdog/__init__.c \ zlib/__init__.c \ diff --git a/py/circuitpy_mpconfig.mk b/py/circuitpy_mpconfig.mk index 0fc82d03704..1d8bedebbc5 100755 --- a/py/circuitpy_mpconfig.mk +++ b/py/circuitpy_mpconfig.mk @@ -469,6 +469,9 @@ CFLAGS += -DCIRCUITPY_MEMORYMONITOR=$(CIRCUITPY_MEMORYMONITOR) CIRCUITPY_MICROCONTROLLER ?= 1 CFLAGS += -DCIRCUITPY_MICROCONTROLLER=$(CIRCUITPY_MICROCONTROLLER) +CIRCUITPY_MIPICSI ?= 0 +CFLAGS += -DCIRCUITPY_MIPICSI=$(CIRCUITPY_MIPICSI) + CIRCUITPY_MIPIDSI ?= 0 CFLAGS += -DCIRCUITPY_MIPIDSI=$(CIRCUITPY_MIPIDSI) @@ -828,6 +831,12 @@ CFLAGS += -DCIRCUITPY_ULAB=$(CIRCUITPY_ULAB) # element-wise array arithmetic roughly 1.5x slower. CIRCUITPY_ULAB_OPTIMIZE_SIZE ?= 0 +# Video pipeline: playing video sources into framebuffers. Requires a port +# with a video source such as mipicsi and a framebuffer output such as +# mipidsi. +CIRCUITPY_VIDEOCORE ?= 0 +CFLAGS += -DCIRCUITPY_VIDEOCORE=$(CIRCUITPY_VIDEOCORE) + CIRCUITPY_WARNINGS ?= $(CIRCUITPY_FULL_BUILD) CFLAGS += -DCIRCUITPY_WARNINGS=$(CIRCUITPY_WARNINGS) diff --git a/shared-bindings/adafruit_pixelbuf/PixelBuf.c b/shared-bindings/adafruit_pixelbuf/PixelBuf.c index 4144f086b98..dd5e4d7b4ea 100644 --- a/shared-bindings/adafruit_pixelbuf/PixelBuf.c +++ b/shared-bindings/adafruit_pixelbuf/PixelBuf.c @@ -31,7 +31,13 @@ static MP_NORETURN void invalid_byteorder(void) { static void parse_byteorder(mp_obj_t byteorder_obj, pixelbuf_byteorder_details_t *parsed); //| class PixelBuf: -//| """A fast RGB[W] pixel buffer for LED and similar devices.""" +//| """A fast RGB[W] pixel buffer for LED and similar devices. +//| +//| Subclasses must define ``_transmit(self, buffer)``, which `show` calls +//| with a `bytearray` of the bytes to send: the header, the pixel +//| values, and the trailer. ``_transmit`` is internal, for PixelBuf's +//| use only; user code shows pixels through `show` and `auto_write`. +//| """ //| //| def __init__( //| self, diff --git a/shared-bindings/busdisplay/BusDisplay.c b/shared-bindings/busdisplay/BusDisplay.c index 0bf171a5d08..57fc1afa71c 100644 --- a/shared-bindings/busdisplay/BusDisplay.c +++ b/shared-bindings/busdisplay/BusDisplay.c @@ -16,6 +16,9 @@ #include "shared-bindings/displayio/Group.h" #include "shared-bindings/microcontroller/Pin.h" #include "shared-bindings/util.h" +#if CIRCUITPY_VIDEOCORE +#include "shared-module/videocore/__init__.h" +#endif #include "shared-module/displayio/__init__.h" //| import displayio @@ -408,6 +411,11 @@ static mp_obj_t busdisplay_busdisplay_obj_set_root_group(mp_obj_t self_in, mp_ob displayio_group_t *group = NULL; if (group_in != mp_const_none) { group = MP_OBJ_TO_PTR(native_group(group_in)); + #if CIRCUITPY_VIDEOCORE + // Validate that draw protocol layers (e.g. a VideoBuffer) fit the + // display they will render on. + videocore_validate_root_group(group_in, self_in); + #endif } common_hal_busdisplay_busdisplay_set_root_group(self, group); diff --git a/shared-bindings/framebufferio/FramebufferDisplay.c b/shared-bindings/framebufferio/FramebufferDisplay.c index 71ce18a1b4b..da7f18e4caf 100644 --- a/shared-bindings/framebufferio/FramebufferDisplay.c +++ b/shared-bindings/framebufferio/FramebufferDisplay.c @@ -17,6 +17,9 @@ #include "shared-bindings/displayio/Group.h" #include "shared-bindings/microcontroller/Pin.h" #include "shared-bindings/util.h" +#if CIRCUITPY_VIDEOCORE +#include "shared-module/videocore/__init__.h" +#endif #include "shared-module/displayio/__init__.h" //| class FramebufferDisplay: @@ -325,6 +328,11 @@ static mp_obj_t framebufferio_framebufferdisplay_obj_set_root_group(mp_obj_t sel displayio_group_t *group = NULL; if (group_in != mp_const_none) { group = MP_OBJ_TO_PTR(native_group(group_in)); + #if CIRCUITPY_VIDEOCORE + // Validate that draw protocol layers (e.g. a VideoBuffer) fit the + // display they will render on. + videocore_validate_root_group(group_in, self_in); + #endif } bool ok = common_hal_framebufferio_framebufferdisplay_set_root_group(self, group); diff --git a/shared-bindings/mipicsi/BayerOrder.c b/shared-bindings/mipicsi/BayerOrder.c new file mode 100644 index 00000000000..9f1058dd3e9 --- /dev/null +++ b/shared-bindings/mipicsi/BayerOrder.c @@ -0,0 +1,50 @@ +// This file is part of the CircuitPython project: https://circuitpython.org +// +// SPDX-FileCopyrightText: Copyright (c) 2026 Scott Shawcroft for Adafruit Industries +// +// SPDX-License-Identifier: MIT + +#include + +#include "py/enum.h" +#include "py/obj.h" +#include "py/runtime.h" + +#include "shared-bindings/mipicsi/__init__.h" + +//| class BayerOrder: +//| """The Bayer element order of a sensor's RAW output, in the sequence a 2x2 +//| pixel block transmits (top-left first, reading order)""" +//| +//| BGGR: BayerOrder +//| """The first element is Blue, then Green, then the next row starts with Green, then Red""" +//| +//| GBRG: BayerOrder +//| """The first element is Green, then Blue, then the next row starts with Red, then Green""" +//| +//| GRBG: BayerOrder +//| """The first element is Green, then Red, then the next row starts with Blue, then Green""" +//| +//| RGGB: BayerOrder +//| """The first element is Red, then Green, then the next row starts with Green, then Blue""" +//| + +MAKE_ENUM_VALUE(mipicsi_bayer_order_type, bayer_order, BGGR, MIPICSI_BAYER_ORDER_BGGR); +MAKE_ENUM_VALUE(mipicsi_bayer_order_type, bayer_order, GBRG, MIPICSI_BAYER_ORDER_GBRG); +MAKE_ENUM_VALUE(mipicsi_bayer_order_type, bayer_order, GRBG, MIPICSI_BAYER_ORDER_GRBG); +MAKE_ENUM_VALUE(mipicsi_bayer_order_type, bayer_order, RGGB, MIPICSI_BAYER_ORDER_RGGB); + +MAKE_ENUM_MAP(mipicsi_bayer_order) { + MAKE_ENUM_MAP_ENTRY(bayer_order, BGGR), + MAKE_ENUM_MAP_ENTRY(bayer_order, GBRG), + MAKE_ENUM_MAP_ENTRY(bayer_order, GRBG), + MAKE_ENUM_MAP_ENTRY(bayer_order, RGGB), +}; + +static MP_DEFINE_CONST_DICT(mipicsi_bayer_order_locals_dict, mipicsi_bayer_order_locals_table); +MAKE_PRINTER(mipicsi, mipicsi_bayer_order); +MAKE_ENUM_TYPE(mipicsi, BayerOrder, mipicsi_bayer_order); + +mipicsi_bayer_order_t validate_bayer_order(mp_obj_t obj, qstr arg_name) { + return cp_enum_value(&mipicsi_bayer_order_type, obj, arg_name); +} diff --git a/shared-bindings/mipicsi/Camera.c b/shared-bindings/mipicsi/Camera.c new file mode 100644 index 00000000000..1106e13ea53 --- /dev/null +++ b/shared-bindings/mipicsi/Camera.c @@ -0,0 +1,244 @@ +// This file is part of the CircuitPython project: https://circuitpython.org +// +// SPDX-FileCopyrightText: Copyright (c) 2025 Jeff Epler for Adafruit Industries +// +// SPDX-License-Identifier: MIT + +#include + +#include "py/mphal.h" +#include "py/obj.h" +#include "py/objproperty.h" +#include "py/runtime.h" + +#include "shared-bindings/displayio/Bitmap.h" +#include "shared-bindings/mipicsi/__init__.h" +#include "shared-bindings/mipicsi/Camera.h" +#include "shared-bindings/util.h" +#include "shared/runtime/context_manager_helpers.h" + +#if CIRCUITPY_VIDEOCORE +#include "shared-module/videocore/__init__.h" +#endif + +//| class Camera: +//| """Receive frames from a camera sensor on a MIPI CSI-2 interface +//| +//| The camera sensor is configured separately (typically over I2C) to +//| transmit frames. The MIPI CSI-2 receiver hardware has dedicated pins, +//| so no pins are configured here. +//| +//| Pass the configured sensor to the constructor; the receiver matches +//| itself to the sensor's current mode when a `videocore.VideoBuffer` +//| plays the camera. The sensor object must provide the properties and +//| methods of ``circuitpython_typing.CameraSensor``: ``width``, ``height``, +//| ``pixel_format``, ``lane_bit_rate_mbps``, ``data_lanes`` and +//| ``bayer_order`` (a `BayerOrder`) properties plus ``_start()`` and +//| ``_stop()`` methods. Those two are internal: the camera calls them +//| itself as the stream starts and stops, so user code should not. +//| All are validated here (and again at play, so the sensor may change +//| its configuration freely while it isn't being played). +//| +//| The sensor decides the output image size, and the receiver is built +//| around it: a sensor cannot change its mode while frames are captured. +//| Sensors are started (``_start()``) when a `videocore.VideoBuffer` +//| starts playing the camera and stopped by ``_stop()`` when it stops +//| playing. While the VideoBuffer is hidden its DMA is paused, but the +//| sensor keeps streaming, so implementations should refuse to +//| reconfigure (raise) while streaming rather than change the output size +//| or format mid-stream, and have a new Camera made when the mode changes. +//| +//| Camera objects implement the internal videosource protocol, so a +//| `videocore.VideoBuffer` can stream their frames into a framebuffer. +//| """ +//| +//| def __init__( +//| self, +//| sensor: circuitpython_typing.CameraSensor, +//| *, +//| output_format: Optional[PixelFormat] = None, +//| ) -> None: +//| """ +//| Configure and initialize the MIPI CSI-2 camera receiver +//| +//| .. code-block:: python +//| +//| sensor = sc2336.SC2336(i2c) +//| camera = mipicsi.Camera(sensor) +//| +//| :param sensor: A camera sensor to receive frames from. The receiver +//| is configured to match the sensor's current mode. The sensor +//| starts streaming when a `videocore.VideoBuffer` plays the +//| camera. +//| :param output_format: The format of the captured frames. When given and different from ``pixel_format``, the ISP converts each frame as it is captured (e.g. the sensor's RAW10 to `PixelFormat.RGB565`). Only RAW formats convert. Default: no conversion. +//| """ +//| +// Read the sensor's parameters off of the sensor object. A missing or +// invalid property fails here rather than later; mp_load_attr raises +// AttributeError for any property the sensor object doesn't provide. When +// pixel_format_override is not None it replaces the sensor's reported +// transmission format. +static mipicsi_pixel_format_t mipicsi_camera_load_sensor_params( + mp_obj_t sensor, + mipicsi_sensor_params_t *params) { + params->width = mp_arg_validate_int_min(mp_obj_get_int(mp_load_attr(sensor, MP_QSTR_width)), 1, MP_QSTR_width); + params->height = mp_arg_validate_int_min(mp_obj_get_int(mp_load_attr(sensor, MP_QSTR_height)), 1, MP_QSTR_height); + params->lane_bit_rate_mbps = mp_arg_validate_int_min(mp_obj_get_int(mp_load_attr(sensor, MP_QSTR_lane_bit_rate_mbps)), 1, MP_QSTR_lane_bit_rate_mbps); + params->data_lanes = mp_arg_validate_int_range(mp_obj_get_int(mp_load_attr(sensor, MP_QSTR_data_lanes)), 1, 2, MP_QSTR_data_lanes); + params->bayer_order = validate_bayer_order(mp_load_attr(sensor, MP_QSTR_bayer_order), MP_QSTR_bayer_order); + return validate_pixel_format(mp_load_attr(sensor, MP_QSTR_pixel_format), MP_QSTR_pixel_format); +} + +static mp_obj_t mipicsi_camera_make_new(const mp_obj_type_t *type, size_t n_args, size_t n_kw, const mp_obj_t *all_args) { + mp_arg_check_num(n_args, n_kw, 1, 1, true); + mp_obj_t sensor = all_args[0]; + + enum { ARG_output_format, NUM_ARGS }; + static const mp_arg_t allowed_args[] = { + { MP_QSTR_output_format, MP_ARG_OBJ | MP_ARG_KW_ONLY, { .u_obj = MP_ROM_NONE } }, + }; + mp_arg_val_t args[MP_ARRAY_SIZE(allowed_args)]; + MP_STATIC_ASSERT(MP_ARRAY_SIZE(allowed_args) == NUM_ARGS); + mp_arg_parse_all_kw_array(0, n_kw, all_args + 1, MP_ARRAY_SIZE(allowed_args), allowed_args, args); + + // All of the sensor's parameters (including the transmission format) are + // read off of the sensor object now, so a missing or invalid property + // fails here rather than at play time. They are read again at play, + // since the sensor is free to reconfigure between streams. + mipicsi_sensor_params_t params; + mipicsi_pixel_format_t pixel_format = mipicsi_camera_load_sensor_params(sensor, ¶ms); + mipicsi_pixel_format_t output_format = pixel_format; + if (args[ARG_output_format].u_obj != mp_const_none) { + output_format = validate_pixel_format(args[ARG_output_format].u_obj, MP_QSTR_output_format); + } + + // Validate the sensor contract methods (_start/_stop) exist now too, + // since the camera drives them from C as the stream starts and stops. + mp_obj_t validated[2]; + mp_load_method(sensor, MP_QSTR__start, validated); + mp_load_method(sensor, MP_QSTR__stop, validated); + + mipicsi_camera_obj_t *self = mp_obj_malloc_with_finaliser(mipicsi_camera_obj_t, &mipicsi_camera_type); + common_hal_mipicsi_camera_construct(self, output_format); + // Keep the sensor alive while receiving from it. It is only started + // streaming when a videocore.VideoBuffer starts playing (see the + // videosource play step): until then nothing captures or transmits. + common_hal_mipicsi_camera_set_sensor(self, sensor); + return MP_OBJ_FROM_PTR(self); +} + +//| def deinit(self) -> None: +//| """Deinitialises the camera receiver and releases all memory resources for reuse.""" +//| ... +//| +static mp_obj_t mipicsi_camera_deinit(mp_obj_t self_in) { + mipicsi_camera_obj_t *self = MP_OBJ_TO_PTR(self_in); + common_hal_mipicsi_camera_deinit(self); + return mp_const_none; +} +static MP_DEFINE_CONST_FUN_OBJ_1(mipicsi_camera_deinit_obj, mipicsi_camera_deinit); + +//| def __enter__(self) -> Camera: +//| """No-op used by Context Managers.""" +//| ... +//| +//| def __exit__(self) -> None: +//| """Automatically deinitializes the hardware when exiting a context. See +//| :ref:`lifetime-and-contextmanagers` for more info.""" +//| ... +//| +//| + +static const mp_rom_map_elem_t mipicsi_camera_locals_dict_table[] = { + { MP_ROM_QSTR(MP_QSTR___del__), MP_ROM_PTR(&mipicsi_camera_deinit_obj) }, + { MP_ROM_QSTR(MP_QSTR___enter__), MP_ROM_PTR(&default___enter___obj) }, + { MP_ROM_QSTR(MP_QSTR___exit__), MP_ROM_PTR(&default___exit___obj) }, + { MP_ROM_QSTR(MP_QSTR_deinit), MP_ROM_PTR(&mipicsi_camera_deinit_obj) }, +}; + +static MP_DEFINE_CONST_DICT(mipicsi_camera_locals_dict, mipicsi_camera_locals_dict_table); + +#if CIRCUITPY_VIDEOCORE +// Implementation of the videocore videosource protocol so a VideoBuffer can +// tell the Camera where to put frames. +static void mipicsi_camera_play(mp_obj_t self_in, bool paused) { + mipicsi_camera_obj_t *self = MP_OBJ_TO_PTR(mp_obj_cast_to_native_base(self_in, &mipicsi_camera_type)); + // The camera always has a sensor from make_new on. deinit() clears it + // (the sensor stops through its own finaliser), so a missing sensor only + // means the camera was deinitialised. + if (self->sensor == mp_const_none) { + raise_deinited_error(); + } + // Re-read the sensor's mode now: it is free to reconfigure between + // streams, and only a mode change while streaming is prevented (the + // sensor refuses it itself). The paused flag is for the receiver, not the + // sensor: the sensor starts streaming either way. + mipicsi_sensor_params_t params; + self->pixel_format = mipicsi_camera_load_sensor_params(self->sensor, ¶ms); + common_hal_mipicsi_camera_videosource_play(self, ¶ms, paused); +} + +static void mipicsi_camera_stop(mp_obj_t self_in) { + mipicsi_camera_obj_t *self = MP_OBJ_TO_PTR(mp_obj_cast_to_native_base(self_in, &mipicsi_camera_type)); + common_hal_mipicsi_camera_videosource_stop(self); +} + +static void mipicsi_camera_pause(mp_obj_t self_in) { + mipicsi_camera_obj_t *self = MP_OBJ_TO_PTR(mp_obj_cast_to_native_base(self_in, &mipicsi_camera_type)); + common_hal_mipicsi_camera_videosource_pause(self); +} + +static void mipicsi_camera_resume(mp_obj_t self_in) { + mipicsi_camera_obj_t *self = MP_OBJ_TO_PTR(mp_obj_cast_to_native_base(self_in, &mipicsi_camera_type)); + common_hal_mipicsi_camera_videosource_resume(self); +} + +static bool mipicsi_camera_set_destination(mp_obj_t self_in, const videocore_destination_t *dest, + videocore_on_frame_fun on_frame, void *ctx) { + mipicsi_camera_obj_t *self = MP_OBJ_TO_PTR(mp_obj_cast_to_native_base(self_in, &mipicsi_camera_type)); + // The protocol has no stop value on the destination: stopping is stop()'s + // job. + return common_hal_mipicsi_camera_set_destination(self, dest->base, dest->stride, on_frame, ctx); +} + +static mp_int_t mipicsi_camera_get_width(mp_obj_t self_in) { + return common_hal_mipicsi_camera_get_width(mp_obj_cast_to_native_base(self_in, &mipicsi_camera_type)); +} + +static mp_int_t mipicsi_camera_get_height(mp_obj_t self_in) { + return common_hal_mipicsi_camera_get_height(mp_obj_cast_to_native_base(self_in, &mipicsi_camera_type)); +} + +static mp_int_t mipicsi_camera_get_bits_per_pixel(mp_obj_t self_in) { + return common_hal_mipicsi_camera_get_bits_per_pixel(mp_obj_cast_to_native_base(self_in, &mipicsi_camera_type)); +} + +static const videosource_p_t mipicsi_camera_videosource_protocol = { + MP_PROTO_IMPLEMENT(MP_QSTR_protocol_videosource) + .play = mipicsi_camera_play, + .pause = mipicsi_camera_pause, + .resume = mipicsi_camera_resume, + .stop = mipicsi_camera_stop, + .set_destination = mipicsi_camera_set_destination, + .get_width = mipicsi_camera_get_width, + .get_height = mipicsi_camera_get_height, + .get_bits_per_pixel = mipicsi_camera_get_bits_per_pixel, +}; + +MP_DEFINE_CONST_OBJ_TYPE( + mipicsi_camera_type, + MP_QSTR_Camera, + MP_TYPE_FLAG_HAS_SPECIAL_ACCESSORS, + make_new, mipicsi_camera_make_new, + locals_dict, &mipicsi_camera_locals_dict, + protocol, &mipicsi_camera_videosource_protocol + ); +#else +MP_DEFINE_CONST_OBJ_TYPE( + mipicsi_camera_type, + MP_QSTR_Camera, + MP_TYPE_FLAG_HAS_SPECIAL_ACCESSORS, + make_new, mipicsi_camera_make_new, + locals_dict, &mipicsi_camera_locals_dict + ); +#endif diff --git a/shared-bindings/mipicsi/Camera.h b/shared-bindings/mipicsi/Camera.h new file mode 100644 index 00000000000..c81c9c42cee --- /dev/null +++ b/shared-bindings/mipicsi/Camera.h @@ -0,0 +1,73 @@ +// This file is part of the CircuitPython project: https://circuitpython.org +// +// SPDX-FileCopyrightText: Copyright (c) 2025 Jeff Epler for Adafruit Industries +// +// SPDX-License-Identifier: MIT + +#pragma once + +#include "py/obj.h" + +#include "common-hal/mipicsi/Camera.h" +#include "shared-bindings/mipicsi/__init__.h" + +extern const mp_obj_type_t mipicsi_camera_type; + +// Record the output format (possibly the sensor's). No hardware is created +// here: that happens in play(). +void common_hal_mipicsi_camera_construct( + mipicsi_camera_obj_t *self, + mipicsi_pixel_format_t output_format); + +// Remember the sensor configured to transmit to this receiver, if any. The +// receiver reads the sensor's mode from it at play time; the sensor stops +// itself (its finaliser does), never the receiver. +void common_hal_mipicsi_camera_set_sensor(mipicsi_camera_obj_t *self, mp_obj_t sensor); + +// Videosource protocol play step: create the receiver hardware for the +// sensor's current mode and start the sensor streaming. When paused is true +// the CSI controller is left stopped, so the camera is active but no frames +// are captured until resume(). Not a user facing method: it is driven by the +// videosource protocol (from VideoBuffer.play), so it may raise. +void common_hal_mipicsi_camera_videosource_play( + mipicsi_camera_obj_t *self, + const mipicsi_sensor_params_t *params, + bool paused); + +// Videosource protocol stop step: stop the stream and the sensor. Not a user +// facing method: it is driven by the videosource protocol (from +// VideoBuffer.stop), so it may raise. +void common_hal_mipicsi_camera_videosource_stop(mipicsi_camera_obj_t *self); + +// Videosource protocol pause step: stop the CSI controller's DMA without +// tearing down the receiver, ISP or sensor, so the camera stays active but no +// frames are captured. Not a user facing method: it is driven by the +// videosource protocol (when a framebuffer target is hidden). +void common_hal_mipicsi_camera_videosource_pause(mipicsi_camera_obj_t *self); + +// Videosource protocol resume step: start the CSI controller again at the most +// recent destination after a pause. Not a user facing method: it is driven by +// the videosource protocol (when a framebuffer target is shown again). +void common_hal_mipicsi_camera_videosource_resume(mipicsi_camera_obj_t *self); + +void common_hal_mipicsi_camera_deinit(mipicsi_camera_obj_t *self); +bool common_hal_mipicsi_camera_deinited(mipicsi_camera_obj_t *self); + +// Stream frames directly into base row after row. base must lead to a +// cache-line-aligned, contiguous DMA region: when base itself isn't aligned, +// the destination is not accepted. The destination must hold the whole +// (padded) frame back to back: the caller validates the fit, since the DMA +// has no stride. Returns whether those conditions hold (false instead of +// raising; the caller may ignore the result). +bool common_hal_mipicsi_camera_set_destination( + mipicsi_camera_obj_t *self, + uint8_t *base, + size_t stride, + void (*on_frame)(void *ctx, const void *frame), + void *ctx); + +mipicsi_pixel_format_t common_hal_mipicsi_camera_get_pixel_format(mipicsi_camera_obj_t *self); +mipicsi_pixel_format_t common_hal_mipicsi_camera_get_output_format(mipicsi_camera_obj_t *self); +mp_int_t common_hal_mipicsi_camera_get_width(mipicsi_camera_obj_t *self); +mp_int_t common_hal_mipicsi_camera_get_height(mipicsi_camera_obj_t *self); +mp_int_t common_hal_mipicsi_camera_get_bits_per_pixel(mipicsi_camera_obj_t *self); diff --git a/shared-bindings/mipicsi/PixelFormat.c b/shared-bindings/mipicsi/PixelFormat.c new file mode 100644 index 00000000000..84340019745 --- /dev/null +++ b/shared-bindings/mipicsi/PixelFormat.c @@ -0,0 +1,66 @@ +// This file is part of the CircuitPython project: https://circuitpython.org +// +// SPDX-FileCopyrightText: Copyright (c) 2025 Jeff Epler for Adafruit Industries +// +// SPDX-License-Identifier: MIT + +#include + +#include "py/enum.h" +#include "py/obj.h" +#include "py/runtime.h" + +#include "shared-bindings/mipicsi/__init__.h" + +//| class PixelFormat: +//| """Format of the data in the captured frames. The camera sensor must be +//| configured (typically over I2C) to transmit the corresponding MIPI CSI-2 +//| data type.""" +//| +//| RGB565: PixelFormat +//| """A 16-bit format with 5 bits of Red, 6 bits of Green, and 5 bits of Blue (MIPI data type ``RGB565``)""" +//| +//| RGB888: PixelFormat +//| """A 24-bit format with 8 bits of Red, Green and Blue (MIPI data type ``RGB888``)""" +//| +//| GRAYSCALE: PixelFormat +//| """An 8-bit format with 8 bits of luminance (MIPI data types for 8-bit grayscale)""" +//| +//| YUV422: PixelFormat +//| """A 16-bit YUV 4:2:2 format, in the byte order the sensor transmits (MIPI data type ``YUV422 8-bit``)""" +//| +//| RAW8: PixelFormat +//| """8 bits per pixel of raw Bayer data (MIPI data type ``RAW8``)""" +//| +//| RAW10: PixelFormat +//| """10 bits per pixel of raw Bayer data (MIPI data type ``RAW10``)""" +//| +//| RAW12: PixelFormat +//| """12 bits per pixel of raw Bayer data (MIPI data type ``RAW12``)""" +//| + +MAKE_ENUM_VALUE(mipicsi_pixel_format_type, pixel_format, RGB565, MIPICSI_FORMAT_RGB565); +MAKE_ENUM_VALUE(mipicsi_pixel_format_type, pixel_format, RGB888, MIPICSI_FORMAT_RGB888); +MAKE_ENUM_VALUE(mipicsi_pixel_format_type, pixel_format, GRAYSCALE, MIPICSI_FORMAT_GRAYSCALE); +MAKE_ENUM_VALUE(mipicsi_pixel_format_type, pixel_format, YUV422, MIPICSI_FORMAT_YUV422); +MAKE_ENUM_VALUE(mipicsi_pixel_format_type, pixel_format, RAW8, MIPICSI_FORMAT_RAW8); +MAKE_ENUM_VALUE(mipicsi_pixel_format_type, pixel_format, RAW10, MIPICSI_FORMAT_RAW10); +MAKE_ENUM_VALUE(mipicsi_pixel_format_type, pixel_format, RAW12, MIPICSI_FORMAT_RAW12); + +MAKE_ENUM_MAP(mipicsi_pixel_format) { + MAKE_ENUM_MAP_ENTRY(pixel_format, RGB565), + MAKE_ENUM_MAP_ENTRY(pixel_format, RGB888), + MAKE_ENUM_MAP_ENTRY(pixel_format, GRAYSCALE), + MAKE_ENUM_MAP_ENTRY(pixel_format, YUV422), + MAKE_ENUM_MAP_ENTRY(pixel_format, RAW8), + MAKE_ENUM_MAP_ENTRY(pixel_format, RAW10), + MAKE_ENUM_MAP_ENTRY(pixel_format, RAW12), +}; + +static MP_DEFINE_CONST_DICT(mipicsi_pixel_format_locals_dict, mipicsi_pixel_format_locals_table); +MAKE_PRINTER(mipicsi, mipicsi_pixel_format); +MAKE_ENUM_TYPE(mipicsi, PixelFormat, mipicsi_pixel_format); + +mipicsi_pixel_format_t validate_pixel_format(mp_obj_t obj, qstr arg_name) { + return cp_enum_value(&mipicsi_pixel_format_type, obj, arg_name); +} diff --git a/shared-bindings/mipicsi/__init__.c b/shared-bindings/mipicsi/__init__.c new file mode 100644 index 00000000000..d46e0eedf1d --- /dev/null +++ b/shared-bindings/mipicsi/__init__.c @@ -0,0 +1,46 @@ +// This file is part of the CircuitPython project: https://circuitpython.org +// +// SPDX-FileCopyrightText: Copyright (c) 2025 Jeff Epler for Adafruit Industries +// +// SPDX-License-Identifier: MIT + +#include + +#include "py/enum.h" +#include "py/obj.h" +#include "py/runtime.h" + +#include "shared-bindings/mipicsi/__init__.h" +#include "shared-bindings/mipicsi/Camera.h" + +//| """Support for MIPI CSI-2 camera interfaces +//| +//| Captures image frames from camera sensors attached to a MIPI CSI-2 +//| receiver, such as the one in the ESP32-P4. The camera sensor itself is +//| configured separately (typically over I2C); this module only receives +//| the image data the sensor transmits. +//| +//| The image data arrives in the format the sensor transmits, described by +//| `PixelFormat`. RAW formats carry Bayer data, whose element order is +//| described by `BayerOrder`. For instance, a sensor transmitting RGB565 +//| data yields frames of RGB565 pixels. Optionally, `mipicsi.Camera` can +//| convert frames with the ISP processor (set ``output_format`` and +//| ``pixel_format``), for example from the sensor's RAW output to RGB565 for +//| display. +//| """ + +static const mp_rom_map_elem_t mipicsi_module_globals_table[] = { + { MP_ROM_QSTR(MP_QSTR___name__), MP_ROM_QSTR(MP_QSTR_mipicsi) }, + { MP_ROM_QSTR(MP_QSTR_PixelFormat), MP_ROM_PTR(&mipicsi_pixel_format_type) }, + { MP_ROM_QSTR(MP_QSTR_BayerOrder), MP_ROM_PTR(&mipicsi_bayer_order_type) }, + { MP_ROM_QSTR(MP_QSTR_Camera), MP_ROM_PTR(&mipicsi_camera_type) }, +}; + +static MP_DEFINE_CONST_DICT(mipicsi_module_globals, mipicsi_module_globals_table); + +const mp_obj_module_t mipicsi_module = { + .base = { &mp_type_module }, + .globals = (mp_obj_dict_t *)&mipicsi_module_globals, +}; + +MP_REGISTER_MODULE(MP_QSTR_mipicsi, mipicsi_module); diff --git a/shared-bindings/mipicsi/__init__.h b/shared-bindings/mipicsi/__init__.h new file mode 100644 index 00000000000..5892379ce88 --- /dev/null +++ b/shared-bindings/mipicsi/__init__.h @@ -0,0 +1,33 @@ +// This file is part of the CircuitPython project: https://circuitpython.org +// +// SPDX-FileCopyrightText: Copyright (c) 2025 Jeff Epler for Adafruit Industries +// +// SPDX-License-Identifier: MIT + +#pragma once + +#include "py/enum.h" + +extern const mp_obj_type_t mipicsi_pixel_format_type; +extern const mp_obj_type_t mipicsi_bayer_order_type; +extern const cp_enum_obj_t pixel_format_RGB565_obj; + +typedef enum { + MIPICSI_FORMAT_RAW8, + MIPICSI_FORMAT_RAW10, + MIPICSI_FORMAT_RAW12, + MIPICSI_FORMAT_RGB565, + MIPICSI_FORMAT_RGB888, + MIPICSI_FORMAT_YUV422, + MIPICSI_FORMAT_GRAYSCALE, +} mipicsi_pixel_format_t; + +typedef enum { + MIPICSI_BAYER_ORDER_BGGR, + MIPICSI_BAYER_ORDER_GBRG, + MIPICSI_BAYER_ORDER_GRBG, + MIPICSI_BAYER_ORDER_RGGB, +} mipicsi_bayer_order_t; + +mipicsi_pixel_format_t validate_pixel_format(mp_obj_t obj, qstr arg_name); +mipicsi_bayer_order_t validate_bayer_order(mp_obj_t obj, qstr arg_name); diff --git a/shared-bindings/vectorio/__init__.h b/shared-bindings/vectorio/__init__.h index c5dcc026567..cc9b523ecb1 100644 --- a/shared-bindings/vectorio/__init__.h +++ b/shared-bindings/vectorio/__init__.h @@ -23,6 +23,9 @@ typedef bool (*draw_get_dirty_area_fun)(mp_obj_t draw_protocol_self, displayio_a typedef void (*draw_update_transform_fun)(mp_obj_t draw_protocol_self, displayio_buffer_transform_t *group_transform); typedef void (*draw_finish_refresh_fun)(mp_obj_t draw_protocol_self); typedef void (*draw_set_dirty_fun)(mp_obj_t draw_protocol_self); +// Optional: propagate a group's hidden state to the layer. May be NULL; the +// group then falls back to draw_set_dirty. +typedef void (*draw_set_hidden_fun)(mp_obj_t draw_protocol_self, bool hidden); typedef bool (*draw_set_in_group_fun)(mp_obj_t draw_protocol_self, bool in_group); typedef displayio_area_t *(*draw_get_refresh_areas_fun)(mp_obj_t draw_protocol_self, displayio_area_t *tail); @@ -33,6 +36,8 @@ typedef struct _vectorio_draw_protocol_impl_t { draw_finish_refresh_fun draw_finish_refresh; draw_get_refresh_areas_fun draw_get_refresh_areas; draw_set_dirty_fun draw_set_dirty; + // May be NULL. + draw_set_hidden_fun draw_set_hidden; draw_set_in_group_fun draw_set_in_group; } vectorio_draw_protocol_impl_t; diff --git a/shared-bindings/videocore/VideoBuffer.c b/shared-bindings/videocore/VideoBuffer.c new file mode 100644 index 00000000000..6563b98af93 --- /dev/null +++ b/shared-bindings/videocore/VideoBuffer.c @@ -0,0 +1,238 @@ +// This file is part of the CircuitPython project: https://circuitpython.org +// +// SPDX-FileCopyrightText: Copyright (c) 2026 Scott Shawcroft for Adafruit Industries +// +// SPDX-License-Identifier: MIT + +#include "py/obj.h" +#include "py/objproperty.h" +#include "py/runtime.h" + +#include "shared-bindings/videocore/VideoBuffer.h" +#include "shared-bindings/util.h" +#include "shared/runtime/context_manager_helpers.h" + +//| class VideoBuffer: +//| """Stream a video source into a framebuffer or buffer. +//| +//| A VideoBuffer can be added to displayio.Groups to show on a display +//| but may have size and position limitations. +//| +//| A VideoBuffer connects a video source (an object implementing the +//| videosource protocol, such as `mipicsi.Camera`) to a target: either a +//| framebuffer (an object implementing the framebuffer protocol, such as +//| `mipidsi.Display`) or a writable buffer such as a ``bytearray`` or +//| ``memoryview``. It is the video counterpart of `audioio.AudioOut`. +//| +//| The VideoBuffer tells the source where to put frames and moves the +//| destination as needed; while shown, the source streams continuously +//| into it. While playing, the target always holds the newest frame. +//| +//| When the target is a framebuffer, position the video with ``x`` and +//| ``y``: they may be changed at any time, including while playing, which +//| moves where the source puts its next frame. Some implementations may +//| restrict size and position. Setting position directly will raise an +//| error but indirectly through Group may result in weird output. For +//| example, on ESP32-P4 a VideoBuffer written into the framebuffer +//| directly must be full width because the ESP can't handle a stride +//| without an intermediate buffer. +//| +//| When the target is a plain buffer, frames are written compactly (row +//| stride equals ``width * bits_per_pixel / 8``) and can be read from +//| Python at any time — for example to process or save camera pixels. +//| +//| Two complete examples are in the `videocore` module overview: one +//| streaming a camera directly into a display, and one capturing a +//| 320x240 window into a ``bytearray`` that displayio places in a Group. +//| """ +//| +//| def __init__( +//| self, +//| framebuffer: Union[ +//| circuitpython_typing.FrameBuffer, circuitpython_typing.WriteableBuffer +//| ], +//| ) -> None: +//| """Create a VideoBuffer that plays video into the given framebuffer +//| +//| :param framebuffer: The framebuffer to copy video frames into, or a +//| writable buffer such as a ``bytearray`` or ``memoryview``. +//| """ +//| +static mp_obj_t videocore_videobuffer_make_new(const mp_obj_type_t *type, size_t n_args, size_t n_kw, const mp_obj_t *all_args) { + enum { ARG_framebuffer, NUM_ARGS }; + static const mp_arg_t allowed_args[] = { + { MP_QSTR_framebuffer, MP_ARG_REQUIRED | MP_ARG_OBJ }, + }; + + mp_arg_val_t args[MP_ARRAY_SIZE(allowed_args)]; + MP_STATIC_ASSERT(MP_ARRAY_SIZE(allowed_args) == NUM_ARGS); + mp_arg_parse_all_kw_array(n_args, n_kw, all_args, MP_ARRAY_SIZE(allowed_args), allowed_args, args); + + videocore_videobuffer_obj_t *self = mp_obj_malloc_with_finaliser(videocore_videobuffer_obj_t, &videocore_videobuffer_type); + common_hal_videocore_videobuffer_construct(self, args[ARG_framebuffer].u_obj); + return MP_OBJ_FROM_PTR(self); +} + +//| def play(self, source: circuitpython_typing.VideoSource) -> None: +//| """Begin streaming the given video source into the target. +//| +//| The source streams continuously, writing each new frame where the +//| VideoBuffer tells it to, replacing the previous one. The source's +//| frame format must be byte aligned. When the target is a +//| framebuffer, its bits per pixel must match the framebuffer color +//| depth and the whole frame must fit within the framebuffer at the +//| current ``x``/``y`` position (rows can't wrap past its end). When +//| the target is a plain buffer, it must be large enough for a whole +//| frame. The source's size and format are then fixed for the +//| duration of the play; sources that change them require ``stop`` +//| and a new ``play``. +//| +//| :param ~circuitpython_typing.VideoSource source: The video source to +//| play, such as `mipicsi.Camera` +//| """ +//| +static mp_obj_t videocore_videobuffer_play(mp_obj_t self_in, mp_obj_t source) { + videocore_videobuffer_obj_t *self = MP_OBJ_TO_PTR(self_in); + common_hal_videocore_videobuffer_play(self, source); + return mp_const_none; +} +static MP_DEFINE_CONST_FUN_OBJ_2(videocore_videobuffer_play_obj, videocore_videobuffer_play); + +//| def stop(self) -> None: +//| """Stop playing the video source.""" +//| ... +//| +static mp_obj_t videocore_videobuffer_stop(mp_obj_t self_in) { + videocore_videobuffer_obj_t *self = MP_OBJ_TO_PTR(self_in); + common_hal_videocore_videobuffer_stop(self); + return mp_const_none; +} +static MP_DEFINE_CONST_FUN_OBJ_1(videocore_videobuffer_stop_obj, videocore_videobuffer_stop); + +//| playing: bool +//| """True when a video source is being played, False otherwise. (read-only)""" +//| +static mp_obj_t videocore_videobuffer_get_playing(mp_obj_t self_in) { + videocore_videobuffer_obj_t *self = MP_OBJ_TO_PTR(self_in); + return mp_obj_new_bool(common_hal_videocore_videobuffer_get_playing(self)); +} +static MP_DEFINE_CONST_FUN_OBJ_1(videocore_videobuffer_get_playing_obj, videocore_videobuffer_get_playing); +MP_PROPERTY_GETTER(videocore_videobuffer_playing_obj, + (mp_obj_t)&videocore_videobuffer_get_playing_obj); + +//| source: Optional[circuitpython_typing.VideoSource] +//| """The video source being played, or None when not playing. (read-only)""" +//| +static mp_obj_t videocore_videobuffer_get_source(mp_obj_t self_in) { + videocore_videobuffer_obj_t *self = MP_OBJ_TO_PTR(self_in); + return common_hal_videocore_videobuffer_get_source(self); +} +static MP_DEFINE_CONST_FUN_OBJ_1(videocore_videobuffer_get_source_obj, videocore_videobuffer_get_source); +MP_PROPERTY_GETTER(videocore_videobuffer_source_obj, + (mp_obj_t)&videocore_videobuffer_get_source_obj); + +//| x: int +//| """The x position of the video within the display. Framebuffer targets +//| must be at x == 0 (video sources DMA whole rows): ``ValueError`` +//| otherwise. Plain buffer targets only use it to place the video in a +//| Group: it doesn't change where frames land within the buffer.""" +//| +//| y: int +//| """The y position of the video within the display.""" +//| +//| +static void videocore_videobuffer_get_position(mp_obj_t self_in, mp_int_t *x, mp_int_t *y) { + videocore_videobuffer_obj_t *self = MP_OBJ_TO_PTR(self_in); + *x = self->x; + *y = self->y; +} + +static mp_obj_t videocore_videobuffer_get_x(mp_obj_t self_in) { + mp_int_t x, y; + videocore_videobuffer_get_position(self_in, &x, &y); + return MP_OBJ_NEW_SMALL_INT(x); +} +static MP_DEFINE_CONST_FUN_OBJ_1(videocore_videobuffer_get_x_obj, videocore_videobuffer_get_x); + +static mp_obj_t videocore_videobuffer_set_x(mp_obj_t self_in, mp_obj_t value) { + videocore_videobuffer_obj_t *self = MP_OBJ_TO_PTR(self_in); + mp_int_t x = mp_obj_get_int(value); + common_hal_videocore_videobuffer_set_position(self, x, self->y); + return mp_const_none; +} +static MP_DEFINE_CONST_FUN_OBJ_2(videocore_videobuffer_set_x_obj, videocore_videobuffer_set_x); +MP_PROPERTY_GETSET(videocore_videobuffer_x_obj, + (mp_obj_t)&videocore_videobuffer_get_x_obj, + (mp_obj_t)&videocore_videobuffer_set_x_obj); + +static mp_obj_t videocore_videobuffer_get_y(mp_obj_t self_in) { + mp_int_t x, y; + videocore_videobuffer_get_position(self_in, &x, &y); + return MP_OBJ_NEW_SMALL_INT(y); +} +static MP_DEFINE_CONST_FUN_OBJ_1(videocore_videobuffer_get_y_obj, videocore_videobuffer_get_y); + +static mp_obj_t videocore_videobuffer_set_y(mp_obj_t self_in, mp_obj_t value) { + videocore_videobuffer_obj_t *self = MP_OBJ_TO_PTR(self_in); + mp_int_t y = mp_obj_get_int(value); + common_hal_videocore_videobuffer_set_position(self, self->x, y); + return mp_const_none; +} +static MP_DEFINE_CONST_FUN_OBJ_2(videocore_videobuffer_set_y_obj, videocore_videobuffer_set_y); +MP_PROPERTY_GETSET(videocore_videobuffer_y_obj, + (mp_obj_t)&videocore_videobuffer_get_y_obj, + (mp_obj_t)&videocore_videobuffer_set_y_obj); + +//| def deinit(self) -> None: +//| """Stop playing. The video source stops itself.""" +//| ... +//| +//| def __enter__(self) -> VideoBuffer: +//| """No-op used by Context Managers.""" +//| ... +//| +//| def __exit__(self) -> None: +//| """Automatically deinitializes the hardware when exiting a context. See +//| :ref:`lifetime-and-contextmanagers` for more info.""" +//| ... +//| +static mp_obj_t videocore_videobuffer_deinit(mp_obj_t self_in) { + videocore_videobuffer_obj_t *self = MP_OBJ_TO_PTR(self_in); + common_hal_videocore_videobuffer_stop(self); + return mp_const_none; +} +static MP_DEFINE_CONST_FUN_OBJ_1(videocore_videobuffer_deinit_obj, videocore_videobuffer_deinit); + +// The finaliser must not raise (it runs during garbage collection), so stop +// without one instead of deinit(): a failed stop stays best-effort, matching +// the source's own finaliser. +static mp_obj_t videocore_videobuffer___del__(mp_obj_t self_in) { + videocore_videobuffer_obj_t *self = MP_OBJ_TO_PTR(self_in); + videocore_videobuffer_stop_quiet(self); + return mp_const_none; +} +static MP_DEFINE_CONST_FUN_OBJ_1(videocore_videobuffer___del___obj, videocore_videobuffer___del__); + +static const mp_rom_map_elem_t videocore_videobuffer_locals_dict_table[] = { + { MP_ROM_QSTR(MP_QSTR___del__), MP_ROM_PTR(&videocore_videobuffer___del___obj) }, + { MP_ROM_QSTR(MP_QSTR___enter__), MP_ROM_PTR(&default___enter___obj) }, + { MP_ROM_QSTR(MP_QSTR___exit__), MP_ROM_PTR(&default___exit___obj) }, + { MP_ROM_QSTR(MP_QSTR_deinit), MP_ROM_PTR(&videocore_videobuffer_deinit_obj) }, + { MP_ROM_QSTR(MP_QSTR_play), MP_ROM_PTR(&videocore_videobuffer_play_obj) }, + { MP_ROM_QSTR(MP_QSTR_stop), MP_ROM_PTR(&videocore_videobuffer_stop_obj) }, + { MP_ROM_QSTR(MP_QSTR_playing), MP_ROM_PTR(&videocore_videobuffer_playing_obj) }, + { MP_ROM_QSTR(MP_QSTR_source), MP_ROM_PTR(&videocore_videobuffer_source_obj) }, + { MP_ROM_QSTR(MP_QSTR_x), MP_ROM_PTR(&videocore_videobuffer_x_obj) }, + { MP_ROM_QSTR(MP_QSTR_y), MP_ROM_PTR(&videocore_videobuffer_y_obj) }, +}; + +static MP_DEFINE_CONST_DICT(videocore_videobuffer_locals_dict, videocore_videobuffer_locals_dict_table); + +MP_DEFINE_CONST_OBJ_TYPE( + videocore_videobuffer_type, + MP_QSTR_VideoBuffer, + MP_TYPE_FLAG_HAS_SPECIAL_ACCESSORS, + make_new, videocore_videobuffer_make_new, + locals_dict, &videocore_videobuffer_locals_dict, + protocol, &videocore_videobuffer_draw_protocol + ); diff --git a/shared-bindings/videocore/VideoBuffer.h b/shared-bindings/videocore/VideoBuffer.h new file mode 100644 index 00000000000..927169bc4b3 --- /dev/null +++ b/shared-bindings/videocore/VideoBuffer.h @@ -0,0 +1,11 @@ +// This file is part of the CircuitPython project: https://circuitpython.org +// +// SPDX-FileCopyrightText: Copyright (c) 2026 Scott Shawcroft for Adafruit Industries +// +// SPDX-License-Identifier: MIT + +#pragma once + +#include "shared-module/videocore/VideoBuffer.h" + +extern const mp_obj_type_t videocore_videobuffer_type; diff --git a/shared-bindings/videocore/__init__.c b/shared-bindings/videocore/__init__.c new file mode 100644 index 00000000000..83c8487941a --- /dev/null +++ b/shared-bindings/videocore/__init__.c @@ -0,0 +1,91 @@ +// This file is part of the CircuitPython project: https://circuitpython.org +// +// SPDX-FileCopyrightText: Copyright (c) 2026 Scott Shawcroft for Adafruit Industries +// +// SPDX-License-Identifier: MIT + +#include "py/obj.h" +#include "py/runtime.h" + +#include "shared-bindings/videocore/VideoBuffer.h" +#include "shared-bindings/videocore/__init__.h" + +//| """Support for playing video sources +//| +//| This module provides the video pipeline shared between video sources and +//| video outputs. It is the video equivalent of the `audiocore` module. +//| +//| Video sources, such as `mipicsi.Camera`, implement the internal videosource +//| protocol. A `VideoBuffer` tells a video source where to put its frames in a +//| framebuffer, such as `mipidsi.Display`, and can move it as needed. +//| +//| Stream a MIPI CSI camera live into the board's display (a MIPI DSI panel +//| on ESP32-P4-Function-EV, for example): +//| +//| .. code-block:: python +//| +//| import board +//| import displayio +//| import mipicsi +//| import sc2336 +//| import videocore +//| +//| display = board.DISPLAY() +//| framebuffer = display.framebuffer +//| sensor = sc2336.SC2336(board.I2C()) +//| sensor.mode = sc2336.Mode.HD_1280x720_RAW10_30FPS +//| # Trim the sensor output to the display size. +//| sensor.width = display.width +//| sensor.height = display.height +//| camera = mipicsi.Camera(sensor, output_format=mipicsi.PixelFormat.RGB565) +//| +//| video = videocore.VideoBuffer(framebuffer) +//| video.play(camera) +//| root = displayio.Group() +//| root.append(video) +//| display.root_group = root +//| +//| Capture a smaller part of the sensor's image into a `bytearray` and let +//| displayio place it in a Group, together with other layers. displayio bulk +//| copies new frames into the group's area of the display (no color +//| conversion): +//| +//| .. code-block:: python +//| +//| import board +//| import displayio +//| import mipicsi +//| import sc2336 +//| import videocore +//| +//| sensor = sc2336.SC2336(board.I2C()) +//| sensor.mode = sc2336.Mode.VGA_640x480_RAW10_50FPS +//| # Trim the sensor output to a 320x240 window. +//| sensor.width = 320 +//| sensor.height = 240 +//| camera = mipicsi.Camera(sensor, output_format=mipicsi.PixelFormat.RGB565) +//| +//| capture = videocore.VideoBuffer(bytearray(320 * 240 * 2)) +//| capture.play(camera) +//| +//| root = displayio.Group() +//| root.append(capture) +//| capture.x = 350 # position of the live video within the group +//| capture.y = 180 +//| board.DISPLAY().root_group = root # live video joins the displayio scene +//| """ +//| + +static const mp_rom_map_elem_t videocore_module_globals_table[] = { + { MP_ROM_QSTR(MP_QSTR___name__), MP_ROM_QSTR(MP_QSTR_videocore) }, + { MP_ROM_QSTR(MP_QSTR_VideoBuffer), MP_ROM_PTR(&videocore_videobuffer_type) }, +}; + +static MP_DEFINE_CONST_DICT(videocore_module_globals, videocore_module_globals_table); + +const mp_obj_module_t videocore_module = { + .base = { &mp_type_module }, + .globals = (mp_obj_dict_t *)&videocore_module_globals, +}; + +MP_REGISTER_MODULE(MP_QSTR_videocore, videocore_module); diff --git a/shared-bindings/videocore/__init__.h b/shared-bindings/videocore/__init__.h new file mode 100644 index 00000000000..d309fcc838b --- /dev/null +++ b/shared-bindings/videocore/__init__.h @@ -0,0 +1,9 @@ +// This file is part of the CircuitPython project: https://circuitpython.org +// +// SPDX-FileCopyrightText: Copyright (c) 2026 Scott Shawcroft for Adafruit Industries +// +// SPDX-License-Identifier: MIT + +#pragma once + +#include "shared-module/videocore/__init__.h" diff --git a/shared-module/displayio/Group.c b/shared-module/displayio/Group.c index 84ebffdab6d..793464320f4 100644 --- a/shared-module/displayio/Group.c +++ b/shared-module/displayio/Group.c @@ -10,7 +10,7 @@ #include "py/objlist.h" #include "shared-bindings/displayio/TileGrid.h" -#if CIRCUITPY_VECTORIO +#if CIRCUITPY_VECTORIO || CIRCUITPY_VIDEOCORE #include "shared-bindings/vectorio/VectorShape.h" #endif @@ -52,11 +52,15 @@ void common_hal_displayio_group_set_hidden(displayio_group_t *self, bool hidden) displayio_group_set_hidden_by_parent(layer, hidden); continue; } - #if CIRCUITPY_VECTORIO + #if CIRCUITPY_VECTORIO || CIRCUITPY_VIDEOCORE const vectorio_draw_protocol_t *draw_protocol = mp_proto_get(MP_QSTR_protocol_draw, self->members->items[i]); if (draw_protocol != NULL) { layer = draw_protocol->draw_get_protocol_self(self->members->items[i]); - draw_protocol->draw_protocol_impl->draw_set_dirty(layer); + if (draw_protocol->draw_protocol_impl->draw_set_hidden != NULL) { + draw_protocol->draw_protocol_impl->draw_set_hidden(layer, hidden); + } else { + draw_protocol->draw_protocol_impl->draw_set_dirty(layer); + } continue; } #endif @@ -87,11 +91,15 @@ void displayio_group_set_hidden_by_parent(displayio_group_t *self, bool hidden) displayio_group_set_hidden_by_parent(layer, hidden); continue; } - #if CIRCUITPY_VECTORIO + #if CIRCUITPY_VECTORIO || CIRCUITPY_VIDEOCORE const vectorio_draw_protocol_t *draw_protocol = mp_proto_get(MP_QSTR_protocol_draw, self->members->items[i]); if (draw_protocol != NULL) { layer = draw_protocol->draw_get_protocol_self(self->members->items[i]); - draw_protocol->draw_protocol_impl->draw_set_dirty(layer); + if (draw_protocol->draw_protocol_impl->draw_set_hidden != NULL) { + draw_protocol->draw_protocol_impl->draw_set_hidden(layer, hidden); + } else { + draw_protocol->draw_protocol_impl->draw_set_dirty(layer); + } continue; } #endif @@ -141,16 +149,19 @@ bool displayio_group_get_previous_area(displayio_group_t *self, displayio_area_t } static void _update_child_transforms(displayio_group_t *self) { - if (!self->in_group) { - return; - } + // Draw protocol layers only act on a transform when we're attached to a + // display. TileGrids and nested Groups also need a transform to preserve + // their group membership, so pass the (zeroed-out when inactive) + // transform to them. + displayio_buffer_transform_t *draw_transform = + self->active ? &self->absolute_transform : NULL; for (size_t i = 0; i < self->members->len; i++) { mp_obj_t layer; - #if CIRCUITPY_VECTORIO + #if CIRCUITPY_VECTORIO || CIRCUITPY_VIDEOCORE const vectorio_draw_protocol_t *draw_protocol = mp_proto_get(MP_QSTR_protocol_draw, self->members->items[i]); if (draw_protocol != NULL) { layer = draw_protocol->draw_get_protocol_self(self->members->items[i]); - draw_protocol->draw_protocol_impl->draw_update_transform(layer, &self->absolute_transform); + draw_protocol->draw_protocol_impl->draw_update_transform(layer, draw_transform); continue; } #endif @@ -171,8 +182,8 @@ static void _update_child_transforms(displayio_group_t *self) { void displayio_group_update_transform(displayio_group_t *self, const displayio_buffer_transform_t *parent_transform) { - self->in_group = parent_transform != NULL; - if (self->in_group) { + self->active = parent_transform != NULL && parent_transform->scale != 0; + if (self->active) { int16_t x = self->x; int16_t y = self->y; if (parent_transform->transpose_xy) { @@ -188,6 +199,10 @@ void displayio_group_update_transform(displayio_group_t *self, self->absolute_transform.mirror_y = parent_transform->mirror_y; self->absolute_transform.scale = parent_transform->scale * self->scale; + } else { + // Clear the transform so inactive layers can't act on a stale one. + displayio_buffer_transform_t no_transform = {}; + self->absolute_transform = no_transform; } _update_child_transforms(self); } @@ -248,14 +263,15 @@ void common_hal_displayio_group_set_y(displayio_group_t *self, mp_int_t y) { static void _add_layer(displayio_group_t *self, mp_obj_t layer) { check_readonly(self); - #if CIRCUITPY_VECTORIO + #if CIRCUITPY_VECTORIO || CIRCUITPY_VIDEOCORE const vectorio_draw_protocol_t *draw_protocol = mp_proto_get(MP_QSTR_protocol_draw, layer); if (draw_protocol != NULL) { mp_obj_t protocol_self = draw_protocol->draw_get_protocol_self(layer); if (draw_protocol->draw_protocol_impl->draw_set_in_group(protocol_self, true)) { mp_raise_ValueError(MP_ERROR_TEXT("Layer already in a group")); } - draw_protocol->draw_protocol_impl->draw_update_transform(protocol_self, &self->absolute_transform); + draw_protocol->draw_protocol_impl->draw_update_transform( + protocol_self, self->active ? &self->absolute_transform : NULL); return; } #endif @@ -293,7 +309,7 @@ static void _remove_layer(displayio_group_t *self, size_t index) { mp_obj_t layer; displayio_area_t layer_area; bool rendered_last_frame = false; - #if CIRCUITPY_VECTORIO + #if CIRCUITPY_VECTORIO || CIRCUITPY_VIDEOCORE const vectorio_draw_protocol_t *draw_protocol = mp_proto_get(MP_QSTR_protocol_draw, self->members->items[index]); if (draw_protocol != NULL) { layer = draw_protocol->draw_get_protocol_self(self->members->items[index]); @@ -368,6 +384,7 @@ void displayio_group_construct(displayio_group_t *self, mp_obj_list_t *members, self->item_removed = false; self->scale = scale; self->in_group = false; + self->active = false; self->readonly = false; } @@ -377,7 +394,7 @@ bool displayio_group_fill_area(displayio_group_t *self, const _displayio_colorsp if (self->hidden == false) { for (int32_t i = self->members->len - 1; i >= 0; i--) { mp_obj_t layer; - #if CIRCUITPY_VECTORIO + #if CIRCUITPY_VECTORIO || CIRCUITPY_VIDEOCORE const vectorio_draw_protocol_t *draw_protocol = mp_proto_get(MP_QSTR_protocol_draw, self->members->items[i]); if (draw_protocol != NULL) { layer = draw_protocol->draw_get_protocol_self(self->members->items[i]); @@ -412,7 +429,7 @@ void displayio_group_finish_refresh(displayio_group_t *self) { self->item_removed = false; for (int32_t i = self->members->len - 1; i >= 0; i--) { mp_obj_t layer; - #if CIRCUITPY_VECTORIO + #if CIRCUITPY_VECTORIO || CIRCUITPY_VIDEOCORE const vectorio_draw_protocol_t *draw_protocol = mp_proto_get(MP_QSTR_protocol_draw, self->members->items[i]); if (draw_protocol != NULL) { layer = draw_protocol->draw_get_protocol_self(self->members->items[i]); @@ -443,7 +460,7 @@ static displayio_area_t *group_get_refresh_areas_impl(displayio_group_t *self, d for (int32_t i = self->members->len - 1; i >= 0; i--) { mp_obj_t layer; - #if CIRCUITPY_VECTORIO + #if CIRCUITPY_VECTORIO || CIRCUITPY_VIDEOCORE const vectorio_draw_protocol_t *draw_protocol = mp_proto_get(MP_QSTR_protocol_draw, self->members->items[i]); if (draw_protocol != NULL) { layer = draw_protocol->draw_get_protocol_self(self->members->items[i]); diff --git a/shared-module/displayio/Group.h b/shared-module/displayio/Group.h index 9ff6c7dca0f..d0a3bd353ec 100644 --- a/shared-module/displayio/Group.h +++ b/shared-module/displayio/Group.h @@ -24,10 +24,11 @@ typedef struct { uint16_t scale; bool item_removed : 1; bool in_group : 1; + bool active : 1; bool hidden : 1; bool hidden_by_parent : 1; bool readonly : 1; - uint8_t padding : 3; + uint8_t padding : 2; } displayio_group_t; void displayio_group_construct(displayio_group_t *self, mp_obj_list_t *members, uint32_t scale, mp_int_t x, mp_int_t y); diff --git a/shared-module/displayio/display_core.c b/shared-module/displayio/display_core.c index dfcddde7796..66837748eb3 100644 --- a/shared-module/displayio/display_core.c +++ b/shared-module/displayio/display_core.c @@ -112,6 +112,10 @@ bool displayio_display_core_set_root_group(displayio_display_core_t *self, displ return false; } if (self->current_group != NULL) { + // Deactivate the outgoing tree recursively. This clears its transforms + // and tells layers (such as a VideoBuffer) that they are no longer + // shown. + displayio_group_update_transform(self->current_group, NULL); self->current_group->in_group = false; } @@ -161,6 +165,9 @@ void displayio_display_core_finish_refresh(displayio_display_core_t *self) { void release_display_core(displayio_display_core_t *self) { if (self->current_group != NULL) { + // Deactivate the tree so layers stop acting on it (e.g. a VideoBuffer + // pauses its source). + displayio_group_update_transform(self->current_group, NULL); self->current_group->in_group = false; } } diff --git a/shared-module/framebufferio/FramebufferDisplay.c b/shared-module/framebufferio/FramebufferDisplay.c index 8116f4b0347..1cfb636420a 100644 --- a/shared-module/framebufferio/FramebufferDisplay.c +++ b/shared-module/framebufferio/FramebufferDisplay.c @@ -182,7 +182,23 @@ static bool _refresh_area(framebufferio_framebufferdisplay_obj_t *self, const di memset(mask, 0, mask_length * sizeof(mask[0])); memset(buffer, 0, buffer_size * sizeof(buffer[0])); - displayio_display_core_fill_area(&self->core, &subrectangle, mask, buffer); + bool covered = displayio_display_core_fill_area(&self->core, &subrectangle, mask, buffer); + + // A framebuffer-target VideoBuffer streams its pixels straight into + // the framebuffer, so when it is the only layer covering this + // subrectangle it returns full coverage without claiming any pixels. + // Nothing was composed into the scratch, so writing it back would + // erase the video; leave the framebuffer alone and just mark the rows. + bool write_back = true; + if (covered) { + write_back = false; + for (uint32_t m = 0; m < mask_length; m++) { + if (mask[m] != 0) { + write_back = true; + break; + } + } + } uint8_t *buf = (uint8_t *)self->bufinfo.buf, *endbuf = buf + self->bufinfo.len; (void)endbuf; // Hint to compiler that endbuf is "used" even if NDEBUG @@ -196,7 +212,9 @@ static bool _refresh_area(framebufferio_framebufferdisplay_obj_t *self, const di for (uint16_t i = subrectangle.y1; i < subrectangle.y2; i++) { assert(dest >= buf && dest < endbuf && dest + rowsize <= endbuf); MARK_ROW_DIRTY(i); - memcpy(dest, src, rowsize); + if (write_back) { + memcpy(dest, src, rowsize); + } dest += rowstride; src += rowsize; } diff --git a/shared-module/videocore/VideoBuffer.c b/shared-module/videocore/VideoBuffer.c new file mode 100644 index 00000000000..1095640b669 --- /dev/null +++ b/shared-module/videocore/VideoBuffer.c @@ -0,0 +1,828 @@ +// This file is part of the CircuitPython project: https://circuitpython.org +// +// SPDX-FileCopyrightText: Copyright (c) 2026 Scott Shawcroft for Adafruit Industries +// +// SPDX-License-Identifier: MIT + +#include + +#include "shared-module/videocore/VideoBuffer.h" + +#include "py/runtime.h" + +#include "shared-bindings/displayio/Group.h" +#include "shared-bindings/videocore/VideoBuffer.h" +#include "shared-bindings/videocore/__init__.h" +#include "shared-module/displayio/Group.h" +#include "shared-module/videocore/__init__.h" + +#if CIRCUITPY_FRAMEBUFFERIO +#include "shared-bindings/framebufferio/FramebufferDisplay.h" +#endif + +#define FB_GETTER(method, default_value) \ + (self->framebuffer_protocol->method \ + ? self->framebuffer_protocol->method(self->framebuffer) \ + : (default_value)) + +// Called from interrupt context when the source lands a frame. The frame +// landed in stream_base, the buffer the accepted destination gave (set when +// the last set_destination returned true), so keep it tiny. +static void videocore_videobuffer_on_frame(void *ctx, const void *frame) { + videocore_videobuffer_obj_t *self = ctx; + if (frame == self->stream_base) { + self->frame_ready = true; + return; + } + // It landed at a destination we have moved away from or paused: the + // frame that was being written at the time. Queue it for the refresh pass + // to repaint, since that area may already have been repainted. + const uint8_t head = self->stray_head; + if ((uint8_t)(head - self->stray_tail) >= VIDEOCORE_STRAY_FRAMES) { + self->stray_overflow = true; + return; + } + self->stray_frames[head % VIDEOCORE_STRAY_FRAMES] = frame; + __atomic_store_n(&self->stray_head, (uint8_t)(head + 1), __ATOMIC_RELEASE); +} + +// Union the screen area of a framebuffer-target frame that landed at frame +// into the dirty area. Framebuffer coordinates are screen coordinates (see +// compute_destination). A shift can start the frame mid-row, wrapping its +// rows; then cover the full width. +static void videocore_videobuffer_dirty_frame_at(videocore_videobuffer_obj_t *self, const void *frame) { + const size_t bytes_per_pixel = self->source_bits_per_pixel / 8; + if (self->framebuffer_protocol == NULL || bytes_per_pixel == 0) { + return; // Raw buffer frames are never shown where they land. + } + mp_buffer_info_t bufinfo; + self->framebuffer_protocol->get_bufinfo(self->framebuffer, &bufinfo); + const mp_int_t fb_width = FB_GETTER(get_width, 0); + const mp_int_t row_stride = FB_GETTER(get_row_stride, fb_width * self->source_bits_per_pixel / 8); + const mp_int_t first_pixel_offset = FB_GETTER(get_first_pixel_offset, 0); + const mp_int_t offset = (const uint8_t *)frame - (const uint8_t *)bufinfo.buf - first_pixel_offset; + if (bufinfo.buf == NULL || row_stride <= 0 || offset < 0 || offset >= (mp_int_t)bufinfo.len) { + return; + } + displayio_area_t area = { + .x1 = (offset % row_stride) / bytes_per_pixel, + .y1 = offset / row_stride, + }; + area.x2 = area.x1 + self->source_width; + area.y2 = area.y1 + self->source_height; + if (area.x2 > fb_width) { + area.x1 = 0; + area.x2 = fb_width; + area.y2 += 1; + } + displayio_area_union(&self->ephemeral_dirty_area, &area, &self->ephemeral_dirty_area); +} + +// Repaint where frames landed away from the current destination (see +// on_frame). Main context only. +static void videocore_videobuffer_dirty_stray_frames(videocore_videobuffer_obj_t *self) { + const uint8_t head = __atomic_load_n(&self->stray_head, __ATOMIC_ACQUIRE); + uint8_t tail = self->stray_tail; + while (tail != head) { + videocore_videobuffer_dirty_frame_at(self, self->stray_frames[tail % VIDEOCORE_STRAY_FRAMES]); + tail++; + } + __atomic_store_n(&self->stray_tail, tail, __ATOMIC_RELEASE); + if (self->stray_overflow) { + // Lost track of some: repaint the whole framebuffer. + self->stray_overflow = false; + if (self->framebuffer_protocol != NULL) { + displayio_area_t all = { + .x1 = 0, + .y1 = 0, + .x2 = FB_GETTER(get_width, 0), + .y2 = FB_GETTER(get_height, 0), + }; + displayio_area_union(&self->ephemeral_dirty_area, &all, &self->ephemeral_dirty_area); + } + } +} + +// Tear down playback state: clear our fields. +static void videocore_videobuffer_clear(videocore_videobuffer_obj_t *self); + +// Union the video's currently rendered rect into the dirty area so the next +// refresh repaints the pixels it leaves behind when it moves. Only meaningful +// while the video is actually shown: when it isn't, there are no pixels of +// ours to clear. Accumulates, so repeated moves before a refresh keep every +// vacated rect. +static void videocore_videobuffer_dirty_current_area(videocore_videobuffer_obj_t *self) { + if (self->absolute_transform.scale == 0 || self->hidden) { + return; + } + displayio_area_union( + &self->ephemeral_dirty_area, &self->current_area, &self->ephemeral_dirty_area); +} + +// Recompute the rendered rect (screen coordinates) of the video from the +// x/y position and the absolute transform. Video only supports scale 1 with +// no mirroring (validated in update_transform), so the rect is the frame +// size translated into place. +static void videocore_videobuffer_update_area(videocore_videobuffer_obj_t *self) { + const displayio_buffer_transform_t *t = &self->absolute_transform; + self->current_area.x1 = self->x + t->x; + self->current_area.y1 = self->y + t->y; + self->current_area.x2 = self->current_area.x1 + self->source_width; + self->current_area.y2 = self->current_area.y1 + self->source_height; + self->current_area.next = NULL; +} + +// Compute the destination for the source in the current target, honoring the +// current x/y position and the absolute transform's translation (a video in +// a group). Raises unless the whole frame fits in the target: the camera +// DMA's whole, contiguous frames with no stride padding, so rows can't wrap +// past the buffer and there is no cropping yet. The caller validates the +// fit (this runs from Python contexts); framebuffers in groups also move +// their destination from draw contexts with clamping (see _move_dest). +static void videocore_videobuffer_compute_destination( + videocore_videobuffer_obj_t *self, videocore_destination_t *dest) { + const size_t bytes_per_pixel = self->source_bits_per_pixel / 8; + const size_t row_bytes = (size_t)self->source_width * bytes_per_pixel; + + if (self->framebuffer_protocol == NULL) { + // Raw buffer mode: frames are written compactly from the start. + dest->base = self->bufinfo.buf; + dest->stride = row_bytes; + dest->row_bytes = row_bytes; + dest->height = self->source_height; + if (self->bufinfo.len < self->source_frame_bytes) { + mp_raise_ValueError(MP_ERROR_TEXT("Buffer too small")); + } + return; + } + + mp_buffer_info_t bufinfo; + self->framebuffer_protocol->get_bufinfo(self->framebuffer, &bufinfo); + if (bufinfo.buf == NULL) { + mp_raise_ValueError(MP_ERROR_TEXT("framebuffer went away")); + } + + const mp_int_t fb_width = FB_GETTER(get_width, 0); + const mp_int_t row_stride = FB_GETTER(get_row_stride, fb_width * self->source_bits_per_pixel / 8); + const mp_int_t first_pixel_offset = FB_GETTER(get_first_pixel_offset, 0); + + // The absolute position includes a containing group's translation. The + // shift may land the frame's rows mid-row: the image won't look right, + // but the contiguous DMA stays contiguous. Only the frame's start and + // end must remain within the buffer. + const displayio_buffer_transform_t *t = &self->absolute_transform; + const mp_int_t shift_x = self->x + t->x; + const mp_int_t shift_y = self->y + t->y; + if (self->x < 0 || self->y < 0 || shift_x < 0 || shift_y < 0 || + first_pixel_offset + shift_y * row_stride + shift_x * bytes_per_pixel + + (mp_int_t)self->source_frame_bytes > + bufinfo.len) { + mp_raise_ValueError(MP_ERROR_TEXT("video doesn't fit in framebuffer")); + } + + dest->base = (uint8_t *)bufinfo.buf + first_pixel_offset + + shift_y * row_stride + shift_x * bytes_per_pixel; + dest->stride = row_stride; + dest->row_bytes = row_bytes; + dest->height = self->source_height; +} + +// Move the framebuffer destination for a new absolute position, capping the +// shift so the frame stays completely within the framebuffer. Best effort: +// never raises, because this runs from displayio's refresh. +static void videocore_videobuffer_move_dest( + videocore_videobuffer_obj_t *self, mp_int_t shift_x, mp_int_t shift_y) { + if (!self->playing || self->paused) { + return; + } + mp_buffer_info_t bufinfo; + self->framebuffer_protocol->get_bufinfo(self->framebuffer, &bufinfo); + if (bufinfo.buf == NULL) { + return; + } + const size_t bytes_per_pixel = self->source_bits_per_pixel / 8; + const size_t row_bytes = (size_t)self->source_width * bytes_per_pixel; + const size_t frame_bytes = (size_t)self->source_height * row_bytes; + const mp_int_t fb_width = FB_GETTER(get_width, 0); + const mp_int_t row_stride = FB_GETTER(get_row_stride, fb_width * self->source_bits_per_pixel / 8); + const mp_int_t first_pixel_offset = FB_GETTER(get_first_pixel_offset, 0); + + if (shift_x < 0) { + shift_x = 0; + } + if (shift_y < 0) { + shift_y = 0; + } + // Cap the start and the end. Cap the rows first: enough rows must fit + // above the frame for it to fit at all. + const mp_int_t max_end = (mp_int_t)bufinfo.len - first_pixel_offset - (mp_int_t)frame_bytes; + if (shift_y > max_end / row_stride) { + shift_y = max_end / row_stride; + } + if (shift_y < 0) { + shift_y = 0; + } + const mp_int_t max_x = (max_end - shift_y * row_stride) / bytes_per_pixel; + if (shift_x > max_x) { + shift_x = max_x; + } + if (shift_x < 0) { + shift_x = 0; + } + + videocore_destination_t dest = { + .base = bufinfo.buf + first_pixel_offset + shift_y * row_stride + shift_x * bytes_per_pixel, + .stride = row_stride, + .row_bytes = row_bytes, + .height = self->source_height, + }; + // The result is ignored here: the source rejects a destination by + // returning false (e.g. a mid-row shift isn't DMA-aligned) and the + // stream continues where it was. on_frame points at the old destination + // until then, which keeps the last-known-good frame. + if (videosource_set_destination(self->source, &dest, videocore_videobuffer_on_frame, self)) { + self->stream_base = dest.base; + } +} + +void common_hal_videocore_videobuffer_construct(videocore_videobuffer_obj_t *self, mp_obj_t framebuffer) { + // The object has a finaliser, so initialise the fields it uses before any + // validation can raise. Otherwise a rejected framebuffer leaves source + // uninitialised and the later finaliser walks a garbage pointer. + self->source = mp_const_none; + self->stream_base = NULL; + self->playing = false; + self->framebuffer = framebuffer; + self->framebuffer_protocol = mp_proto_get(MP_QSTR_protocol_framebuffer, framebuffer); + if (self->framebuffer_protocol != NULL) { + if (self->framebuffer_protocol->get_bufinfo == NULL || + self->framebuffer_protocol->get_width == NULL || + self->framebuffer_protocol->get_height == NULL) { + mp_raise_ValueError(MP_ERROR_TEXT("framebuffer missing required functions")); + } + } else if (!mp_get_buffer(framebuffer, &self->bufinfo, MP_BUFFER_WRITE)) { + // Not a framebuffer and not a writable buffer such as a bytearray or + // memoryview. + mp_raise_TypeError_varg(MP_ERROR_TEXT("%q must be of type %q or %q, not %q"), + MP_QSTR_framebuffer, MP_QSTR_buffer, MP_QSTR_framebuffer, + mp_obj_get_type(framebuffer)->name); + } + + self->x = 0; + self->y = 0; + self->frame_ready = false; + self->stray_head = 0; + self->stray_tail = 0; + self->stray_overflow = false; + self->in_group = false; + self->hidden = false; + self->stale_size = false; + displayio_area_t empty = {}; + self->current_area = empty; + self->ephemeral_dirty_area = empty; + displayio_buffer_transform_t empty_transform = {}; + self->absolute_transform = empty_transform; + self->playing = false; + // Nothing is shown until a Group attached to a display updates our + // transform. + self->paused = true; +} + +// Start the source's hardware and capture its frame format. It is not pointed +// at a destination yet, so it does not write frames: the caller either starts +// streaming or leaves it paused. When paused is true the source comes up +// paused, so no frame can slip out between starting and pausing. The source +// may reconfigure itself (e.g. a sensor picks its mode), which fixes what +// get_* report from here on. Runs from Python contexts (play) so it may raise. +static void videocore_videobuffer_start_source(videocore_videobuffer_obj_t *self, bool paused) { + mp_obj_t source = self->source; + const videosource_p_t *protocol = videosource_check(source); + + videosource_play(source, paused); + + mp_int_t bits_per_pixel = protocol->get_bits_per_pixel(source); + if (bits_per_pixel % 8 != 0) { + mp_raise_ValueError(MP_ERROR_TEXT("video format must be byte aligned")); + } + mp_int_t width = protocol->get_width(source); + mp_int_t height = protocol->get_height(source); + size_t frame_bytes = (size_t)((width * height * bits_per_pixel) / 8); + + if (self->framebuffer_protocol != NULL) { + mp_int_t color_depth = FB_GETTER(get_color_depth, 16); + if (color_depth % 8 != 0 || bits_per_pixel != color_depth) { + mp_raise_ValueError(MP_ERROR_TEXT("video format doesn't match framebuffer")); + } + } + + self->source_width = width; + self->source_height = height; + self->source_bits_per_pixel = bits_per_pixel; + self->source_frame_bytes = frame_bytes; + + if (self->framebuffer_protocol == NULL && (mp_int_t)self->bufinfo.len < (mp_int_t)frame_bytes) { + mp_raise_ValueError(MP_ERROR_TEXT("Buffer too small")); + } +} + +// Point a started source at the current destination and let it stream. Runs +// from Python contexts (play and draw_update_transform) so it may raise. +static void videocore_videobuffer_start_stream(videocore_videobuffer_obj_t *self) { + videocore_destination_t dest; + videocore_videobuffer_compute_destination(self, &dest); + + // Tell the source where to put its pixels. The source streams from now + // on; on_frame tells us when a new frame has landed. It is the + // displayio group tree (and this object's Python references) that keep + // the VideoBuffer alive; a GC'd buffer's finaliser stops the stream. + if (!videosource_set_destination(self->source, &dest, videocore_videobuffer_on_frame, self)) { + mp_raise_ValueError(MP_ERROR_TEXT("video source rejected the destination")); + } + self->stream_base = dest.base; + self->paused = false; +} + +// Un-show a framebuffer target by pausing its source. The play session and +// the source's hardware stay alive, but no frames are written to the target. +// Best effort, like the finaliser's stop: this runs from a group operation, so +// it must not raise. +static void videocore_videobuffer_pause_stream(videocore_videobuffer_obj_t *self) { + if (self->paused || self->source == mp_const_none) { + return; // Already paused, or nothing has been started yet. + } + // Drop the destination before pausing the source: the source finishes its + // pause at the next frame boundary, so the frame being written still + // lands after this, and on_frame queues it as a stray to repaint over. + self->stream_base = NULL; + self->paused = true; + nlr_buf_t nlr; + if (nlr_push(&nlr) == 0) { + videosource_pause(self->source); + nlr_pop(); + } + self->frame_ready = false; +} + +// Re-show a framebuffer target by pointing its paused source at the current +// destination and resuming it. Position changes while hidden are applied here; +// the source's format was fixed at play(), so reject a changed source rather +// than resuming into a wrong-sized frame. +static void videocore_videobuffer_resume_stream(videocore_videobuffer_obj_t *self) { + const videosource_p_t *protocol = videosource_check(self->source); + if (protocol->get_width(self->source) != self->source_width || + protocol->get_height(self->source) != self->source_height || + protocol->get_bits_per_pixel(self->source) != self->source_bits_per_pixel) { + self->stale_size = true; + return; + } + + videocore_destination_t dest; + videocore_videobuffer_compute_destination(self, &dest); + if (!videosource_set_destination(self->source, &dest, videocore_videobuffer_on_frame, self)) { + mp_raise_ValueError(MP_ERROR_TEXT("video source rejected the destination")); + } + videosource_resume(self->source); + self->stream_base = dest.base; + self->paused = false; + // Wait for the first frame at the new destination. + self->frame_ready = false; +} + +// Start, resume or pause a framebuffer target's source to match whether it is +// shown (on a display and not hidden). It streams only while shown; while +// hidden the source stays active but paused, so showing it again is quick. +// Plain buffer targets stream regardless; displayio renders them wherever they +// are. +static void videocore_videobuffer_update_stream(videocore_videobuffer_obj_t *self) { + if (self->framebuffer_protocol == NULL) { + return; // Plain buffer targets always stream. + } + const bool shown = self->absolute_transform.scale != 0 && !self->hidden; + if (!self->playing) { + // Remember whether the target is shown so play() can start the source + // in the right state. + self->paused = !shown; + return; + } + if (shown && !self->stale_size) { + if (self->paused) { + videocore_videobuffer_resume_stream(self); + } + } else if (!shown && !self->paused) { + videocore_videobuffer_pause_stream(self); + } +} + +void common_hal_videocore_videobuffer_play(videocore_videobuffer_obj_t *self, mp_obj_t source) { + if (self->playing) { + mp_raise_RuntimeError(MP_ERROR_TEXT("already playing")); + } + videosource_check(source); + self->source = source; + self->stale_size = false; + + // A framebuffer target streams only while shown: DMA'ing into a buffer + // displayio isn't presenting would be meaningless. Start the source now + // either way, but start it paused when it isn't shown. Passing paused to + // play() (rather than pausing afterwards) keeps it from writing a frame + // in between. Plain buffer targets start streaming immediately. + const bool start_paused = self->framebuffer_protocol != NULL && self->paused; + videocore_videobuffer_start_source(self, start_paused); + self->playing = true; + if (start_paused) { + // No destination is set until the VideoBuffer is shown, so the source + // has nowhere to put a frame while paused. + self->frame_ready = false; + self->stream_base = NULL; + } else { + videocore_videobuffer_start_stream(self); + } +} + +void common_hal_videocore_videobuffer_set_position(videocore_videobuffer_obj_t *self, mp_int_t x, mp_int_t y) { + if (self->framebuffer_protocol != NULL && x != 0) { + // The source DMAs whole, contiguous frames with no stride, so every + // frame row must land on a framebuffer row: the video's x must be 0. + mp_raise_ValueError(MP_ERROR_TEXT("video in a framebuffer must be at x == 0")); + } + // Repaint where the video used to be. Union (rather than overwrite) so a + // move before the previous one was refreshed isn't lost. + videocore_videobuffer_dirty_current_area(self); + self->x = x; + self->y = y; + videocore_videobuffer_update_area(self); + if (!self->playing || self->paused || self->framebuffer_protocol == NULL) { + // Raw buffer targets are rendered by displayio; their capture + // destination doesn't move with the position. A paused framebuffer + // target has no destination to move yet. + return; + } + // The source's frame size was fixed when play() was called. Reject + // sources that changed it (e.g. a sensor mode change) rather than + // blitting at stale geometry. + const videosource_p_t *protocol = videosource_check(self->source); + mp_int_t width = protocol->get_width(self->source); + mp_int_t height = protocol->get_height(self->source); + mp_int_t bits_per_pixel = protocol->get_bits_per_pixel(self->source); + if (width != self->source_width || height != self->source_height || + bits_per_pixel != self->source_bits_per_pixel) { + mp_raise_ValueError(MP_ERROR_TEXT("video size changed since play; stop and play again")); + } + // Move the destination between frames; the source picks up the change on + // its next frame. The result may be ignored here (the source rejects a + // destination by returning false; the stream continues where it was). + videocore_destination_t dest; + videocore_videobuffer_compute_destination(self, &dest); + if (videosource_set_destination(self->source, &dest, videocore_videobuffer_on_frame, self)) { + self->stream_base = dest.base; + } +} + +static void videocore_videobuffer_clear(videocore_videobuffer_obj_t *self) { + self->playing = false; + self->frame_ready = false; + self->stream_base = NULL; + self->source = mp_const_none; +} + +void common_hal_videocore_videobuffer_stop(videocore_videobuffer_obj_t *self) { + // Stop the source; the camera also stops the sensor so its settings can + // be changed. That may raise: surface it, after cleaning up, since the + // stream did stop. + if (self->source != mp_const_none) { + nlr_buf_t nlr; + if (nlr_push(&nlr) == 0) { + videosource_stop(self->source); + nlr_pop(); + } else { + videocore_videobuffer_clear(self); + nlr_raise(nlr.ret_val); + } + } + videocore_videobuffer_clear(self); +} + +// Stop without raising: runs from background contexts (reload). A failed +// stop is best-effort; the source's own finaliser stops it as well. +void videocore_videobuffer_stop_quiet(videocore_videobuffer_obj_t *self) { + nlr_buf_t nlr; + if (nlr_push(&nlr) == 0) { + if (self->source != mp_const_none) { + videosource_stop(self->source); + } + nlr_pop(); + } + videocore_videobuffer_clear(self); +} + +bool common_hal_videocore_videobuffer_get_playing(videocore_videobuffer_obj_t *self) { + return self->playing; +} + +mp_obj_t common_hal_videocore_videobuffer_get_source(videocore_videobuffer_obj_t *self) { + return self->source; +} + +// Draw protocol implementation: a VideoBuffer holding a plain buffer gets +// bulk copied into the display by displayio's rendering; a framebuffer +// target's pixels are streamed into its framebuffer directly by the source, +// so displayio's fill reads them back into the scratch buffer it composes in +// (the refresh writes that scratch buffer back over the framebuffer). + +static mp_obj_t videocore_videobuffer_draw_protocol_self(mp_obj_t protocol_container) { + return protocol_container; +} + +static bool videocore_videobuffer_draw_fill_area(mp_obj_t self_in, const _displayio_colorspace_t *colorspace, const displayio_area_t *area, uint32_t *mask, uint32_t *buffer) { + videocore_videobuffer_obj_t *self = MP_OBJ_TO_PTR(self_in); + // buffer isn't necessarily the framebuffer. It may be a scratch buffer. + if (!self->playing || self->stream_base == NULL || self->hidden) { + return false; + } + + displayio_area_t overlap; + if (!displayio_area_compute_overlap(area, &self->current_area, &overlap)) { + return false; + } + const bool full_coverage = displayio_area_equal(area, &overlap); + const uint16_t span_px = displayio_area_width(area); + const size_t overlap_width = displayio_area_width(&overlap); + + if (self->framebuffer_protocol != NULL) { + // Framebuffer target: the source streamed the pixels straight into the + // framebuffer, so there is nothing to copy *into* the framebuffer (the + // DMA did that). If the video is the only thing covering this area, + // claiming nothing signals the driver to skip its write-back and + // leave those pixels alone. Otherwise displayio composes into a + // scratch buffer and writes it back over the framebuffer, so read the + // video's pixels into the scratch (where a higher layer hasn't already + // drawn) and claim them so lower layers don't draw over them. + const size_t bytes_per_pixel = self->source_bits_per_pixel / 8; + const mp_int_t fb_width = FB_GETTER(get_width, 0); + const mp_int_t row_stride = FB_GETTER(get_row_stride, fb_width * self->source_bits_per_pixel / 8); + const uint8_t *frame = (const uint8_t *)self->stream_base; + uint8_t *out = (uint8_t *)buffer; + + // Check whole words. This may include pixels between overlap rows or + // just outside the overlap, which can only make us take the slower + // per-pixel path below; it never misses a claimed pixel. + const size_t first_px = (size_t)(overlap.y1 - area->y1) * span_px + + (overlap.x1 - area->x1); + const size_t end_px = (size_t)(overlap.y2 - 1 - area->y1) * span_px + + (overlap.x2 - area->x1); + bool masked = false; + for (size_t word = first_px / 32; word <= (end_px - 1) / 32; word++) { + if (mask[word] != 0) { + masked = true; + break; + } + } + if (full_coverage && !masked) { + // Sole coverer: nothing was composed, so the driver skips the copy. + return true; + } + + for (int16_t y = overlap.y1; y < overlap.y2; y++) { + const size_t px = (size_t)(y - area->y1) * span_px + (overlap.x1 - area->x1); + const int16_t fy = y - self->current_area.y1; + const uint8_t *row = frame + (size_t)fy * row_stride + + (size_t)(overlap.x1 - self->current_area.x1) * bytes_per_pixel; + size_t i = 0; + while (i < overlap_width) { + // Skip pixels a higher layer already claimed or drew. + while (i < overlap_width && (mask[(px + i) / 32] & (1u << ((px + i) % 32)))) { + i++; + } + const size_t start = i; + while (i < overlap_width && !(mask[(px + i) / 32] & (1u << ((px + i) % 32)))) { + mask[(px + i) / 32] |= 1u << ((px + i) % 32); + i++; + } + if (i > start) { + memcpy( + out + (px + start) * bytes_per_pixel, + row + start * bytes_per_pixel, + (i - start) * bytes_per_pixel); + } + } + } + return full_coverage; + } + + // Raw buffer target: bulk copy, no color conversion: the display depth + // must match the video's pixel format byte for byte. + if (colorspace->depth != self->source_bits_per_pixel || colorspace->depth != 16) { + return false; + } + + // Opaque: claim the pixels. + for (int16_t y = overlap.y1; y < overlap.y2; y++) { + const size_t px = (size_t)(y - area->y1) * span_px + (overlap.x1 - area->x1); + for (size_t i = 0; i < overlap_width; i++) { + mask[(px + i) / 32] |= 1u << ((px + i) % 32); + } + } + + const size_t bytes_per_pixel = self->source_bits_per_pixel / 8; + const size_t row_bytes = (size_t)self->source_width * bytes_per_pixel; + const uint8_t *frame = (const uint8_t *)self->stream_base; + + for (int16_t y = overlap.y1; y < overlap.y2; y++) { + const size_t px = (size_t)(y - area->y1) * span_px + (overlap.x1 - area->x1); + const int16_t fy = y - self->current_area.y1; + memcpy( + (uint16_t *)buffer + px, + frame + (size_t)fy * row_bytes + (size_t)(overlap.x1 - self->current_area.x1) * bytes_per_pixel, + overlap_width * bytes_per_pixel); + } + return full_coverage; +} + +static bool videocore_videobuffer_draw_get_dirty_area(mp_obj_t self_in, displayio_area_t *out_area) { + videocore_videobuffer_obj_t *self = MP_OBJ_TO_PTR(self_in); + out_area->x1 = out_area->x2; + displayio_area_union(&self->ephemeral_dirty_area, &self->current_area, out_area); + return true; +} + +static void videocore_videobuffer_draw_update_transform(mp_obj_t self_in, displayio_buffer_transform_t *group_transform) { + videocore_videobuffer_obj_t *self = MP_OBJ_TO_PTR(self_in); + // Remember where the video was so a move can repaint the pixels it + // vacates. The area is only valid while the video is shown; a layer being + // attached or detached is handled by the display's full refresh. + displayio_area_t previous_area = self->current_area; + const bool was_shown = self->absolute_transform.scale != 0 && !self->hidden; + // A group that isn't attached to a display has no transform, and its + // update_transform clears the absolute transform we receive. A real + // (shown) transform always has scale >= 1. + if (group_transform == NULL || group_transform->scale == 0) { + displayio_buffer_transform_t no_transform = {}; + self->absolute_transform = no_transform; + } else { + // Validate the transform: video frames are bulk copied (raw targets) + // or DMA'd contiguously (framebuffer targets), so only translation + // moves. + if (group_transform->scale != 0 && + (group_transform->scale != 1 || + group_transform->mirror_x || group_transform->mirror_y || + group_transform->transpose_xy || + group_transform->dx != 1 || group_transform->dy != 1)) { + mp_raise_ValueError(MP_ERROR_TEXT("VideoBuffer does not support transforms")); + } + self->absolute_transform = *group_transform; + } + + // A framebuffer target starts or stops with whether it is shown (the + // absolute transform is known now). This runs from Python contexts, so a + // (re)start may raise. + videocore_videobuffer_update_stream(self); + + videocore_videobuffer_update_area(self); + + // Moved by a containing Group (or a transform change): dirty the rect it + // used to occupy so the refresh repaints those pixels instead of leaving + // the previous frame there. + if (was_shown && self->absolute_transform.scale != 0 && + !displayio_area_equal(&previous_area, &self->current_area)) { + displayio_area_union( + &self->ephemeral_dirty_area, &previous_area, &self->ephemeral_dirty_area); + } + + // Framebuffer targets stream straight into the buffer: move the + // destination along with the shift. Best effort, since this can't raise: + // the shift may land the start mid-row, which won't look right but is + // obvious; the frame is capped to stay completely within the buffer. + if (self->framebuffer_protocol != NULL) { + videocore_videobuffer_move_dest( + self, + self->x + self->absolute_transform.x, + self->y + self->absolute_transform.y); + } +} + +static void videocore_videobuffer_draw_finish_refresh(mp_obj_t self_in) { + videocore_videobuffer_obj_t *self = MP_OBJ_TO_PTR(self_in); + // The frame (or move) rendered; stop flagging it. + self->frame_ready = false; + displayio_area_t empty = {}; + self->ephemeral_dirty_area = empty; +} + +static displayio_area_t *videocore_videobuffer_draw_get_refresh_areas(mp_obj_t self_in, displayio_area_t *tail) { + videocore_videobuffer_obj_t *self = MP_OBJ_TO_PTR(self_in); + // Frames that landed after the video moved or paused, possibly over the + // repaint of the area they landed in: repaint those areas again. + videocore_videobuffer_dirty_stray_frames(self); + // Always repaint the area the video vacated, even once it is hidden or + // stopped, so its last frame doesn't stay on the target. + if (!displayio_area_empty(&self->ephemeral_dirty_area)) { + // The video moved or was hidden: repaint with the lower layers. + self->ephemeral_dirty_area.next = tail; + tail = &self->ephemeral_dirty_area; + } + if (!self->playing || self->paused || self->hidden || self->stale_size) { + return tail; + } + if (self->frame_ready) { + // New frame: bulk copy it in. + self->current_area.next = tail; + tail = &self->current_area; + } + return tail; +} + +static void videocore_videobuffer_draw_set_dirty(mp_obj_t self_in) { + (void)self_in; +} + +static void videocore_videobuffer_draw_set_hidden(mp_obj_t self_in, bool hidden) { + videocore_videobuffer_obj_t *self = MP_OBJ_TO_PTR(self_in); + if (self->hidden == hidden) { + return; + } + self->hidden = hidden; + // Repaint what the video showed or vacated. + self->ephemeral_dirty_area = self->current_area; + if (!hidden && self->playing) { + // The source normally can't change while the sensor stays streaming, + // but validate its sizing again in case it did: if it no longer + // matches, don't raise; simply stop showing the stream until stop and + // play again. + const videosource_p_t *protocol = mp_proto_get(MP_QSTR_protocol_videosource, self->source); + if (protocol != NULL && + (protocol->get_width(self->source) != self->source_width || + protocol->get_height(self->source) != self->source_height || + protocol->get_bits_per_pixel(self->source) != self->source_bits_per_pixel)) { + self->stale_size = true; + } + } + // Hiding or unhiding a framebuffer target pauses or resumes its source. + videocore_videobuffer_update_stream(self); +} + +static bool videocore_videobuffer_draw_set_in_group(mp_obj_t self_in, bool in_group) { + videocore_videobuffer_obj_t *self = MP_OBJ_TO_PTR(self_in); + bool was_in_group = self->in_group; + self->in_group = in_group; + return was_in_group; +} + +static vectorio_draw_protocol_impl_t videocore_videobuffer_draw_protocol_impl = { + .draw_fill_area = videocore_videobuffer_draw_fill_area, + .draw_get_dirty_area = videocore_videobuffer_draw_get_dirty_area, + .draw_update_transform = videocore_videobuffer_draw_update_transform, + .draw_finish_refresh = videocore_videobuffer_draw_finish_refresh, + .draw_get_refresh_areas = videocore_videobuffer_draw_get_refresh_areas, + .draw_set_dirty = videocore_videobuffer_draw_set_dirty, + .draw_set_hidden = videocore_videobuffer_draw_set_hidden, + .draw_set_in_group = videocore_videobuffer_draw_set_in_group, +}; + +const vectorio_draw_protocol_t videocore_videobuffer_draw_protocol = { + MP_PROTO_IMPLEMENT(MP_QSTR_protocol_draw) + .draw_get_protocol_self = videocore_videobuffer_draw_protocol_self, + .draw_protocol_impl = &videocore_videobuffer_draw_protocol_impl, +}; + +// Root group validation: a VideoBuffer that streams straight into a +// framebuffer bypasses displayio rendering, so it may only be shown on a +// display that renders that same framebuffer. Plain buffer targets are drawn +// by displayio and are fine anywhere. Walks the group tree and raises when a +// VideoBuffer doesn't match the display it would render on. +static void videocore_validate_member(mp_obj_t member, mp_obj_t display) { + displayio_group_t *group = mp_obj_cast_to_native_base(member, &displayio_group_type); + if (group != MP_OBJ_NULL) { + for (size_t i = 0; i < group->members->len; i++) { + videocore_validate_member(group->members->items[i], display); + } + return; + } + const vectorio_draw_protocol_t *draw_protocol = mp_proto_get(MP_QSTR_protocol_draw, member); + if (draw_protocol == NULL) { + return; + } + mp_obj_t layer = draw_protocol->draw_get_protocol_self(member); + if (mp_obj_get_type(layer) != &videocore_videobuffer_type) { + return; + } + videocore_videobuffer_obj_t *self = MP_OBJ_TO_PTR(layer); + if (self->framebuffer_protocol == NULL || display == self->framebuffer) { + return; + } + #if CIRCUITPY_FRAMEBUFFERIO + // A FramebufferDisplay renders the framebuffer it wraps. That is the only + // kind of display a framebuffer-target VideoBuffer can be shown on. + mp_obj_t native_display = mp_obj_cast_to_native_base(display, &framebufferio_framebufferdisplay_type); + if (native_display != MP_OBJ_NULL) { + framebufferio_framebufferdisplay_obj_t *fd = MP_OBJ_TO_PTR(native_display); + if (fd->framebuffer == self->framebuffer) { + return; + } + } + #endif + mp_raise_ValueError(MP_ERROR_TEXT("VideoBuffer is playing into a different framebuffer")); +} + +void videocore_validate_root_group(mp_obj_t root_group, mp_obj_t display) { + videocore_validate_member(root_group, display); +} diff --git a/shared-module/videocore/VideoBuffer.h b/shared-module/videocore/VideoBuffer.h new file mode 100644 index 00000000000..3a21b9b6d54 --- /dev/null +++ b/shared-module/videocore/VideoBuffer.h @@ -0,0 +1,96 @@ +// This file is part of the CircuitPython project: https://circuitpython.org +// +// SPDX-FileCopyrightText: Copyright (c) 2026 Scott Shawcroft for Adafruit Industries +// +// SPDX-License-Identifier: MIT + +#pragma once + +#include "py/obj.h" + +#include "shared-bindings/vectorio/__init__.h" // draw protocol +#include "shared-module/displayio/area.h" + +// How many frames that landed away from the current destination (see +// stray_frames) can wait for a refresh to repaint them. One lands per move or +// pause; more pile up only when the video moves faster than the display +// refreshes, and an overflow repaints the whole framebuffer instead. +#define VIDEOCORE_STRAY_FRAMES 4 +#include "shared-module/framebufferio/FramebufferDisplay.h" // framebuffer protocol +#include "shared-module/videocore/__init__.h" + +typedef struct { + mp_obj_base_t base; + // Object frames are streamed into: an object implementing the framebuffer + // protocol, such as mipidsi.Display, or a writable buffer such as a + // bytearray or memoryview. + mp_obj_t framebuffer; + const framebuffer_p_t *framebuffer_protocol; + // Target buffer info in raw buffer mode (framebuffer_protocol == NULL). + mp_buffer_info_t bufinfo; + // The video source being played, or None. + mp_obj_t source; + // Destination position of the video within the target. + mp_int_t x; + mp_int_t y; + mp_int_t source_width; + mp_int_t source_height; + mp_int_t source_bits_per_pixel; + size_t source_frame_bytes; + // Where the source streams frames into: the requested destination, or a + // later, hardware-aligned position inside it. "Later in the buffer" only + // happens for plain-buffer targets; set by play() and the x/y + // properties. + const void *stream_base; + // Set from interrupt context when the source lands a frame in the + // destination; cleared by the refresh pass once the new frame has been + // rendered into the display. + volatile bool frame_ready; + // Frames that landed somewhere other than stream_base: the frame being + // written when the destination changes, or when the source is paused, + // still lands at the previous destination, possibly after the refresh + // that repainted the vacated area. on_frame queues where they landed + // (interrupt context, the only producer) and the refresh pass repaints + // those areas with the lower layers (the only consumer). + const void *volatile stray_frames[VIDEOCORE_STRAY_FRAMES]; + volatile uint8_t stray_head; + volatile uint8_t stray_tail; + volatile bool stray_overflow; + // displayio draw protocol state: the rect currently rendered (in screen + // coordinates) and the rect that changed since the last refresh. + displayio_area_t current_area; + displayio_area_t ephemeral_dirty_area; + displayio_buffer_transform_t absolute_transform; + bool in_group; + // Set by the group when hidden; while hidden the video doesn't show and + // a framebuffer target's source is paused (it is revalidated on unhide). + bool hidden; + // True when the source's sizing no longer matches what play() captured: + // the video won't show until stop and play again. + bool stale_size; + bool playing; + // True when the framebuffer target isn't shown (not attached to a display + // or hidden), so the source should not stream to the target. play() passes + // this to the source so it starts paused; otherwise showing/hiding the + // target pauses and resumes the source. It also keeps this state while + // stopped, so a later play() starts correctly. Plain buffer targets are + // never paused. + bool paused; +} videocore_videobuffer_obj_t; + +void common_hal_videocore_videobuffer_construct(videocore_videobuffer_obj_t *self, mp_obj_t framebuffer); +void common_hal_videocore_videobuffer_play(videocore_videobuffer_obj_t *self, mp_obj_t source); +void common_hal_videocore_videobuffer_set_position(videocore_videobuffer_obj_t *self, mp_int_t x, mp_int_t y); +void common_hal_videocore_videobuffer_stop(videocore_videobuffer_obj_t *self); +// Stop without raising (from the finaliser). A failed stop is best-effort; +// the source's own finaliser stops it as well. +void videocore_videobuffer_stop_quiet(videocore_videobuffer_obj_t *self); +bool common_hal_videocore_videobuffer_get_playing(videocore_videobuffer_obj_t *self); +mp_obj_t common_hal_videocore_videobuffer_get_source(videocore_videobuffer_obj_t *self); + +// The draw protocol implementation that lets a VideoBuffer be added to a +// displayio Group: plain-buffer targets get bulk copied into the framebuffer +// by displayio's rendering; framebuffer targets claim their area (a no-op +// fill: the source streams their pixels straight into the framebuffer) so +// displayio's refresh pushes the video's rows to the framebuffer. +extern const vectorio_draw_protocol_t videocore_videobuffer_draw_protocol; diff --git a/shared-module/videocore/__init__.c b/shared-module/videocore/__init__.c new file mode 100644 index 00000000000..6f45f783688 --- /dev/null +++ b/shared-module/videocore/__init__.c @@ -0,0 +1,56 @@ +// This file is part of the CircuitPython project: https://circuitpython.org +// +// SPDX-FileCopyrightText: Copyright (c) 2026 Scott Shawcroft for Adafruit Industries +// +// SPDX-License-Identifier: MIT + +#include "shared-module/videocore/__init__.h" +#include "shared-module/videocore/VideoBuffer.h" + +// Playback, presentation and garbage collection are tied to the displayio +// group the VideoBuffer is in: displayio's refresh drives frame presentation +// through the draw protocol, and the group (or a Python reference) keeps the +// object alive; its finaliser stops the stream. + +const videosource_p_t *videosource_check(mp_obj_t obj) { + return (const videosource_p_t *)mp_proto_get_or_throw(MP_QSTR_protocol_videosource, obj); +} + +void videosource_play(mp_obj_t source, bool paused) { + const videosource_p_t *protocol = videosource_check(source); + if (protocol->play != NULL) { + protocol->play(source, paused); + } +} + +void videosource_stop(mp_obj_t source) { + const videosource_p_t *protocol = videosource_check(source); + if (protocol->stop != NULL) { + protocol->stop(source); + } +} + +void videosource_pause(mp_obj_t source) { + videosource_check(source)->pause(source); +} + +void videosource_resume(mp_obj_t source) { + videosource_check(source)->resume(source); +} + +bool videosource_set_destination(mp_obj_t source, const videocore_destination_t *dest, + videocore_on_frame_fun on_frame, void *ctx) { + return videosource_check(source)->set_destination(source, dest, on_frame, ctx); +} + +mp_int_t videosource_get_width(mp_obj_t source) { + return videosource_check(source)->get_width(source); +} + +mp_int_t videosource_get_height(mp_obj_t source) { + return videosource_check(source)->get_height(source); +} + +mp_int_t videosource_get_bits_per_pixel(mp_obj_t source) { + return videosource_check(source)->get_bits_per_pixel(source); +} diff --git a/shared-module/videocore/__init__.h b/shared-module/videocore/__init__.h new file mode 100644 index 00000000000..86eddae7c6e --- /dev/null +++ b/shared-module/videocore/__init__.h @@ -0,0 +1,120 @@ +// This file is part of the CircuitPython project: https://circuitpython.org +// +// SPDX-FileCopyrightText: Copyright (c) 2026 Scott Shawcroft for Adafruit Industries +// +// SPDX-License-Identifier: MIT + +#pragma once + +#include +#include +#include + +#include "py/obj.h" +#include "py/proto.h" + +// Destination for a video stream: where to put pixels, as a pointer to the +// first pixel and the destination row stride. row_bytes/height describe the +// source frame; the source assumes the destination holds the whole frame +// (the ESP CSI DMA is contiguous-only), so the rect is validated before +// streaming starts and when the position changes. +typedef struct { + // First pixel of the destination. + uint8_t *base; + // Destination row stride, in bytes. + size_t stride; + // Source frame row size, in bytes. + size_t row_bytes; + // Source frame height, in rows. + size_t height; +} videocore_destination_t; + +// Called (in interrupt context) when a frame has landed. frame is the +// destination base it landed at: usually the latest accepted destination, but +// a frame already being written when the destination changed (or when the +// source was paused) still lands at the previous one. +typedef void (*videocore_on_frame_fun)(void *ctx, const void *frame); + +// Protocol implemented by video streamers such as mipicsi.Camera. A source +// streams continuously while playing (unless paused); the target sets (and can +// change) where the pixels go by calling set_destination() at any time, +// including between frames. +// +// This is the video equivalent of the audiosample protocol in the audiocore +// module, inverted: instead of the target pulling frames, the source writes +// where it is told to. +typedef struct _videosource_p_t { + MP_PROTOCOL_HEAD // MP_QSTR_protocol_videosource + + // Create whatever streaming hardware the source needs and start it (e.g. + // start a sensor streaming). Called once before the first + // set_destination of a stream, from Python (VideoBuffer.play), so it may + // raise. Raises when the source is already streaming for someone else. + // May be NULL for sources that need no preparation. The source must not + // write frames before set_destination(). When paused is true, the source's + // hardware must come up active but not writing frames, as if pause() had + // been called before it started; this avoids a play-then-pause race when + // the target isn't shown yet. + void (*play)(mp_obj_t self, bool paused); + + // Pause streaming without stopping the camera or releasing its hardware: + // e.g. stop the CSI controller's DMA while the sensor keeps streaming. + // No frames are written until resume(). Called when a framebuffer target + // stops being shown. set_destination() while paused updates where resume() + // will stream. Must not raise: it runs from displayio group operations. + void (*pause)(mp_obj_t self); + + // Resume streaming after pause() into the most recent destination. Called + // from Python contexts (when the framebuffer target is shown again), so it + // may raise; the source must stay paused if it does. + void (*resume)(mp_obj_t self); + + // Direct future frames into dest. The destination may be changed + // between frames; there is no NULL/stop value: stopping is stop()'s + // job. Never raises: returns whether the stream accepted the + // destination (the caller may ignore the result). on_frame(ctx, frame) + // is called from interrupt context whenever a frame has landed, with the + // destination base it landed at (the source never relocates frames + // elsewhere). A change takes effect from the next frame: the frame being + // written still lands at the previous destination. + bool (*set_destination)(mp_obj_t self, const videocore_destination_t *dest, + videocore_on_frame_fun on_frame, void *ctx); + + // Stop streaming and release the streaming hardware (e.g. stop the + // sensor so its settings can be changed). Called once per stream, from + // Python (VideoBuffer.stop), so it may raise; exceptions there are + // clearer than implicit stops. Only called when play() succeeded. + void (*stop)(mp_obj_t self); + + // Width of the frames the source streams, in pixels. + mp_int_t (*get_width)(mp_obj_t self); + // Height of the frames the source streams, in lines. + mp_int_t (*get_height)(mp_obj_t self); + // Bits per pixel of the frames the source streams. + mp_int_t (*get_bits_per_pixel)(mp_obj_t self); +} videosource_p_t; + +// Protocol function helpers. videosource_play/stop are no-ops for sources +// whose protocol doesn't implement them. pause/resume are required: they +// keep a source's hardware alive while a framebuffer target is hidden. +void videosource_play(mp_obj_t source, bool paused); +void videosource_stop(mp_obj_t source); +void videosource_pause(mp_obj_t source); +void videosource_resume(mp_obj_t source); + +// Return the videosource protocol of obj or raise TypeError. +const videosource_p_t *videosource_check(mp_obj_t obj); + +// Dispatch helpers mirroring the audiosample_* functions. videosource_play +// is a no-op for sources whose protocol has no play. +bool videosource_set_destination(mp_obj_t source, const videocore_destination_t *dest, + videocore_on_frame_fun on_frame, void *ctx); +mp_int_t videosource_get_width(mp_obj_t source); +mp_int_t videosource_get_height(mp_obj_t source); +mp_int_t videosource_get_bits_per_pixel(mp_obj_t source); + +// Raise when the group tree contains a VideoBuffer that streams into a +// framebuffer other than the one this display renders. Called when a group +// is made a display's root group. Implemented in shared-module/videocore/ +// VideoBuffer.c. +void videocore_validate_root_group(mp_obj_t root_group, mp_obj_t display); From 4655aacb6bfb9cb860357b9e674e68b6f833fa53 Mon Sep 17 00:00:00 2001 From: Scott Shawcroft Date: Tue, 20 Oct 2026 14:33:20 -0700 Subject: [PATCH 2/3] Temporarily point at the esp-idf branch with camera fixes --- .gitmodules | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.gitmodules b/.gitmodules index 6ac7a079035..93fc7dc0c6f 100644 --- a/.gitmodules +++ b/.gitmodules @@ -142,8 +142,8 @@ url = https://github.com/adafruit/Adafruit_CircuitPython_RFM69.git [submodule "ports/espressif/esp-idf"] path = ports/espressif/esp-idf - url = https://github.com/adafruit/esp-idf.git - branch = circuitpython-v6.1 + url = https://github.com/tannewt/esp-idf.git + branch = camera_pausing [submodule "ports/espressif/esp-protocols"] path = ports/espressif/esp-protocols url = https://github.com/adafruit/esp-protocols.git From 77d4fc4c15b580e3fb3f165f4d85b291c7234646 Mon Sep 17 00:00:00 2001 From: Scott Shawcroft Date: Tue, 20 Oct 2026 15:27:55 -0700 Subject: [PATCH 3/3] Fix non-videocore builds --- shared-module/displayio/Group.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/shared-module/displayio/Group.c b/shared-module/displayio/Group.c index 793464320f4..2f96a0fd578 100644 --- a/shared-module/displayio/Group.c +++ b/shared-module/displayio/Group.c @@ -153,8 +153,10 @@ static void _update_child_transforms(displayio_group_t *self) { // display. TileGrids and nested Groups also need a transform to preserve // their group membership, so pass the (zeroed-out when inactive) // transform to them. + #if CIRCUITPY_VECTORIO || CIRCUITPY_VIDEOCORE displayio_buffer_transform_t *draw_transform = self->active ? &self->absolute_transform : NULL; + #endif for (size_t i = 0; i < self->members->len; i++) { mp_obj_t layer; #if CIRCUITPY_VECTORIO || CIRCUITPY_VIDEOCORE