diff --git a/.gitmodules b/.gitmodules index 6ac7a079035..93fc7dc0c6f 100644 --- a/.gitmodules +++ b/.gitmodules @@ -142,8 +142,8 @@ url = https://github.com/adafruit/Adafruit_CircuitPython_RFM69.git [submodule "ports/espressif/esp-idf"] path = ports/espressif/esp-idf - url = https://github.com/adafruit/esp-idf.git - branch = circuitpython-v6.1 + url = https://github.com/tannewt/esp-idf.git + branch = camera_pausing [submodule "ports/espressif/esp-protocols"] path = ports/espressif/esp-protocols url = https://github.com/adafruit/esp-protocols.git diff --git a/locale/circuitpython.pot b/locale/circuitpython.pot index c03d1bd0ae4..97b798f7f5f 100644 --- a/locale/circuitpython.pot +++ b/locale/circuitpython.pot @@ -974,7 +974,6 @@ msgstr "" #: ports/atmel-samd/common-hal/pulseio/PulseIn.c #: ports/atmel-samd/common-hal/rotaryio/IncrementalEncoder.c #: ports/cxd56/common-hal/pulseio/PulseIn.c -#: ports/espressif/common-hal/mii/RMII.c #: ports/raspberrypi/common-hal/picodvi/Framebuffer_RP2350.c #: ports/zephyr-cp/common-hal/busio/I2C.c #: ports/zephyr-cp/common-hal/busio/SPI.c @@ -1182,6 +1181,7 @@ msgid "Another send is already active" msgstr "" #: ports/atmel-samd/common-hal/sdioio/SDCard.c +#: ports/espressif/common-hal/mii/Ethernet.c #: ports/espressif/common-hal/qspibus/QSPIBus.c shared-bindings/emmcio/EMMC.c #: shared-bindings/socketpool/SocketPool.c msgid "%q failure: %d" @@ -1219,6 +1219,7 @@ msgstr "" #: shared-bindings/busdisplay/BusDisplay.c #: shared-bindings/framebufferio/FramebufferDisplay.c #: shared-bindings/struct/__init__.c shared-module/struct/__init__.c +#: shared-module/videocore/VideoBuffer.c msgid "Buffer too small" msgstr "" @@ -1517,7 +1518,8 @@ msgid "%q must be 8, 16, 24, or 32" msgstr "" #: ports/espressif/common-hal/audiobusio/__init__.c -#: ports/espressif/common-hal/audioi2sin/I2SIn.c shared-bindings/emmcio/EMMC.c +#: ports/espressif/common-hal/audioi2sin/I2SIn.c +#: ports/espressif/common-hal/mii/Ethernet.c shared-bindings/emmcio/EMMC.c msgid "Peripheral in use" msgstr "" @@ -1563,6 +1565,8 @@ msgid "Can't construct AudioOut because continuous channel already open" msgstr "" #: ports/espressif/common-hal/audioio/AudioOut.c +#: ports/espressif/common-hal/mipicsi/Camera.c +#: shared-module/videocore/VideoBuffer.c msgid "already playing" msgstr "" @@ -1751,6 +1755,18 @@ msgstr "" msgid "Address range not allowed" msgstr "" +#: ports/espressif/common-hal/mipicsi/Camera.c +msgid "pixel_format must be RAW8, RAW10 or RAW12 when output_format differs" +msgstr "" + +#: ports/espressif/common-hal/mipicsi/Camera.c +msgid "output_format must be RGB565 or RGB888" +msgstr "" + +#: ports/espressif/common-hal/mipicsi/Camera.c +msgid "ISP conversion not supported on this chip" +msgstr "" + #: ports/espressif/common-hal/nvm/ByteArray.c msgid "NVS Error" msgstr "" @@ -4771,6 +4787,50 @@ msgstr "" msgid "Polygon needs at least 3 points" msgstr "" +#: shared-module/videocore/VideoBuffer.c +msgid "framebuffer went away" +msgstr "" + +#: shared-module/videocore/VideoBuffer.c +msgid "video doesn't fit in framebuffer" +msgstr "" + +#: shared-module/videocore/VideoBuffer.c +msgid "framebuffer missing required functions" +msgstr "" + +#: shared-module/videocore/VideoBuffer.c +msgid "expected FrameBuffer or %q" +msgstr "" + +#: shared-module/videocore/VideoBuffer.c +msgid "video format must be byte aligned" +msgstr "" + +#: shared-module/videocore/VideoBuffer.c +msgid "video format doesn't match framebuffer" +msgstr "" + +#: shared-module/videocore/VideoBuffer.c +msgid "video source rejected the destination" +msgstr "" + +#: shared-module/videocore/VideoBuffer.c +msgid "video in a framebuffer must be at x == 0" +msgstr "" + +#: shared-module/videocore/VideoBuffer.c +msgid "video size changed since play; stop and play again" +msgstr "" + +#: shared-module/videocore/VideoBuffer.c +msgid "VideoBuffer does not support transforms" +msgstr "" + +#: shared-module/videocore/VideoBuffer.c +msgid "VideoBuffer is playing into a different framebuffer" +msgstr "" + #: supervisor/shared/bluetooth/bluetooth.c msgid "Reconnecting" msgstr "" diff --git a/ports/espressif/CMakeLists.txt b/ports/espressif/CMakeLists.txt index 608d4f4c7b3..979afdad548 100644 --- a/ports/espressif/CMakeLists.txt +++ b/ports/espressif/CMakeLists.txt @@ -5,7 +5,22 @@ cmake_minimum_required(VERSION 3.16) set(ENV{IDF_PATH} ${CMAKE_SOURCE_DIR}/esp-idf) # The component list here determines what options we get in menuconfig and what the ninja file can build. -set(COMPONENTS bt driver esp_driver_dac esp_driver_gpio esp_driver_gptimer esp_driver_i2c esp_driver_i2s esp_driver_ledc esp_driver_pcnt esp_driver_rmt esp_driver_sdmmc esp_driver_spi esp_driver_touch_sens esp_driver_tsens esp_driver_uart esp_eth esp-tls esp_adc esp_adc_cal esp_event esp_netif esp_psram esp_security esp_wifi esptool_py freertos log lwip main mbedtls mdns pthread soc ulp wpa_supplicant esp-camera esp_lcd vfs esp_vfs_console esp_stdio sdmmc usb) +# Keep this split into separate set()/list() calls so additions merge easily. +# Core components +set(COMPONENTS bt driver esptool_py freertos log main mbedtls pthread soc) +# Peripheral drivers +list(APPEND COMPONENTS esp_driver_dac esp_driver_gpio esp_driver_gptimer esp_driver_i2c esp_driver_i2s esp_driver_ledc esp_driver_pcnt esp_driver_rmt esp_driver_sdmmc esp_driver_spi esp_driver_touch_sens esp_driver_tsens esp_driver_uart) +# Networking +list(APPEND COMPONENTS esp_eth esp-tls esp_adc esp_adc_cal esp_event esp_netif esp_psram esp_security esp_wifi lwip mdns wpa_supplicant) +# Misc +list(APPEND COMPONENTS ulp vfs esp_vfs_console esp_stdio sdmmc usb) +# Display and camera +list(APPEND COMPONENTS esp-camera esp_lcd) + +if(CIRCUITPY_MIPICSI) + list(APPEND COMPONENTS esp_driver_cam esp_hal_cam) +endif() + set(EXTRA_COMPONENT_DIRS "esp-protocols/components/mdns" "esp-camera") include($ENV{IDF_PATH}/tools/cmake/project.cmake) diff --git a/ports/espressif/Makefile b/ports/espressif/Makefile index 1d713916168..415eeb0238c 100644 --- a/ports/espressif/Makefile +++ b/ports/espressif/Makefile @@ -226,6 +226,13 @@ REGISTRATION_FUNCTIONS = \ -u esp_security_init_include_impl \ -u mbedtls_psa_crypto_init_include_impl +# Video DMA destinations come from the Python heap (user bytearrays), and +# the cache line is 64 bytes on the P4, so make GC blocks 64 bytes as well. +# supervisor/port.c aligns the PSRAM heap areas to the same value. +ifeq ($(IDF_TARGET),esp32p4) +CFLAGS += -DMICROPY_BYTES_PER_GC_BLOCK=64 +endif + # RISC-V picolibc ships optimized memcpy/memset/etc. that need forcing # (gated by CONFIG_LIBC_OPTIMIZED_MISALIGNED_ACCESS in IDF's esp_libc CMake). ifeq ($(IDF_TARGET_ARCH),riscv) @@ -504,9 +511,12 @@ endif CHIP_COMPONENTS = \ + esp_driver_cam \ + esp_driver_isp \ esp_driver_touch_sens \ esp_driver_tsens \ esp_driver_usb_serial_jtag \ + esp_hal_cam \ esp_hal_dma \ esp_hal_emac \ esp_hal_i2s \ @@ -736,6 +746,14 @@ CFLAGS += \ -isystem esp-idf/components/esp_lcd/dsi/include endif +ifneq ($(CIRCUITPY_MIPICSI),0) +CFLAGS += \ + -isystem esp-idf/components/esp_driver_cam/include \ + -isystem esp-idf/components/esp_driver_cam/csi/include \ + -isystem esp-idf/components/esp_driver_isp/include \ + -isystem esp-idf/components/esp_hal_cam/include +endif + ifneq ($(CIRCUITPY_ESPCAMERA),0) SRC_CAMERA := \ $(wildcard common-hal/espcamera/*.c) \ @@ -936,7 +954,7 @@ do-sdkconfig: $(BUILD)/esp-idf/config/sdkconfig.h QSTR_GLOBAL_REQUIREMENTS += $(BUILD)/esp-idf/config/sdkconfig.h $(BUILD)/esp-idf/config/sdkconfig.h: boards/$(BOARD)/sdkconfig boards/$(BOARD)/mpconfigboard.mk CMakeLists.txt | $(BUILD)/esp-idf $(STEPECHO) "Create $@" - $(Q)env IDF_PATH=$(IDF_PATH) IDF_COMPONENT_MANAGER=0 cmake -S . -B $(BUILD)/esp-idf -DSDKCONFIG=$(BUILD)/esp-idf/sdkconfig -DSDKCONFIG_DEFAULTS="$(SDKCONFIGS)" -DCMAKE_TOOLCHAIN_FILE=$(IDF_PATH)/tools/cmake/toolchain-$(IDF_TARGET).cmake -DIDF_TARGET=$(IDF_TARGET) -GNinja + $(Q)env IDF_PATH=$(IDF_PATH) IDF_COMPONENT_MANAGER=0 cmake -S . -B $(BUILD)/esp-idf -DSDKCONFIG=$(BUILD)/esp-idf/sdkconfig -DSDKCONFIG_DEFAULTS="$(SDKCONFIGS)" -DCMAKE_TOOLCHAIN_FILE=$(IDF_PATH)/tools/cmake/toolchain-$(IDF_TARGET).cmake -DIDF_TARGET=$(IDF_TARGET) -DCIRCUITPY_MIPICSI=$(CIRCUITPY_MIPICSI) -GNinja $(Q)$(PYTHON) tools/check-sdkconfig.py \ CIRCUITPY_DUALBANK=$(CIRCUITPY_DUALBANK) \ CIRCUITPY_STORAGE_EXTEND=$(CIRCUITPY_STORAGE_EXTEND) \ @@ -1097,6 +1115,9 @@ endif ifneq ($(CIRCUITPY_MIPIDSI),0) ESP_IDF_COMPONENTS_LINK += esp_lcd endif +ifneq ($(CIRCUITPY_MIPICSI),0) + ESP_IDF_COMPONENTS_LINK += esp_driver_cam esp_driver_isp esp_hal_cam +endif ifneq ($(CIRCUITPY_PARALLELDISPLAYBUS),0) ESP_IDF_COMPONENTS_LINK += esp_lcd endif diff --git a/ports/espressif/boards/espressif_esp32p4_function_ev/sdkconfig b/ports/espressif/boards/espressif_esp32p4_function_ev/sdkconfig index fb8986613ff..8ad98a7b081 100644 --- a/ports/espressif/boards/espressif_esp32p4_function_ev/sdkconfig +++ b/ports/espressif/boards/espressif_esp32p4_function_ev/sdkconfig @@ -1,3 +1,27 @@ +# +# Espressif IoT Development Framework Configuration +# +# Bootloader +# +CONFIG_BOOTLOADER_LOG_LEVEL_WARN=y +CONFIG_BOOTLOADER_LOG_LEVEL=2 +# end of Bootloader + +# This board ships with pre-v3 (eco2) silicon: link the 2nd stage bootloader +# into the lower memory layout used by revisions 0.x and 1.x. CONFIG_ESP32P4_SELECTS_REV_LESS_V3=y +# This board's silicon is rev v1.3; match the working BSP-demo build (REV_MIN +# v1.0) so IDF compiles the same errata/clock handling as the reference stack. CONFIG_ESP32P4_REV_MIN_100=y CONFIG_ETH_USE_ESP32_EMAC=y + +# ESP-STDIO: send the ESP-IDF console to the USB-Serial-JTAG port so bootloader +# logs are visible on the board's serial connector. +# +CONFIG_ESP_CONSOLE_USB_SERIAL_JTAG=y +CONFIG_ESP_CONSOLE_USB_SERIAL_JTAG_ENABLED=y +CONFIG_ESP_CONSOLE_UART_NUM=-1 +CONFIG_ESP_CONSOLE_ROM_SERIAL_PORT_NUM=6 +# end of ESP-STDIO + +# CONFIG_ESP_SYSTEM_MEMPROT is not set diff --git a/ports/espressif/common-hal/mipicsi/Camera.c b/ports/espressif/common-hal/mipicsi/Camera.c new file mode 100644 index 00000000000..be2e234ae3c --- /dev/null +++ b/ports/espressif/common-hal/mipicsi/Camera.c @@ -0,0 +1,539 @@ +// This file is part of the CircuitPython project: https://circuitpython.org +// +// SPDX-FileCopyrightText: Copyright (c) 2025 Jeff Epler for Adafruit Industries +// +// SPDX-License-Identifier: MIT + +#include "py/mperrno.h" +#include "py/mphal.h" +#include "py/nlr.h" +#include "py/runtime.h" + +#include "common-hal/mipicsi/Camera.h" +#include "shared-bindings/mipicsi/Camera.h" +#include "shared-bindings/mipicsi/__init__.h" +#include "bindings/espidf/__init__.h" + +#include "common-hal/mipidsi/Bus.h" + +#include +#include "esp_attr.h" +#include "esp_cache.h" +#include "esp_cam_ctlr.h" +#include "esp_private/esp_cache_private.h" +#include "esp_cam_ctlr_csi.h" +#include "hal/color_hal.h" + +// MIPI CSI receives data from a fixed, dedicated set of pins, so no pin +// configuration or claiming is done here. + +#define ISP_CLK_HZ (80 * 1000 * 1000) + +static cam_ctlr_color_t mipicsi_color_for(mipicsi_pixel_format_t format) { + switch (format) { + case MIPICSI_FORMAT_RAW8: + return CAM_CTLR_COLOR_RAW8; + case MIPICSI_FORMAT_RAW10: + return CAM_CTLR_COLOR_RAW10; + case MIPICSI_FORMAT_RAW12: + return CAM_CTLR_COLOR_RAW12; + case MIPICSI_FORMAT_RGB888: + return CAM_CTLR_COLOR_RGB888; + case MIPICSI_FORMAT_YUV422: + return CAM_CTLR_COLOR_YUV422_YUYV; + case MIPICSI_FORMAT_GRAYSCALE: + return CAM_CTLR_COLOR_GRAY8; + case MIPICSI_FORMAT_RGB565: + default: + return CAM_CTLR_COLOR_RGB565; + } +} + +static mp_int_t mipicsi_bits_per_pixel_for(mipicsi_pixel_format_t format) { + switch (format) { + case MIPICSI_FORMAT_RAW10: + return 10; + case MIPICSI_FORMAT_RAW12: + return 12; + case MIPICSI_FORMAT_RGB888: + return 24; + case MIPICSI_FORMAT_RGB565: + case MIPICSI_FORMAT_YUV422: + return 16; + case MIPICSI_FORMAT_RAW8: + case MIPICSI_FORMAT_GRAYSCALE: + default: + return 8; + } +} + +#if SOC_ISP_SUPPORTED +// The ISP color types share their FOURCC values with the CAM_CTLR color +// types, except for GRAYSCALE (no ISP grey mode) and YUV422 (the ISP input is +// UYVY, the CAM_CTLR YUYV). Only sensor RAW formats convert here. +static isp_color_t mipicsi_isp_color_for(mipicsi_pixel_format_t format) { + switch (format) { + case MIPICSI_FORMAT_RAW8: + return ISP_COLOR_RAW8; + case MIPICSI_FORMAT_RAW10: + return ISP_COLOR_RAW10; + case MIPICSI_FORMAT_RAW12: + return ISP_COLOR_RAW12; + case MIPICSI_FORMAT_RGB565: + return ISP_COLOR_RGB565; + case MIPICSI_FORMAT_RGB888: + return ISP_COLOR_RGB888; + case MIPICSI_FORMAT_YUV422: + case MIPICSI_FORMAT_GRAYSCALE: + default: + return 0; + } +} +#endif + +// Called by the driver (in ISR context) when the CSI host detects an error +// (PHY, packet, frame or CRC problems). Only installed for diagnostics: the +// counts are printed to the ROM console so they show up on the serial console. +static IRAM_ATTR bool mipicsi_on_error( + esp_cam_ctlr_handle_t handle, const esp_cam_ctlr_error_event_data_t *edata, void *user_data) { + mipicsi_camera_obj_t *self = user_data; + self->error_count++; + self->error_events |= edata->csi_host_err_evts; + if (self->error_count <= 3) { + esp_rom_printf("mipicsi: csi error #%lu events 0x%lx\n", + (unsigned long)self->error_count, (unsigned long)edata->csi_host_err_evts); + } + return false; +} + +// Called by the driver (in ISR context) to get the next buffer to DMA into. +static IRAM_ATTR bool mipicsi_on_get_new_trans( + esp_cam_ctlr_handle_t handle, esp_cam_ctlr_trans_t *trans, void *user_data) { + mipicsi_camera_obj_t *self = user_data; + + // Direct streaming: DMA the whole frame into the current destination. + // The videobuffer can change the destination between frames; the change + // is picked up here on the next frame. + // + // The controller only runs while a destination is set, so there is + // always somewhere to DMA into here; set_destination(NULL) stops the + // controller before clearing the destination. With the driver's backup + // buffer disabled, no buffer to hand over is unrecoverable (the driver + // asserts, or DMAs to address 0 in release builds), so there is no + // "give NULL" path. + trans->buffer = (void *)self->dest_base; + trans->buflen = self->buffer_len; + return false; +} + +// Called by the driver (in ISR context) when a frame has been written. +static IRAM_ATTR bool mipicsi_on_trans_finished( + esp_cam_ctlr_handle_t handle, esp_cam_ctlr_trans_t *trans, void *user_data) { + mipicsi_camera_obj_t *self = user_data; + + // Report where the frame actually landed: after a destination change + // (or a pause) the frame that was in flight lands at the previous one. + if (self->dest_base != NULL && self->dest_on_frame != NULL) { + self->dest_on_frame(self->dest_ctx, trans->buffer); + } + return false; +} + +void common_hal_mipicsi_camera_construct( + mipicsi_camera_obj_t *self, + mipicsi_pixel_format_t output_format) { + + self->width = 1; + self->height = 1; + self->pixel_format = output_format; + self->output_format = output_format; + self->dest_base = NULL; + self->dest_on_frame = NULL; + self->dest_ctx = NULL; + self->ctlr_started = false; + self->paused = false; + self->error_count = 0; + self->error_events = 0; + self->sensor = mp_const_none; + self->phy_ldo = NULL; + self->isp_proc = NULL; + self->spinlock = (portMUX_TYPE)portMUX_INITIALIZER_UNLOCKED; + // No receiver hardware yet: it is created in the videosource play step, + // where the sensor's current mode is known. +} + +// Videosource protocol play step: create the receiver hardware for the +// given (freshly read) sensor parameters and start the sensor streaming. +// When paused is true the CSI controller is left stopped, so the camera is +// active but no frames are DMAd until resume(). Raises when the camera is +// already streaming for someone else. Not a user facing method: it is driven +// by the videosource protocol (from VideoBuffer.play), so it may raise. +void common_hal_mipicsi_camera_videosource_play( + mipicsi_camera_obj_t *self, + const mipicsi_sensor_params_t *params, + bool paused) { + + if (self->handle != NULL) { + mp_raise_RuntimeError(MP_ERROR_TEXT("already playing")); + } + self->paused = paused; + + // Adopt what we need from the sensor's current mode; it may have + // changed since the Camera was made (sensors are free to reconfigure + // while they aren't being played). + self->width = params->width; + self->height = params->height; + self->bayer_order = params->bayer_order; + self->data_lanes = params->data_lanes; + self->lane_bit_rate_mbps = params->lane_bit_rate_mbps; + + const cam_ctlr_color_t color = mipicsi_color_for(self->pixel_format); + + #if SOC_ISP_SUPPORTED + if (self->output_format != self->pixel_format) { + // The sensor transmits in a different format than the one requested. + // On gen-1 P4 silicon the CSI receiver cannot convert, so route it + // through the ISP processor instead, mirroring the IDF + // esp_driver_cam csi test. + const isp_color_t isp_in = mipicsi_isp_color_for(self->pixel_format); + const isp_color_t isp_out = mipicsi_isp_color_for(self->output_format); + if (isp_in == 0 || (isp_in != ISP_COLOR_RAW8 && isp_in != ISP_COLOR_RAW10 && isp_in != ISP_COLOR_RAW12)) { + mp_raise_ValueError(MP_ERROR_TEXT("pixel_format must be RAW8, RAW10 or RAW12 when output_format differs")); + } + if (isp_out != ISP_COLOR_RGB565 && isp_out != ISP_COLOR_RGB888) { + mp_raise_ValueError(MP_ERROR_TEXT("output_format must be RGB565 or RGB888")); + } + esp_isp_processor_cfg_t isp_config = { + .clk_hz = ISP_CLK_HZ, + .input_data_source = ISP_INPUT_DATA_SOURCE_CSI, + .input_data_color_type = isp_in, + .output_data_color_type = isp_out, + .has_line_start_packet = false, + .has_line_end_packet = false, + .h_res = self->width, + .v_res = self->height, + .bayer_order = (color_raw_element_order_t)self->bayer_order, + }; + CHECK_ESP_RESULT(esp_isp_new_processor(&isp_config, &self->isp_proc)); + } + #else + if (self->output_format != self->pixel_format) { + mp_raise_NotImplementedError(MP_ERROR_TEXT("ISP conversion not supported on this chip")); + } + #endif + + // Power the MIPI D-PHY. The CSI and DSI PHYs share LDO channel 3; the + // shared reference count (see mipidsi_phy_ldo_acquire) keeps the rail up + // while either the camera or a mipidsi.Bus is alive. The sensor's I2C + // control interface is on a different rail and works without this. + self->phy_ldo = mipidsi_phy_ldo_acquire(); + + // The received images are not converted by the CSI bridge (gen-1 P4 + // silicon can't convert RAW to RGB565), so the input and output color + // types are the same and bridge conversion is bypassed. The ISP converts + // to the output format instead, so tell the CSI driver the memory-side + // frame is the ISP output depth. Without this it sizes the transfer and + // buffer from the 10 bpp RAW input (384,000 bytes for 1024x300) instead of + // the 16 bpp RGB565 output (614,400 bytes), which lets the DMA run past + // the destination. bits_per_pixel also makes the driver bypass bridge + // conversion; keep its default CSI packet data-type range. + esp_cam_ctlr_csi_config_t csi_config = { + .ctlr_id = 0, + .h_res = self->width, + .v_res = self->height, + .data_lane_num = self->data_lanes, + .lane_bit_rate_mbps = self->lane_bit_rate_mbps, + .input_data_color_type = color, + .output_data_color_type = color, + .data_type = { + .bits_per_pixel = mipicsi_bits_per_pixel_for(self->output_format), + .data_type_min = 0x12, + .data_type_max = 0x2f, + }, + // One transaction queue item; frames go straight to the current + // destination. The controller is only started while a destination + // is set, so the driver's backup buffer is never needed and its + // frame-sized allocation is skipped. + .queue_items = 1, + .bk_buffer_dis = true, + }; + + esp_err_t result = esp_cam_new_csi_ctlr(&csi_config, &self->handle); + if (result != ESP_OK) { + mipidsi_phy_ldo_release(); + self->phy_ldo = NULL; + raise_esp_error(result); + } + + // Exact frame size. Destinations are written directly (no cache-line + // padding is ever DMA'd past the last row), so they only need to be wide + // enough to hold the whole frame; the DSI framebuffer always is. When + // the ISP is in the pipeline, the CSI receiver hands its converted + // output to us. + self->buffer_len = ((size_t)self->width * self->height * mipicsi_bits_per_pixel_for(self->output_format) + 7) / 8; + + esp_cam_ctlr_evt_cbs_t cbs = { + .on_get_new_trans = mipicsi_on_get_new_trans, + .on_trans_finished = mipicsi_on_trans_finished, + .on_error = mipicsi_on_error, + }; + result = esp_cam_ctlr_register_event_callbacks(self->handle, &cbs, self); + if (result != ESP_OK) { + common_hal_mipicsi_camera_deinit(self); + self->sensor = mp_const_none; + raise_esp_error(result); + } + + result = esp_cam_ctlr_enable(self->handle); + if (result != ESP_OK) { + common_hal_mipicsi_camera_deinit(self); + self->sensor = mp_const_none; + raise_esp_error(result); + } + #if SOC_ISP_SUPPORTED + if (self->isp_proc) { + result = esp_isp_enable(self->isp_proc); + if (result != ESP_OK) { + common_hal_mipicsi_camera_deinit(self); + self->sensor = mp_const_none; + raise_esp_error(result); + } + } + #endif + + // Start the sensor streaming so frames exist to capture once a + // destination is set. Sensors' _start() is expected to be idempotent. + // The camera never calls the sensor's stop(): when a stream ends (or on + // reload) only the CSI receiver and ISP are torn down; the sensor stops + // through its own finaliser (deinit/__del__). + if (self->sensor != mp_const_none) { + nlr_buf_t nlr; + if (nlr_push(&nlr) == 0) { + mp_obj_t start_method[2]; + mp_load_method(self->sensor, MP_QSTR__start, start_method); + mp_call_method_n_kw(0, 0, start_method); + nlr_pop(); + } else { + common_hal_mipicsi_camera_deinit(self); + self->sensor = mp_const_none; + nlr_raise(nlr.ret_val); + } + } + + // Don't start the controller yet: it only runs while a destination is + // set (see common_hal_mipicsi_camera_set_destination), so no frame is + // ever captured without a place to put it. +} + +// Tear down the receiver hardware quietly: this runs from Python (a +// VideoBuffer stopped) and from background contexts (destination dropped on +// reload), so it must not raise. The sensor is left alone: its own finaliser +// stops it. Only the CSI receiver and ISP turn off. +static void mipicsi_camera_teardown(mipicsi_camera_obj_t *self) { + // Always stop, even when paused: a pause's stop finishes in the + // background, and it must be complete before the controller is disabled. + // esp_cam_ctlr_stop waits for it (and is a no-op once stopped). + if (self->handle != NULL) { + esp_cam_ctlr_stop(self->handle); + } + self->ctlr_started = false; + portENTER_CRITICAL(&self->spinlock); + self->dest_base = NULL; + self->dest_on_frame = NULL; + self->dest_ctx = NULL; + self->paused = false; + portEXIT_CRITICAL(&self->spinlock); + + nlr_buf_t nlr; + if (nlr_push(&nlr) == 0) { + if (self->handle != NULL) { + esp_cam_ctlr_disable(self->handle); + esp_cam_ctlr_del(self->handle); + self->handle = NULL; + } + #if SOC_ISP_SUPPORTED + if (self->isp_proc) { + esp_isp_disable(self->isp_proc); + esp_isp_del_processor(self->isp_proc); + self->isp_proc = NULL; + } + #endif + mipidsi_phy_ldo_release(); + self->phy_ldo = NULL; + nlr_pop(); + } else { + // Nothing to do about a failed teardown, not even a partially + // released one; swallow the exception and continue. + } +} + +void common_hal_mipicsi_camera_set_sensor(mipicsi_camera_obj_t *self, mp_obj_t sensor) { + self->sensor = sensor; +} + +void common_hal_mipicsi_camera_deinit(mipicsi_camera_obj_t *self) { + if (self->handle == NULL) { + // Not currently playing: nothing to tear down. + return; + } + + // Only turn off the CSI receiver and ISP here (including from background + // contexts such as reload): the sensor stops itself through its own + // finaliser (deinit/__del__). + mipicsi_camera_teardown(self); + self->sensor = mp_const_none; +} +// Videosource protocol stop step: tear down the receiver hardware and stop +// the sensor so its settings can be changed. Not a user facing method: it is +// driven by the videosource protocol (from VideoBuffer.stop), so it may +// raise; exceptions from the sensor's stop() propagate. +void common_hal_mipicsi_camera_videosource_stop(mipicsi_camera_obj_t *self) { + if (self->handle == NULL) { + return; // Not playing. + } + + // The receiver first: after the CSI controller stops, no more frames are + // captured, whatever the sensor does. + mipicsi_camera_teardown(self); + + // Stop the sensor (e.g. the sc2336 stops streaming and unlocks its + // mode). Sensors' _stop() is expected to be idempotent. + if (self->sensor != mp_const_none) { + mp_obj_t stop_method[2]; + mp_load_method(self->sensor, MP_QSTR__stop, stop_method); + mp_call_method_n_kw(0, 0, stop_method); + } +} + +// Videosource protocol pause step: stop the CSI controller's DMA without +// tearing down the receiver, ISP or sensor. The sensor keeps streaming, so the +// camera stays active and resume is quick; no frames are written to memory +// while stopped. Best effort: this runs from displayio group operations, so it +// must not raise. Called when the target is hidden. The destination is kept so +// resume() can use the latest one. +void common_hal_mipicsi_camera_videosource_pause(mipicsi_camera_obj_t *self) { + if (self->handle == NULL) { + return; // Not playing. + } + + // Keep the ISP enabled too: disabling it would also stop its AE/AWB/etc. + // processing and require a pipeline restart. Stopping the CSI controller + // disables the CSI bridge and its GDMA channel, which is enough to keep + // frames out of memory. + // + // Don't wait for it: the controller stops at the end of the frame being + // captured (stopping mid-frame would leave a partial frame behind), up to + // a frame time from now. That frame still lands at the current + // destination and is reported through on_frame, so the target can + // repaint over it. + if (self->ctlr_started) { + // Only fails when not started, which ctlr_started rules out. + (void)esp_cam_ctlr_csi_stop_at_frame_end(self->handle); + self->ctlr_started = false; + } + self->paused = true; +} + +// Videosource protocol resume step: start the CSI controller again at the most +// recent destination. The receiver hardware, ISP and sensor were never torn +// down. Called from Python contexts, so it may raise; paused stays set if the +// controller can't start. +void common_hal_mipicsi_camera_videosource_resume(mipicsi_camera_obj_t *self) { + if (self->handle == NULL) { + return; // Not playing. + } + + if (!self->ctlr_started && self->dest_base != NULL) { + // Starting while the pause's stop is still pending cancels it; the + // frame in flight still lands at the destination from pause time and + // on_frame reports it there. + CHECK_ESP_RESULT(esp_cam_ctlr_start(self->handle)); + self->ctlr_started = true; + } + self->paused = false; +} + +bool common_hal_mipicsi_camera_deinited(mipicsi_camera_obj_t *self) { + return self->handle == NULL; +} + +bool common_hal_mipicsi_camera_set_destination( + mipicsi_camera_obj_t *self, + uint8_t *base, + size_t stride, + void (*on_frame)(void *ctx, const void *frame), + void *ctx) { + const size_t row_bytes = ((size_t)self->width * mipicsi_bits_per_pixel_for(self->output_format) + 7) / 8; + const size_t frame_bytes = (size_t)self->height * row_bytes; + + // There is no stop value on the destination: stopping is the videosource + // stop step's job. The protocol never passes NULL, + // so this guard is only about internal misuse. + assert(base != NULL); + + // DMA destinations must be cache-line aligned: the driver syncs the + // buffer on every frame with esp_cache_msync, which rejects anything + // else. PSRAM heap areas (and with 64-byte GC blocks, every GC + // allocation in them, e.g. bytearrays) are allocated 64-byte aligned on + // this chip, so this is a safety net. + size_t cache_line = 0; + esp_cache_get_alignment(MALLOC_CAP_SPIRAM, &cache_line); + if (cache_line == 0) { + cache_line = 64; + } + if ((uintptr_t)base % cache_line != 0) { + return false; + } + + // The destination is assumed to hold the frame rows with no stride + // padding: the CSI link-list DMA only does contiguous transfers in this + // IDF. The videobuffer validates the fit before getting here; the whole + // (padded) frame must lie within the destination: rows can't wrap past + // its end, so there is no cropping or overhang. + if (stride != row_bytes) { + return false; + } + + // Size handed to the driver: the frame, cache-line padded at the end so + // the driver's per-frame sync is happy. + const size_t buffer_len = (frame_bytes + cache_line - 1) / cache_line * cache_line; + + portENTER_CRITICAL(&self->spinlock); + self->dest_base = base; + self->dest_on_frame = on_frame; + self->dest_ctx = ctx; + self->buffer_len = buffer_len; + portEXIT_CRITICAL(&self->spinlock); + + // The sensor streaming and the receiver hardware exist from play(); + // here only the controller runs or doesn't. While paused, just remember + // the new destination: resume() starts the controller there. + if (!self->paused && !self->ctlr_started) { + esp_err_t result = esp_cam_ctlr_start(self->handle); + if (result != ESP_OK) { + return false; + } + self->ctlr_started = true; + } + return true; +} + +mipicsi_pixel_format_t common_hal_mipicsi_camera_get_pixel_format(mipicsi_camera_obj_t *self) { + return self->pixel_format; +} + +mipicsi_pixel_format_t common_hal_mipicsi_camera_get_output_format(mipicsi_camera_obj_t *self) { + return self->output_format; +} + +mp_int_t common_hal_mipicsi_camera_get_width(mipicsi_camera_obj_t *self) { + return self->width; +} + +mp_int_t common_hal_mipicsi_camera_get_height(mipicsi_camera_obj_t *self) { + return self->height; +} + +mp_int_t common_hal_mipicsi_camera_get_bits_per_pixel(mipicsi_camera_obj_t *self) { + return mipicsi_bits_per_pixel_for(self->output_format); +} diff --git a/ports/espressif/common-hal/mipicsi/Camera.h b/ports/espressif/common-hal/mipicsi/Camera.h new file mode 100644 index 00000000000..86cd86bcc2f --- /dev/null +++ b/ports/espressif/common-hal/mipicsi/Camera.h @@ -0,0 +1,95 @@ +// This file is part of the CircuitPython project: https://circuitpython.org +// +// SPDX-FileCopyrightText: Copyright (c) 2025 Jeff Epler for Adafruit Industries +// +// SPDX-License-Identifier: MIT + +#pragma once + +#include + +#include "py/obj.h" + +#include "freertos/FreeRTOS.h" +#include "freertos/portmacro.h" + +#include "esp_cam_ctlr.h" +#include "esp_ldo_regulator.h" + +#if SOC_ISP_SUPPORTED +#include "driver/isp.h" +#endif + +#include "shared-bindings/mipicsi/__init__.h" + +typedef struct mipicsi_camera_obj mipicsi_camera_obj_t; + +// Sensor parameters used to build the receiver. Read off the sensor object +// when a stream starts (the sensor is free to reconfigure between streams). +typedef struct { + mp_int_t width; + mp_int_t height; + mp_int_t bayer_order; + mp_int_t data_lanes; + mp_int_t lane_bit_rate_mbps; +} mipicsi_sensor_params_t; + +struct mipicsi_camera_obj { + mp_obj_base_t base; + esp_cam_ctlr_handle_t handle; + + // Power for the MIPI D-PHY, shared with the DSI transmitter. Acquired + // when a stream starts and released when it (or the camera) ends. + esp_ldo_channel_handle_t phy_ldo; + + // Camera sensor configured to transmit to this receiver, or None. + mp_obj_t sensor; + + // ISP processor used to convert sensor frames to output_format, only + // present when the CSI receiver itself cannot convert (e.g. gen-1 P4). + mipicsi_pixel_format_t output_format; + #if SOC_ISP_SUPPORTED + isp_proc_handle_t isp_proc; + #endif + + // The receiver hardware (LDO, ISP, CSI controller) is created when a + // stream starts (see play) and torn down when it ends. The sensor's + // parameters (width, height, data_lanes, lane_bit_rate_mbps, bayer_order) + // come from the sensor at play time as well; pixel_format only when it + // wasn't overridden at construction. + // Streaming destination, set by videocore.VideoBuffer. While base is + // NULL the CSI controller is stopped: no capture buffers are used at + // all. The destination is assumed to be a whole frame: DMA'd straight + // into base, stride equal to the row size. on_frame(ctx, frame) is called + // from ISR context each time a frame lands, with the base it landed at + // (a frame in flight when base changes lands at the previous base). + // The DMA destination: set_destination requires it to be cache-line + // aligned (the driver's per-frame cache sync needs that) and reports a + // rejected destination with a false return, rather than shifting frames + // elsewhere. The frame must be contiguous. + volatile void *dest_base; + void (*dest_on_frame)(void *ctx, const void *frame); + void *dest_ctx; + bool ctlr_started; + // True between pause and resume: the receiver hardware, ISP and sensor stay + // alive, but the CSI controller is stopped so no frame is DMAd. A + // destination can still be changed while paused; resume starts the + // controller again at the latest one. + bool paused; + + // Diagnostics from the CSI host error interrupt. + volatile uint32_t error_count; + volatile uint32_t error_events; + + portMUX_TYPE spinlock; + + mp_int_t width; + mp_int_t height; + mipicsi_pixel_format_t pixel_format; + mp_int_t data_lanes; + mp_int_t lane_bit_rate_mbps; + // Bayer element order of the sensor's RAW output, only used when the ISP + // converts from a RAW format. + mp_int_t bayer_order; + size_t buffer_len; // Length handed to the driver, includes cache-alignment padding. +}; diff --git a/ports/espressif/common-hal/mipicsi/__init__.c b/ports/espressif/common-hal/mipicsi/__init__.c new file mode 100644 index 00000000000..816fea93ae9 --- /dev/null +++ b/ports/espressif/common-hal/mipicsi/__init__.c @@ -0,0 +1,7 @@ +// This file is part of the CircuitPython project: https://circuitpython.org +// +// SPDX-FileCopyrightText: Copyright (c) 2025 Jeff Epler for Adafruit Industries +// +// SPDX-License-Identifier: MIT + +#include "shared-bindings/mipicsi/__init__.h" diff --git a/ports/espressif/common-hal/mipidsi/Bus.c b/ports/espressif/common-hal/mipidsi/Bus.c index dbb559ea30f..dab781f8cc9 100644 --- a/ports/espressif/common-hal/mipidsi/Bus.c +++ b/ports/espressif/common-hal/mipidsi/Bus.c @@ -9,6 +9,39 @@ #include "py/runtime.h" #include +// Shared MIPI PHY LDO rail. DSI buses and CSI receivers both need it; it is +// acquired once and released only when the last user goes away. +static esp_ldo_channel_handle_t mipidsi_phy_ldo = NULL; +static uint8_t mipidsi_phy_ldo_users = 0; + +esp_ldo_channel_handle_t mipidsi_phy_ldo_acquire(void) { + if (mipidsi_phy_ldo_users == 0) { + esp_ldo_channel_handle_t ldo = NULL; + esp_ldo_channel_config_t ldo_config = { + .chan_id = 3, + .voltage_mv = 2500, + // Fixed-voltage (non-adjustable) acquire so both users share the + // rail. An adjustable acquire would block the other user. + .flags.adjustable = false, + }; + ESP_ERROR_CHECK(esp_ldo_acquire_channel(&ldo_config, &ldo)); + mipidsi_phy_ldo = ldo; + } + mipidsi_phy_ldo_users++; + return mipidsi_phy_ldo; +} + +void mipidsi_phy_ldo_release(void) { + if (mipidsi_phy_ldo_users == 0) { + return; + } + mipidsi_phy_ldo_users--; + if (mipidsi_phy_ldo_users == 0 && mipidsi_phy_ldo != NULL) { + esp_ldo_release_channel(mipidsi_phy_ldo); + mipidsi_phy_ldo = NULL; + } +} + void common_hal_mipidsi_bus_construct(mipidsi_bus_obj_t *self, mp_uint_t frequency, uint8_t num_lanes) { self->frequency = frequency; self->num_data_lanes = num_lanes; @@ -18,12 +51,7 @@ void common_hal_mipidsi_bus_construct(mipidsi_bus_obj_t *self, mp_uint_t frequen mp_raise_ValueError_varg(MP_ERROR_TEXT("%q in use"), MP_QSTR_mipidsi); } - esp_ldo_channel_handle_t ldo_mipi_phy = NULL; - esp_ldo_channel_config_t ldo_mipi_phy_config = { - .chan_id = 3, - .voltage_mv = 2500, - }; - ESP_ERROR_CHECK(esp_ldo_acquire_channel(&ldo_mipi_phy_config, &ldo_mipi_phy)); + self->phy_ldo = mipidsi_phy_ldo_acquire(); // Create the DSI bus esp_lcd_dsi_bus_config_t bus_config = { @@ -50,6 +78,11 @@ void common_hal_mipidsi_bus_deinit(mipidsi_bus_obj_t *self) { self->bus_handle = NULL; } + // Release our hold on the shared MIPI PHY LDO; it stays powered as long + // as a CSI receiver (or other user) still holds it. + mipidsi_phy_ldo_release(); + self->phy_ldo = NULL; + self->frequency = 0; self->num_data_lanes = 0; } diff --git a/ports/espressif/common-hal/mipidsi/Bus.h b/ports/espressif/common-hal/mipidsi/Bus.h index b6257fd7223..1920275d430 100644 --- a/ports/espressif/common-hal/mipidsi/Bus.h +++ b/ports/espressif/common-hal/mipidsi/Bus.h @@ -8,14 +8,23 @@ #include "py/obj.h" #include +#include typedef struct { mp_obj_base_t base; mp_uint_t frequency; esp_lcd_dsi_bus_handle_t bus_handle; + esp_ldo_channel_handle_t phy_ldo; uint8_t num_data_lanes; uint8_t use_count; // Up to 4 displays } mipidsi_bus_obj_t; void mipidsi_bus_increment_use_count(mipidsi_bus_obj_t *self); void mipidsi_bus_decrement_use_count(mipidsi_bus_obj_t *self); + +// Acquire the shared MIPI PHY LDO rail (LDO channel 3, 2.5 V). DSI and CSI +// users share the rail; a shared reference count keeps it powered until the +// last user releases it, so the CSI receiver can stay active while a DSI bus +// comes and goes (and vice versa). +esp_ldo_channel_handle_t mipidsi_phy_ldo_acquire(void); +void mipidsi_phy_ldo_release(void); diff --git a/ports/espressif/common-hal/mipidsi/Display.c b/ports/espressif/common-hal/mipidsi/Display.c index ea204924fe6..9ebaec1bd35 100644 --- a/ports/espressif/common-hal/mipidsi/Display.c +++ b/ports/espressif/common-hal/mipidsi/Display.c @@ -13,6 +13,7 @@ #include "bindings/espidf/__init__.h" #include #include +#include #include "py/runtime.h" // Cache write-back function (should be from rom/cache.h but it's not always available) @@ -49,6 +50,7 @@ void common_hal_mipidsi_display_construct(mipidsi_display_obj_t *self, self->framebuffer = NULL; self->dbi_io_handle = NULL; self->dpi_panel_handle = NULL; + self->backlight_inout.base.type = &mp_type_NoneType; // Create the DBI interface for sending commands esp_lcd_dbi_io_config_t dbi_config = { @@ -111,6 +113,10 @@ void common_hal_mipidsi_display_construct(mipidsi_display_obj_t *self, self->framebuffer = (uint8_t *)fb; self->framebuffer_size = width * height * (color_depth / 8); + // The panel's reset line is not managed here, matching the other displayio + // display drivers: callers pulse/hold reset themselves (e.g. with + // digitalio) when a board needs it. + // Send initialization sequence (format matches busdisplay) #define DELAY 0x80 uint32_t i = 0; @@ -141,8 +147,12 @@ void common_hal_mipidsi_display_construct(mipidsi_display_obj_t *self, CHECK_ESP_RESULT(ret); } + // Send display-ON (0x29) after the DPI/video stream is running, matching + // the reference BSP flow order. + esp_lcd_panel_io_tx_param(self->dbi_io_handle, 0x29, NULL, 0); + common_hal_time_delay_ms(20); + // Setup backlight PWM if pin is provided - self->backlight_inout.base.type = &mp_type_NoneType; if (backlight_pin != NULL && common_hal_mcu_pin_is_free(backlight_pin)) { #if (CIRCUITPY_PWMIO) pwmout_result_t result = common_hal_pwmio_pwmout_construct(&self->backlight_pwm, backlight_pin, 0, 50000, false); @@ -192,6 +202,7 @@ void common_hal_mipidsi_display_deinit(mipidsi_display_obj_t *self) { common_hal_digitalio_digitalinout_deinit(&self->backlight_inout); } + // Delete the DPI panel if (self->dpi_panel_handle != NULL) { esp_lcd_panel_del(self->dpi_panel_handle); diff --git a/ports/espressif/mpconfigport.mk b/ports/espressif/mpconfigport.mk index 87ba43d1ebf..20c8cbf8b01 100644 --- a/ports/espressif/mpconfigport.mk +++ b/ports/espressif/mpconfigport.mk @@ -404,6 +404,13 @@ CIRCUITPY_ESPCAMERA = 0 # P4 has MIPI-DSI CIRCUITPY_MIPIDSI = 1 +# P4 has MIPI-CSI +CIRCUITPY_MIPICSI = 1 + +# P4 can play MIPI-CSI video into MIPI-DSI framebuffers +CIRCUITPY_VIDEOCORE = 1 + + #### esp32s31 ######################################################### else ifeq ($(IDF_TARGET),esp32s31) diff --git a/ports/espressif/supervisor/port.c b/ports/espressif/supervisor/port.c index 91fe8cf5a2e..81eb4714db6 100644 --- a/ports/espressif/supervisor/port.c +++ b/ports/espressif/supervisor/port.c @@ -343,9 +343,11 @@ void *port_malloc(size_t size, bool dma_capable) { } void *ptr = NULL; - // Try SPIRAM first if available. + // Try SPIRAM first if available. Aligned to the GC block size so heap + // areas (and with 64-byte GC blocks, every GC allocation in them) are + // cache-line aligned, which video DMA destinations require. #ifdef CONFIG_SPIRAM - ptr = heap_caps_malloc(size, MALLOC_CAP_8BIT | MALLOC_CAP_SPIRAM); + ptr = heap_caps_aligned_alloc(MICROPY_BYTES_PER_GC_BLOCK, size, MALLOC_CAP_8BIT | MALLOC_CAP_SPIRAM); #endif if (ptr == NULL) { ptr = heap_caps_malloc(size, MALLOC_CAP_8BIT); diff --git a/py/circuitpy_defns.mk b/py/circuitpy_defns.mk index ec84d3ec381..8a4b9c59a18 100755 --- a/py/circuitpy_defns.mk +++ b/py/circuitpy_defns.mk @@ -324,6 +324,9 @@ endif ifeq ($(CIRCUITPY_MICROCONTROLLER),1) SRC_PATTERNS += microcontroller/% endif +ifeq ($(CIRCUITPY_MIPICSI),1) +SRC_PATTERNS += mipicsi/% +endif ifeq ($(CIRCUITPY_MIPIDSI),1) SRC_PATTERNS += mipidsi/% endif @@ -492,6 +495,9 @@ endif ifeq ($(CIRCUITPY_VECTORIO),1) SRC_PATTERNS += vectorio/% endif +ifeq ($(CIRCUITPY_VIDEOCORE),1) +SRC_PATTERNS += videocore/% +endif ifeq ($(CIRCUITPY_WARNINGS),1) SRC_PATTERNS += warnings/% endif @@ -586,6 +592,8 @@ SRC_COMMON_HAL_ALL = \ mdns/__init__.c \ mdns/Server.c \ mdns/RemoteService.c \ + mipicsi/__init__.c \ + mipicsi/Camera.c \ mipidsi/Bus.c \ mipidsi/Display.c \ mipidsi/__init__.c \ @@ -690,6 +698,8 @@ $(filter $(SRC_PATTERNS), \ imagecapture/ParallelImageCapture.c \ locale/__init__.c \ math/__init__.c \ + mipicsi/BayerOrder.c \ + mipicsi/PixelFormat.c \ microcontroller/ResetReason.c \ microcontroller/RunMode.c \ msgpack/__init__.c \ @@ -895,6 +905,8 @@ SRC_SHARED_MODULE_ALL = \ vectorio/Rectangle.c \ vectorio/VectorShape.c \ vectorio/__init__.c \ + videocore/VideoBuffer.c \ + videocore/__init__.c \ warnings/__init__.c \ watchdog/__init__.c \ zlib/__init__.c \ diff --git a/py/circuitpy_mpconfig.mk b/py/circuitpy_mpconfig.mk index 0fc82d03704..1d8bedebbc5 100755 --- a/py/circuitpy_mpconfig.mk +++ b/py/circuitpy_mpconfig.mk @@ -469,6 +469,9 @@ CFLAGS += -DCIRCUITPY_MEMORYMONITOR=$(CIRCUITPY_MEMORYMONITOR) CIRCUITPY_MICROCONTROLLER ?= 1 CFLAGS += -DCIRCUITPY_MICROCONTROLLER=$(CIRCUITPY_MICROCONTROLLER) +CIRCUITPY_MIPICSI ?= 0 +CFLAGS += -DCIRCUITPY_MIPICSI=$(CIRCUITPY_MIPICSI) + CIRCUITPY_MIPIDSI ?= 0 CFLAGS += -DCIRCUITPY_MIPIDSI=$(CIRCUITPY_MIPIDSI) @@ -828,6 +831,12 @@ CFLAGS += -DCIRCUITPY_ULAB=$(CIRCUITPY_ULAB) # element-wise array arithmetic roughly 1.5x slower. CIRCUITPY_ULAB_OPTIMIZE_SIZE ?= 0 +# Video pipeline: playing video sources into framebuffers. Requires a port +# with a video source such as mipicsi and a framebuffer output such as +# mipidsi. +CIRCUITPY_VIDEOCORE ?= 0 +CFLAGS += -DCIRCUITPY_VIDEOCORE=$(CIRCUITPY_VIDEOCORE) + CIRCUITPY_WARNINGS ?= $(CIRCUITPY_FULL_BUILD) CFLAGS += -DCIRCUITPY_WARNINGS=$(CIRCUITPY_WARNINGS) diff --git a/shared-bindings/adafruit_pixelbuf/PixelBuf.c b/shared-bindings/adafruit_pixelbuf/PixelBuf.c index 4144f086b98..dd5e4d7b4ea 100644 --- a/shared-bindings/adafruit_pixelbuf/PixelBuf.c +++ b/shared-bindings/adafruit_pixelbuf/PixelBuf.c @@ -31,7 +31,13 @@ static MP_NORETURN void invalid_byteorder(void) { static void parse_byteorder(mp_obj_t byteorder_obj, pixelbuf_byteorder_details_t *parsed); //| class PixelBuf: -//| """A fast RGB[W] pixel buffer for LED and similar devices.""" +//| """A fast RGB[W] pixel buffer for LED and similar devices. +//| +//| Subclasses must define ``_transmit(self, buffer)``, which `show` calls +//| with a `bytearray` of the bytes to send: the header, the pixel +//| values, and the trailer. ``_transmit`` is internal, for PixelBuf's +//| use only; user code shows pixels through `show` and `auto_write`. +//| """ //| //| def __init__( //| self, diff --git a/shared-bindings/busdisplay/BusDisplay.c b/shared-bindings/busdisplay/BusDisplay.c index 0bf171a5d08..57fc1afa71c 100644 --- a/shared-bindings/busdisplay/BusDisplay.c +++ b/shared-bindings/busdisplay/BusDisplay.c @@ -16,6 +16,9 @@ #include "shared-bindings/displayio/Group.h" #include "shared-bindings/microcontroller/Pin.h" #include "shared-bindings/util.h" +#if CIRCUITPY_VIDEOCORE +#include "shared-module/videocore/__init__.h" +#endif #include "shared-module/displayio/__init__.h" //| import displayio @@ -408,6 +411,11 @@ static mp_obj_t busdisplay_busdisplay_obj_set_root_group(mp_obj_t self_in, mp_ob displayio_group_t *group = NULL; if (group_in != mp_const_none) { group = MP_OBJ_TO_PTR(native_group(group_in)); + #if CIRCUITPY_VIDEOCORE + // Validate that draw protocol layers (e.g. a VideoBuffer) fit the + // display they will render on. + videocore_validate_root_group(group_in, self_in); + #endif } common_hal_busdisplay_busdisplay_set_root_group(self, group); diff --git a/shared-bindings/framebufferio/FramebufferDisplay.c b/shared-bindings/framebufferio/FramebufferDisplay.c index 71ce18a1b4b..da7f18e4caf 100644 --- a/shared-bindings/framebufferio/FramebufferDisplay.c +++ b/shared-bindings/framebufferio/FramebufferDisplay.c @@ -17,6 +17,9 @@ #include "shared-bindings/displayio/Group.h" #include "shared-bindings/microcontroller/Pin.h" #include "shared-bindings/util.h" +#if CIRCUITPY_VIDEOCORE +#include "shared-module/videocore/__init__.h" +#endif #include "shared-module/displayio/__init__.h" //| class FramebufferDisplay: @@ -325,6 +328,11 @@ static mp_obj_t framebufferio_framebufferdisplay_obj_set_root_group(mp_obj_t sel displayio_group_t *group = NULL; if (group_in != mp_const_none) { group = MP_OBJ_TO_PTR(native_group(group_in)); + #if CIRCUITPY_VIDEOCORE + // Validate that draw protocol layers (e.g. a VideoBuffer) fit the + // display they will render on. + videocore_validate_root_group(group_in, self_in); + #endif } bool ok = common_hal_framebufferio_framebufferdisplay_set_root_group(self, group); diff --git a/shared-bindings/mipicsi/BayerOrder.c b/shared-bindings/mipicsi/BayerOrder.c new file mode 100644 index 00000000000..9f1058dd3e9 --- /dev/null +++ b/shared-bindings/mipicsi/BayerOrder.c @@ -0,0 +1,50 @@ +// This file is part of the CircuitPython project: https://circuitpython.org +// +// SPDX-FileCopyrightText: Copyright (c) 2026 Scott Shawcroft for Adafruit Industries +// +// SPDX-License-Identifier: MIT + +#include + +#include "py/enum.h" +#include "py/obj.h" +#include "py/runtime.h" + +#include "shared-bindings/mipicsi/__init__.h" + +//| class BayerOrder: +//| """The Bayer element order of a sensor's RAW output, in the sequence a 2x2 +//| pixel block transmits (top-left first, reading order)""" +//| +//| BGGR: BayerOrder +//| """The first element is Blue, then Green, then the next row starts with Green, then Red""" +//| +//| GBRG: BayerOrder +//| """The first element is Green, then Blue, then the next row starts with Red, then Green""" +//| +//| GRBG: BayerOrder +//| """The first element is Green, then Red, then the next row starts with Blue, then Green""" +//| +//| RGGB: BayerOrder +//| """The first element is Red, then Green, then the next row starts with Green, then Blue""" +//| + +MAKE_ENUM_VALUE(mipicsi_bayer_order_type, bayer_order, BGGR, MIPICSI_BAYER_ORDER_BGGR); +MAKE_ENUM_VALUE(mipicsi_bayer_order_type, bayer_order, GBRG, MIPICSI_BAYER_ORDER_GBRG); +MAKE_ENUM_VALUE(mipicsi_bayer_order_type, bayer_order, GRBG, MIPICSI_BAYER_ORDER_GRBG); +MAKE_ENUM_VALUE(mipicsi_bayer_order_type, bayer_order, RGGB, MIPICSI_BAYER_ORDER_RGGB); + +MAKE_ENUM_MAP(mipicsi_bayer_order) { + MAKE_ENUM_MAP_ENTRY(bayer_order, BGGR), + MAKE_ENUM_MAP_ENTRY(bayer_order, GBRG), + MAKE_ENUM_MAP_ENTRY(bayer_order, GRBG), + MAKE_ENUM_MAP_ENTRY(bayer_order, RGGB), +}; + +static MP_DEFINE_CONST_DICT(mipicsi_bayer_order_locals_dict, mipicsi_bayer_order_locals_table); +MAKE_PRINTER(mipicsi, mipicsi_bayer_order); +MAKE_ENUM_TYPE(mipicsi, BayerOrder, mipicsi_bayer_order); + +mipicsi_bayer_order_t validate_bayer_order(mp_obj_t obj, qstr arg_name) { + return cp_enum_value(&mipicsi_bayer_order_type, obj, arg_name); +} diff --git a/shared-bindings/mipicsi/Camera.c b/shared-bindings/mipicsi/Camera.c new file mode 100644 index 00000000000..1106e13ea53 --- /dev/null +++ b/shared-bindings/mipicsi/Camera.c @@ -0,0 +1,244 @@ +// This file is part of the CircuitPython project: https://circuitpython.org +// +// SPDX-FileCopyrightText: Copyright (c) 2025 Jeff Epler for Adafruit Industries +// +// SPDX-License-Identifier: MIT + +#include + +#include "py/mphal.h" +#include "py/obj.h" +#include "py/objproperty.h" +#include "py/runtime.h" + +#include "shared-bindings/displayio/Bitmap.h" +#include "shared-bindings/mipicsi/__init__.h" +#include "shared-bindings/mipicsi/Camera.h" +#include "shared-bindings/util.h" +#include "shared/runtime/context_manager_helpers.h" + +#if CIRCUITPY_VIDEOCORE +#include "shared-module/videocore/__init__.h" +#endif + +//| class Camera: +//| """Receive frames from a camera sensor on a MIPI CSI-2 interface +//| +//| The camera sensor is configured separately (typically over I2C) to +//| transmit frames. The MIPI CSI-2 receiver hardware has dedicated pins, +//| so no pins are configured here. +//| +//| Pass the configured sensor to the constructor; the receiver matches +//| itself to the sensor's current mode when a `videocore.VideoBuffer` +//| plays the camera. The sensor object must provide the properties and +//| methods of ``circuitpython_typing.CameraSensor``: ``width``, ``height``, +//| ``pixel_format``, ``lane_bit_rate_mbps``, ``data_lanes`` and +//| ``bayer_order`` (a `BayerOrder`) properties plus ``_start()`` and +//| ``_stop()`` methods. Those two are internal: the camera calls them +//| itself as the stream starts and stops, so user code should not. +//| All are validated here (and again at play, so the sensor may change +//| its configuration freely while it isn't being played). +//| +//| The sensor decides the output image size, and the receiver is built +//| around it: a sensor cannot change its mode while frames are captured. +//| Sensors are started (``_start()``) when a `videocore.VideoBuffer` +//| starts playing the camera and stopped by ``_stop()`` when it stops +//| playing. While the VideoBuffer is hidden its DMA is paused, but the +//| sensor keeps streaming, so implementations should refuse to +//| reconfigure (raise) while streaming rather than change the output size +//| or format mid-stream, and have a new Camera made when the mode changes. +//| +//| Camera objects implement the internal videosource protocol, so a +//| `videocore.VideoBuffer` can stream their frames into a framebuffer. +//| """ +//| +//| def __init__( +//| self, +//| sensor: circuitpython_typing.CameraSensor, +//| *, +//| output_format: Optional[PixelFormat] = None, +//| ) -> None: +//| """ +//| Configure and initialize the MIPI CSI-2 camera receiver +//| +//| .. code-block:: python +//| +//| sensor = sc2336.SC2336(i2c) +//| camera = mipicsi.Camera(sensor) +//| +//| :param sensor: A camera sensor to receive frames from. The receiver +//| is configured to match the sensor's current mode. The sensor +//| starts streaming when a `videocore.VideoBuffer` plays the +//| camera. +//| :param output_format: The format of the captured frames. When given and different from ``pixel_format``, the ISP converts each frame as it is captured (e.g. the sensor's RAW10 to `PixelFormat.RGB565`). Only RAW formats convert. Default: no conversion. +//| """ +//| +// Read the sensor's parameters off of the sensor object. A missing or +// invalid property fails here rather than later; mp_load_attr raises +// AttributeError for any property the sensor object doesn't provide. When +// pixel_format_override is not None it replaces the sensor's reported +// transmission format. +static mipicsi_pixel_format_t mipicsi_camera_load_sensor_params( + mp_obj_t sensor, + mipicsi_sensor_params_t *params) { + params->width = mp_arg_validate_int_min(mp_obj_get_int(mp_load_attr(sensor, MP_QSTR_width)), 1, MP_QSTR_width); + params->height = mp_arg_validate_int_min(mp_obj_get_int(mp_load_attr(sensor, MP_QSTR_height)), 1, MP_QSTR_height); + params->lane_bit_rate_mbps = mp_arg_validate_int_min(mp_obj_get_int(mp_load_attr(sensor, MP_QSTR_lane_bit_rate_mbps)), 1, MP_QSTR_lane_bit_rate_mbps); + params->data_lanes = mp_arg_validate_int_range(mp_obj_get_int(mp_load_attr(sensor, MP_QSTR_data_lanes)), 1, 2, MP_QSTR_data_lanes); + params->bayer_order = validate_bayer_order(mp_load_attr(sensor, MP_QSTR_bayer_order), MP_QSTR_bayer_order); + return validate_pixel_format(mp_load_attr(sensor, MP_QSTR_pixel_format), MP_QSTR_pixel_format); +} + +static mp_obj_t mipicsi_camera_make_new(const mp_obj_type_t *type, size_t n_args, size_t n_kw, const mp_obj_t *all_args) { + mp_arg_check_num(n_args, n_kw, 1, 1, true); + mp_obj_t sensor = all_args[0]; + + enum { ARG_output_format, NUM_ARGS }; + static const mp_arg_t allowed_args[] = { + { MP_QSTR_output_format, MP_ARG_OBJ | MP_ARG_KW_ONLY, { .u_obj = MP_ROM_NONE } }, + }; + mp_arg_val_t args[MP_ARRAY_SIZE(allowed_args)]; + MP_STATIC_ASSERT(MP_ARRAY_SIZE(allowed_args) == NUM_ARGS); + mp_arg_parse_all_kw_array(0, n_kw, all_args + 1, MP_ARRAY_SIZE(allowed_args), allowed_args, args); + + // All of the sensor's parameters (including the transmission format) are + // read off of the sensor object now, so a missing or invalid property + // fails here rather than at play time. They are read again at play, + // since the sensor is free to reconfigure between streams. + mipicsi_sensor_params_t params; + mipicsi_pixel_format_t pixel_format = mipicsi_camera_load_sensor_params(sensor, ¶ms); + mipicsi_pixel_format_t output_format = pixel_format; + if (args[ARG_output_format].u_obj != mp_const_none) { + output_format = validate_pixel_format(args[ARG_output_format].u_obj, MP_QSTR_output_format); + } + + // Validate the sensor contract methods (_start/_stop) exist now too, + // since the camera drives them from C as the stream starts and stops. + mp_obj_t validated[2]; + mp_load_method(sensor, MP_QSTR__start, validated); + mp_load_method(sensor, MP_QSTR__stop, validated); + + mipicsi_camera_obj_t *self = mp_obj_malloc_with_finaliser(mipicsi_camera_obj_t, &mipicsi_camera_type); + common_hal_mipicsi_camera_construct(self, output_format); + // Keep the sensor alive while receiving from it. It is only started + // streaming when a videocore.VideoBuffer starts playing (see the + // videosource play step): until then nothing captures or transmits. + common_hal_mipicsi_camera_set_sensor(self, sensor); + return MP_OBJ_FROM_PTR(self); +} + +//| def deinit(self) -> None: +//| """Deinitialises the camera receiver and releases all memory resources for reuse.""" +//| ... +//| +static mp_obj_t mipicsi_camera_deinit(mp_obj_t self_in) { + mipicsi_camera_obj_t *self = MP_OBJ_TO_PTR(self_in); + common_hal_mipicsi_camera_deinit(self); + return mp_const_none; +} +static MP_DEFINE_CONST_FUN_OBJ_1(mipicsi_camera_deinit_obj, mipicsi_camera_deinit); + +//| def __enter__(self) -> Camera: +//| """No-op used by Context Managers.""" +//| ... +//| +//| def __exit__(self) -> None: +//| """Automatically deinitializes the hardware when exiting a context. See +//| :ref:`lifetime-and-contextmanagers` for more info.""" +//| ... +//| +//| + +static const mp_rom_map_elem_t mipicsi_camera_locals_dict_table[] = { + { MP_ROM_QSTR(MP_QSTR___del__), MP_ROM_PTR(&mipicsi_camera_deinit_obj) }, + { MP_ROM_QSTR(MP_QSTR___enter__), MP_ROM_PTR(&default___enter___obj) }, + { MP_ROM_QSTR(MP_QSTR___exit__), MP_ROM_PTR(&default___exit___obj) }, + { MP_ROM_QSTR(MP_QSTR_deinit), MP_ROM_PTR(&mipicsi_camera_deinit_obj) }, +}; + +static MP_DEFINE_CONST_DICT(mipicsi_camera_locals_dict, mipicsi_camera_locals_dict_table); + +#if CIRCUITPY_VIDEOCORE +// Implementation of the videocore videosource protocol so a VideoBuffer can +// tell the Camera where to put frames. +static void mipicsi_camera_play(mp_obj_t self_in, bool paused) { + mipicsi_camera_obj_t *self = MP_OBJ_TO_PTR(mp_obj_cast_to_native_base(self_in, &mipicsi_camera_type)); + // The camera always has a sensor from make_new on. deinit() clears it + // (the sensor stops through its own finaliser), so a missing sensor only + // means the camera was deinitialised. + if (self->sensor == mp_const_none) { + raise_deinited_error(); + } + // Re-read the sensor's mode now: it is free to reconfigure between + // streams, and only a mode change while streaming is prevented (the + // sensor refuses it itself). The paused flag is for the receiver, not the + // sensor: the sensor starts streaming either way. + mipicsi_sensor_params_t params; + self->pixel_format = mipicsi_camera_load_sensor_params(self->sensor, ¶ms); + common_hal_mipicsi_camera_videosource_play(self, ¶ms, paused); +} + +static void mipicsi_camera_stop(mp_obj_t self_in) { + mipicsi_camera_obj_t *self = MP_OBJ_TO_PTR(mp_obj_cast_to_native_base(self_in, &mipicsi_camera_type)); + common_hal_mipicsi_camera_videosource_stop(self); +} + +static void mipicsi_camera_pause(mp_obj_t self_in) { + mipicsi_camera_obj_t *self = MP_OBJ_TO_PTR(mp_obj_cast_to_native_base(self_in, &mipicsi_camera_type)); + common_hal_mipicsi_camera_videosource_pause(self); +} + +static void mipicsi_camera_resume(mp_obj_t self_in) { + mipicsi_camera_obj_t *self = MP_OBJ_TO_PTR(mp_obj_cast_to_native_base(self_in, &mipicsi_camera_type)); + common_hal_mipicsi_camera_videosource_resume(self); +} + +static bool mipicsi_camera_set_destination(mp_obj_t self_in, const videocore_destination_t *dest, + videocore_on_frame_fun on_frame, void *ctx) { + mipicsi_camera_obj_t *self = MP_OBJ_TO_PTR(mp_obj_cast_to_native_base(self_in, &mipicsi_camera_type)); + // The protocol has no stop value on the destination: stopping is stop()'s + // job. + return common_hal_mipicsi_camera_set_destination(self, dest->base, dest->stride, on_frame, ctx); +} + +static mp_int_t mipicsi_camera_get_width(mp_obj_t self_in) { + return common_hal_mipicsi_camera_get_width(mp_obj_cast_to_native_base(self_in, &mipicsi_camera_type)); +} + +static mp_int_t mipicsi_camera_get_height(mp_obj_t self_in) { + return common_hal_mipicsi_camera_get_height(mp_obj_cast_to_native_base(self_in, &mipicsi_camera_type)); +} + +static mp_int_t mipicsi_camera_get_bits_per_pixel(mp_obj_t self_in) { + return common_hal_mipicsi_camera_get_bits_per_pixel(mp_obj_cast_to_native_base(self_in, &mipicsi_camera_type)); +} + +static const videosource_p_t mipicsi_camera_videosource_protocol = { + MP_PROTO_IMPLEMENT(MP_QSTR_protocol_videosource) + .play = mipicsi_camera_play, + .pause = mipicsi_camera_pause, + .resume = mipicsi_camera_resume, + .stop = mipicsi_camera_stop, + .set_destination = mipicsi_camera_set_destination, + .get_width = mipicsi_camera_get_width, + .get_height = mipicsi_camera_get_height, + .get_bits_per_pixel = mipicsi_camera_get_bits_per_pixel, +}; + +MP_DEFINE_CONST_OBJ_TYPE( + mipicsi_camera_type, + MP_QSTR_Camera, + MP_TYPE_FLAG_HAS_SPECIAL_ACCESSORS, + make_new, mipicsi_camera_make_new, + locals_dict, &mipicsi_camera_locals_dict, + protocol, &mipicsi_camera_videosource_protocol + ); +#else +MP_DEFINE_CONST_OBJ_TYPE( + mipicsi_camera_type, + MP_QSTR_Camera, + MP_TYPE_FLAG_HAS_SPECIAL_ACCESSORS, + make_new, mipicsi_camera_make_new, + locals_dict, &mipicsi_camera_locals_dict + ); +#endif diff --git a/shared-bindings/mipicsi/Camera.h b/shared-bindings/mipicsi/Camera.h new file mode 100644 index 00000000000..c81c9c42cee --- /dev/null +++ b/shared-bindings/mipicsi/Camera.h @@ -0,0 +1,73 @@ +// This file is part of the CircuitPython project: https://circuitpython.org +// +// SPDX-FileCopyrightText: Copyright (c) 2025 Jeff Epler for Adafruit Industries +// +// SPDX-License-Identifier: MIT + +#pragma once + +#include "py/obj.h" + +#include "common-hal/mipicsi/Camera.h" +#include "shared-bindings/mipicsi/__init__.h" + +extern const mp_obj_type_t mipicsi_camera_type; + +// Record the output format (possibly the sensor's). No hardware is created +// here: that happens in play(). +void common_hal_mipicsi_camera_construct( + mipicsi_camera_obj_t *self, + mipicsi_pixel_format_t output_format); + +// Remember the sensor configured to transmit to this receiver, if any. The +// receiver reads the sensor's mode from it at play time; the sensor stops +// itself (its finaliser does), never the receiver. +void common_hal_mipicsi_camera_set_sensor(mipicsi_camera_obj_t *self, mp_obj_t sensor); + +// Videosource protocol play step: create the receiver hardware for the +// sensor's current mode and start the sensor streaming. When paused is true +// the CSI controller is left stopped, so the camera is active but no frames +// are captured until resume(). Not a user facing method: it is driven by the +// videosource protocol (from VideoBuffer.play), so it may raise. +void common_hal_mipicsi_camera_videosource_play( + mipicsi_camera_obj_t *self, + const mipicsi_sensor_params_t *params, + bool paused); + +// Videosource protocol stop step: stop the stream and the sensor. Not a user +// facing method: it is driven by the videosource protocol (from +// VideoBuffer.stop), so it may raise. +void common_hal_mipicsi_camera_videosource_stop(mipicsi_camera_obj_t *self); + +// Videosource protocol pause step: stop the CSI controller's DMA without +// tearing down the receiver, ISP or sensor, so the camera stays active but no +// frames are captured. Not a user facing method: it is driven by the +// videosource protocol (when a framebuffer target is hidden). +void common_hal_mipicsi_camera_videosource_pause(mipicsi_camera_obj_t *self); + +// Videosource protocol resume step: start the CSI controller again at the most +// recent destination after a pause. Not a user facing method: it is driven by +// the videosource protocol (when a framebuffer target is shown again). +void common_hal_mipicsi_camera_videosource_resume(mipicsi_camera_obj_t *self); + +void common_hal_mipicsi_camera_deinit(mipicsi_camera_obj_t *self); +bool common_hal_mipicsi_camera_deinited(mipicsi_camera_obj_t *self); + +// Stream frames directly into base row after row. base must lead to a +// cache-line-aligned, contiguous DMA region: when base itself isn't aligned, +// the destination is not accepted. The destination must hold the whole +// (padded) frame back to back: the caller validates the fit, since the DMA +// has no stride. Returns whether those conditions hold (false instead of +// raising; the caller may ignore the result). +bool common_hal_mipicsi_camera_set_destination( + mipicsi_camera_obj_t *self, + uint8_t *base, + size_t stride, + void (*on_frame)(void *ctx, const void *frame), + void *ctx); + +mipicsi_pixel_format_t common_hal_mipicsi_camera_get_pixel_format(mipicsi_camera_obj_t *self); +mipicsi_pixel_format_t common_hal_mipicsi_camera_get_output_format(mipicsi_camera_obj_t *self); +mp_int_t common_hal_mipicsi_camera_get_width(mipicsi_camera_obj_t *self); +mp_int_t common_hal_mipicsi_camera_get_height(mipicsi_camera_obj_t *self); +mp_int_t common_hal_mipicsi_camera_get_bits_per_pixel(mipicsi_camera_obj_t *self); diff --git a/shared-bindings/mipicsi/PixelFormat.c b/shared-bindings/mipicsi/PixelFormat.c new file mode 100644 index 00000000000..84340019745 --- /dev/null +++ b/shared-bindings/mipicsi/PixelFormat.c @@ -0,0 +1,66 @@ +// This file is part of the CircuitPython project: https://circuitpython.org +// +// SPDX-FileCopyrightText: Copyright (c) 2025 Jeff Epler for Adafruit Industries +// +// SPDX-License-Identifier: MIT + +#include + +#include "py/enum.h" +#include "py/obj.h" +#include "py/runtime.h" + +#include "shared-bindings/mipicsi/__init__.h" + +//| class PixelFormat: +//| """Format of the data in the captured frames. The camera sensor must be +//| configured (typically over I2C) to transmit the corresponding MIPI CSI-2 +//| data type.""" +//| +//| RGB565: PixelFormat +//| """A 16-bit format with 5 bits of Red, 6 bits of Green, and 5 bits of Blue (MIPI data type ``RGB565``)""" +//| +//| RGB888: PixelFormat +//| """A 24-bit format with 8 bits of Red, Green and Blue (MIPI data type ``RGB888``)""" +//| +//| GRAYSCALE: PixelFormat +//| """An 8-bit format with 8 bits of luminance (MIPI data types for 8-bit grayscale)""" +//| +//| YUV422: PixelFormat +//| """A 16-bit YUV 4:2:2 format, in the byte order the sensor transmits (MIPI data type ``YUV422 8-bit``)""" +//| +//| RAW8: PixelFormat +//| """8 bits per pixel of raw Bayer data (MIPI data type ``RAW8``)""" +//| +//| RAW10: PixelFormat +//| """10 bits per pixel of raw Bayer data (MIPI data type ``RAW10``)""" +//| +//| RAW12: PixelFormat +//| """12 bits per pixel of raw Bayer data (MIPI data type ``RAW12``)""" +//| + +MAKE_ENUM_VALUE(mipicsi_pixel_format_type, pixel_format, RGB565, MIPICSI_FORMAT_RGB565); +MAKE_ENUM_VALUE(mipicsi_pixel_format_type, pixel_format, RGB888, MIPICSI_FORMAT_RGB888); +MAKE_ENUM_VALUE(mipicsi_pixel_format_type, pixel_format, GRAYSCALE, MIPICSI_FORMAT_GRAYSCALE); +MAKE_ENUM_VALUE(mipicsi_pixel_format_type, pixel_format, YUV422, MIPICSI_FORMAT_YUV422); +MAKE_ENUM_VALUE(mipicsi_pixel_format_type, pixel_format, RAW8, MIPICSI_FORMAT_RAW8); +MAKE_ENUM_VALUE(mipicsi_pixel_format_type, pixel_format, RAW10, MIPICSI_FORMAT_RAW10); +MAKE_ENUM_VALUE(mipicsi_pixel_format_type, pixel_format, RAW12, MIPICSI_FORMAT_RAW12); + +MAKE_ENUM_MAP(mipicsi_pixel_format) { + MAKE_ENUM_MAP_ENTRY(pixel_format, RGB565), + MAKE_ENUM_MAP_ENTRY(pixel_format, RGB888), + MAKE_ENUM_MAP_ENTRY(pixel_format, GRAYSCALE), + MAKE_ENUM_MAP_ENTRY(pixel_format, YUV422), + MAKE_ENUM_MAP_ENTRY(pixel_format, RAW8), + MAKE_ENUM_MAP_ENTRY(pixel_format, RAW10), + MAKE_ENUM_MAP_ENTRY(pixel_format, RAW12), +}; + +static MP_DEFINE_CONST_DICT(mipicsi_pixel_format_locals_dict, mipicsi_pixel_format_locals_table); +MAKE_PRINTER(mipicsi, mipicsi_pixel_format); +MAKE_ENUM_TYPE(mipicsi, PixelFormat, mipicsi_pixel_format); + +mipicsi_pixel_format_t validate_pixel_format(mp_obj_t obj, qstr arg_name) { + return cp_enum_value(&mipicsi_pixel_format_type, obj, arg_name); +} diff --git a/shared-bindings/mipicsi/__init__.c b/shared-bindings/mipicsi/__init__.c new file mode 100644 index 00000000000..d46e0eedf1d --- /dev/null +++ b/shared-bindings/mipicsi/__init__.c @@ -0,0 +1,46 @@ +// This file is part of the CircuitPython project: https://circuitpython.org +// +// SPDX-FileCopyrightText: Copyright (c) 2025 Jeff Epler for Adafruit Industries +// +// SPDX-License-Identifier: MIT + +#include + +#include "py/enum.h" +#include "py/obj.h" +#include "py/runtime.h" + +#include "shared-bindings/mipicsi/__init__.h" +#include "shared-bindings/mipicsi/Camera.h" + +//| """Support for MIPI CSI-2 camera interfaces +//| +//| Captures image frames from camera sensors attached to a MIPI CSI-2 +//| receiver, such as the one in the ESP32-P4. The camera sensor itself is +//| configured separately (typically over I2C); this module only receives +//| the image data the sensor transmits. +//| +//| The image data arrives in the format the sensor transmits, described by +//| `PixelFormat`. RAW formats carry Bayer data, whose element order is +//| described by `BayerOrder`. For instance, a sensor transmitting RGB565 +//| data yields frames of RGB565 pixels. Optionally, `mipicsi.Camera` can +//| convert frames with the ISP processor (set ``output_format`` and +//| ``pixel_format``), for example from the sensor's RAW output to RGB565 for +//| display. +//| """ + +static const mp_rom_map_elem_t mipicsi_module_globals_table[] = { + { MP_ROM_QSTR(MP_QSTR___name__), MP_ROM_QSTR(MP_QSTR_mipicsi) }, + { MP_ROM_QSTR(MP_QSTR_PixelFormat), MP_ROM_PTR(&mipicsi_pixel_format_type) }, + { MP_ROM_QSTR(MP_QSTR_BayerOrder), MP_ROM_PTR(&mipicsi_bayer_order_type) }, + { MP_ROM_QSTR(MP_QSTR_Camera), MP_ROM_PTR(&mipicsi_camera_type) }, +}; + +static MP_DEFINE_CONST_DICT(mipicsi_module_globals, mipicsi_module_globals_table); + +const mp_obj_module_t mipicsi_module = { + .base = { &mp_type_module }, + .globals = (mp_obj_dict_t *)&mipicsi_module_globals, +}; + +MP_REGISTER_MODULE(MP_QSTR_mipicsi, mipicsi_module); diff --git a/shared-bindings/mipicsi/__init__.h b/shared-bindings/mipicsi/__init__.h new file mode 100644 index 00000000000..5892379ce88 --- /dev/null +++ b/shared-bindings/mipicsi/__init__.h @@ -0,0 +1,33 @@ +// This file is part of the CircuitPython project: https://circuitpython.org +// +// SPDX-FileCopyrightText: Copyright (c) 2025 Jeff Epler for Adafruit Industries +// +// SPDX-License-Identifier: MIT + +#pragma once + +#include "py/enum.h" + +extern const mp_obj_type_t mipicsi_pixel_format_type; +extern const mp_obj_type_t mipicsi_bayer_order_type; +extern const cp_enum_obj_t pixel_format_RGB565_obj; + +typedef enum { + MIPICSI_FORMAT_RAW8, + MIPICSI_FORMAT_RAW10, + MIPICSI_FORMAT_RAW12, + MIPICSI_FORMAT_RGB565, + MIPICSI_FORMAT_RGB888, + MIPICSI_FORMAT_YUV422, + MIPICSI_FORMAT_GRAYSCALE, +} mipicsi_pixel_format_t; + +typedef enum { + MIPICSI_BAYER_ORDER_BGGR, + MIPICSI_BAYER_ORDER_GBRG, + MIPICSI_BAYER_ORDER_GRBG, + MIPICSI_BAYER_ORDER_RGGB, +} mipicsi_bayer_order_t; + +mipicsi_pixel_format_t validate_pixel_format(mp_obj_t obj, qstr arg_name); +mipicsi_bayer_order_t validate_bayer_order(mp_obj_t obj, qstr arg_name); diff --git a/shared-bindings/vectorio/__init__.h b/shared-bindings/vectorio/__init__.h index c5dcc026567..cc9b523ecb1 100644 --- a/shared-bindings/vectorio/__init__.h +++ b/shared-bindings/vectorio/__init__.h @@ -23,6 +23,9 @@ typedef bool (*draw_get_dirty_area_fun)(mp_obj_t draw_protocol_self, displayio_a typedef void (*draw_update_transform_fun)(mp_obj_t draw_protocol_self, displayio_buffer_transform_t *group_transform); typedef void (*draw_finish_refresh_fun)(mp_obj_t draw_protocol_self); typedef void (*draw_set_dirty_fun)(mp_obj_t draw_protocol_self); +// Optional: propagate a group's hidden state to the layer. May be NULL; the +// group then falls back to draw_set_dirty. +typedef void (*draw_set_hidden_fun)(mp_obj_t draw_protocol_self, bool hidden); typedef bool (*draw_set_in_group_fun)(mp_obj_t draw_protocol_self, bool in_group); typedef displayio_area_t *(*draw_get_refresh_areas_fun)(mp_obj_t draw_protocol_self, displayio_area_t *tail); @@ -33,6 +36,8 @@ typedef struct _vectorio_draw_protocol_impl_t { draw_finish_refresh_fun draw_finish_refresh; draw_get_refresh_areas_fun draw_get_refresh_areas; draw_set_dirty_fun draw_set_dirty; + // May be NULL. + draw_set_hidden_fun draw_set_hidden; draw_set_in_group_fun draw_set_in_group; } vectorio_draw_protocol_impl_t; diff --git a/shared-bindings/videocore/VideoBuffer.c b/shared-bindings/videocore/VideoBuffer.c new file mode 100644 index 00000000000..6563b98af93 --- /dev/null +++ b/shared-bindings/videocore/VideoBuffer.c @@ -0,0 +1,238 @@ +// This file is part of the CircuitPython project: https://circuitpython.org +// +// SPDX-FileCopyrightText: Copyright (c) 2026 Scott Shawcroft for Adafruit Industries +// +// SPDX-License-Identifier: MIT + +#include "py/obj.h" +#include "py/objproperty.h" +#include "py/runtime.h" + +#include "shared-bindings/videocore/VideoBuffer.h" +#include "shared-bindings/util.h" +#include "shared/runtime/context_manager_helpers.h" + +//| class VideoBuffer: +//| """Stream a video source into a framebuffer or buffer. +//| +//| A VideoBuffer can be added to displayio.Groups to show on a display +//| but may have size and position limitations. +//| +//| A VideoBuffer connects a video source (an object implementing the +//| videosource protocol, such as `mipicsi.Camera`) to a target: either a +//| framebuffer (an object implementing the framebuffer protocol, such as +//| `mipidsi.Display`) or a writable buffer such as a ``bytearray`` or +//| ``memoryview``. It is the video counterpart of `audioio.AudioOut`. +//| +//| The VideoBuffer tells the source where to put frames and moves the +//| destination as needed; while shown, the source streams continuously +//| into it. While playing, the target always holds the newest frame. +//| +//| When the target is a framebuffer, position the video with ``x`` and +//| ``y``: they may be changed at any time, including while playing, which +//| moves where the source puts its next frame. Some implementations may +//| restrict size and position. Setting position directly will raise an +//| error but indirectly through Group may result in weird output. For +//| example, on ESP32-P4 a VideoBuffer written into the framebuffer +//| directly must be full width because the ESP can't handle a stride +//| without an intermediate buffer. +//| +//| When the target is a plain buffer, frames are written compactly (row +//| stride equals ``width * bits_per_pixel / 8``) and can be read from +//| Python at any time — for example to process or save camera pixels. +//| +//| Two complete examples are in the `videocore` module overview: one +//| streaming a camera directly into a display, and one capturing a +//| 320x240 window into a ``bytearray`` that displayio places in a Group. +//| """ +//| +//| def __init__( +//| self, +//| framebuffer: Union[ +//| circuitpython_typing.FrameBuffer, circuitpython_typing.WriteableBuffer +//| ], +//| ) -> None: +//| """Create a VideoBuffer that plays video into the given framebuffer +//| +//| :param framebuffer: The framebuffer to copy video frames into, or a +//| writable buffer such as a ``bytearray`` or ``memoryview``. +//| """ +//| +static mp_obj_t videocore_videobuffer_make_new(const mp_obj_type_t *type, size_t n_args, size_t n_kw, const mp_obj_t *all_args) { + enum { ARG_framebuffer, NUM_ARGS }; + static const mp_arg_t allowed_args[] = { + { MP_QSTR_framebuffer, MP_ARG_REQUIRED | MP_ARG_OBJ }, + }; + + mp_arg_val_t args[MP_ARRAY_SIZE(allowed_args)]; + MP_STATIC_ASSERT(MP_ARRAY_SIZE(allowed_args) == NUM_ARGS); + mp_arg_parse_all_kw_array(n_args, n_kw, all_args, MP_ARRAY_SIZE(allowed_args), allowed_args, args); + + videocore_videobuffer_obj_t *self = mp_obj_malloc_with_finaliser(videocore_videobuffer_obj_t, &videocore_videobuffer_type); + common_hal_videocore_videobuffer_construct(self, args[ARG_framebuffer].u_obj); + return MP_OBJ_FROM_PTR(self); +} + +//| def play(self, source: circuitpython_typing.VideoSource) -> None: +//| """Begin streaming the given video source into the target. +//| +//| The source streams continuously, writing each new frame where the +//| VideoBuffer tells it to, replacing the previous one. The source's +//| frame format must be byte aligned. When the target is a +//| framebuffer, its bits per pixel must match the framebuffer color +//| depth and the whole frame must fit within the framebuffer at the +//| current ``x``/``y`` position (rows can't wrap past its end). When +//| the target is a plain buffer, it must be large enough for a whole +//| frame. The source's size and format are then fixed for the +//| duration of the play; sources that change them require ``stop`` +//| and a new ``play``. +//| +//| :param ~circuitpython_typing.VideoSource source: The video source to +//| play, such as `mipicsi.Camera` +//| """ +//| +static mp_obj_t videocore_videobuffer_play(mp_obj_t self_in, mp_obj_t source) { + videocore_videobuffer_obj_t *self = MP_OBJ_TO_PTR(self_in); + common_hal_videocore_videobuffer_play(self, source); + return mp_const_none; +} +static MP_DEFINE_CONST_FUN_OBJ_2(videocore_videobuffer_play_obj, videocore_videobuffer_play); + +//| def stop(self) -> None: +//| """Stop playing the video source.""" +//| ... +//| +static mp_obj_t videocore_videobuffer_stop(mp_obj_t self_in) { + videocore_videobuffer_obj_t *self = MP_OBJ_TO_PTR(self_in); + common_hal_videocore_videobuffer_stop(self); + return mp_const_none; +} +static MP_DEFINE_CONST_FUN_OBJ_1(videocore_videobuffer_stop_obj, videocore_videobuffer_stop); + +//| playing: bool +//| """True when a video source is being played, False otherwise. (read-only)""" +//| +static mp_obj_t videocore_videobuffer_get_playing(mp_obj_t self_in) { + videocore_videobuffer_obj_t *self = MP_OBJ_TO_PTR(self_in); + return mp_obj_new_bool(common_hal_videocore_videobuffer_get_playing(self)); +} +static MP_DEFINE_CONST_FUN_OBJ_1(videocore_videobuffer_get_playing_obj, videocore_videobuffer_get_playing); +MP_PROPERTY_GETTER(videocore_videobuffer_playing_obj, + (mp_obj_t)&videocore_videobuffer_get_playing_obj); + +//| source: Optional[circuitpython_typing.VideoSource] +//| """The video source being played, or None when not playing. (read-only)""" +//| +static mp_obj_t videocore_videobuffer_get_source(mp_obj_t self_in) { + videocore_videobuffer_obj_t *self = MP_OBJ_TO_PTR(self_in); + return common_hal_videocore_videobuffer_get_source(self); +} +static MP_DEFINE_CONST_FUN_OBJ_1(videocore_videobuffer_get_source_obj, videocore_videobuffer_get_source); +MP_PROPERTY_GETTER(videocore_videobuffer_source_obj, + (mp_obj_t)&videocore_videobuffer_get_source_obj); + +//| x: int +//| """The x position of the video within the display. Framebuffer targets +//| must be at x == 0 (video sources DMA whole rows): ``ValueError`` +//| otherwise. Plain buffer targets only use it to place the video in a +//| Group: it doesn't change where frames land within the buffer.""" +//| +//| y: int +//| """The y position of the video within the display.""" +//| +//| +static void videocore_videobuffer_get_position(mp_obj_t self_in, mp_int_t *x, mp_int_t *y) { + videocore_videobuffer_obj_t *self = MP_OBJ_TO_PTR(self_in); + *x = self->x; + *y = self->y; +} + +static mp_obj_t videocore_videobuffer_get_x(mp_obj_t self_in) { + mp_int_t x, y; + videocore_videobuffer_get_position(self_in, &x, &y); + return MP_OBJ_NEW_SMALL_INT(x); +} +static MP_DEFINE_CONST_FUN_OBJ_1(videocore_videobuffer_get_x_obj, videocore_videobuffer_get_x); + +static mp_obj_t videocore_videobuffer_set_x(mp_obj_t self_in, mp_obj_t value) { + videocore_videobuffer_obj_t *self = MP_OBJ_TO_PTR(self_in); + mp_int_t x = mp_obj_get_int(value); + common_hal_videocore_videobuffer_set_position(self, x, self->y); + return mp_const_none; +} +static MP_DEFINE_CONST_FUN_OBJ_2(videocore_videobuffer_set_x_obj, videocore_videobuffer_set_x); +MP_PROPERTY_GETSET(videocore_videobuffer_x_obj, + (mp_obj_t)&videocore_videobuffer_get_x_obj, + (mp_obj_t)&videocore_videobuffer_set_x_obj); + +static mp_obj_t videocore_videobuffer_get_y(mp_obj_t self_in) { + mp_int_t x, y; + videocore_videobuffer_get_position(self_in, &x, &y); + return MP_OBJ_NEW_SMALL_INT(y); +} +static MP_DEFINE_CONST_FUN_OBJ_1(videocore_videobuffer_get_y_obj, videocore_videobuffer_get_y); + +static mp_obj_t videocore_videobuffer_set_y(mp_obj_t self_in, mp_obj_t value) { + videocore_videobuffer_obj_t *self = MP_OBJ_TO_PTR(self_in); + mp_int_t y = mp_obj_get_int(value); + common_hal_videocore_videobuffer_set_position(self, self->x, y); + return mp_const_none; +} +static MP_DEFINE_CONST_FUN_OBJ_2(videocore_videobuffer_set_y_obj, videocore_videobuffer_set_y); +MP_PROPERTY_GETSET(videocore_videobuffer_y_obj, + (mp_obj_t)&videocore_videobuffer_get_y_obj, + (mp_obj_t)&videocore_videobuffer_set_y_obj); + +//| def deinit(self) -> None: +//| """Stop playing. The video source stops itself.""" +//| ... +//| +//| def __enter__(self) -> VideoBuffer: +//| """No-op used by Context Managers.""" +//| ... +//| +//| def __exit__(self) -> None: +//| """Automatically deinitializes the hardware when exiting a context. See +//| :ref:`lifetime-and-contextmanagers` for more info.""" +//| ... +//| +static mp_obj_t videocore_videobuffer_deinit(mp_obj_t self_in) { + videocore_videobuffer_obj_t *self = MP_OBJ_TO_PTR(self_in); + common_hal_videocore_videobuffer_stop(self); + return mp_const_none; +} +static MP_DEFINE_CONST_FUN_OBJ_1(videocore_videobuffer_deinit_obj, videocore_videobuffer_deinit); + +// The finaliser must not raise (it runs during garbage collection), so stop +// without one instead of deinit(): a failed stop stays best-effort, matching +// the source's own finaliser. +static mp_obj_t videocore_videobuffer___del__(mp_obj_t self_in) { + videocore_videobuffer_obj_t *self = MP_OBJ_TO_PTR(self_in); + videocore_videobuffer_stop_quiet(self); + return mp_const_none; +} +static MP_DEFINE_CONST_FUN_OBJ_1(videocore_videobuffer___del___obj, videocore_videobuffer___del__); + +static const mp_rom_map_elem_t videocore_videobuffer_locals_dict_table[] = { + { MP_ROM_QSTR(MP_QSTR___del__), MP_ROM_PTR(&videocore_videobuffer___del___obj) }, + { MP_ROM_QSTR(MP_QSTR___enter__), MP_ROM_PTR(&default___enter___obj) }, + { MP_ROM_QSTR(MP_QSTR___exit__), MP_ROM_PTR(&default___exit___obj) }, + { MP_ROM_QSTR(MP_QSTR_deinit), MP_ROM_PTR(&videocore_videobuffer_deinit_obj) }, + { MP_ROM_QSTR(MP_QSTR_play), MP_ROM_PTR(&videocore_videobuffer_play_obj) }, + { MP_ROM_QSTR(MP_QSTR_stop), MP_ROM_PTR(&videocore_videobuffer_stop_obj) }, + { MP_ROM_QSTR(MP_QSTR_playing), MP_ROM_PTR(&videocore_videobuffer_playing_obj) }, + { MP_ROM_QSTR(MP_QSTR_source), MP_ROM_PTR(&videocore_videobuffer_source_obj) }, + { MP_ROM_QSTR(MP_QSTR_x), MP_ROM_PTR(&videocore_videobuffer_x_obj) }, + { MP_ROM_QSTR(MP_QSTR_y), MP_ROM_PTR(&videocore_videobuffer_y_obj) }, +}; + +static MP_DEFINE_CONST_DICT(videocore_videobuffer_locals_dict, videocore_videobuffer_locals_dict_table); + +MP_DEFINE_CONST_OBJ_TYPE( + videocore_videobuffer_type, + MP_QSTR_VideoBuffer, + MP_TYPE_FLAG_HAS_SPECIAL_ACCESSORS, + make_new, videocore_videobuffer_make_new, + locals_dict, &videocore_videobuffer_locals_dict, + protocol, &videocore_videobuffer_draw_protocol + ); diff --git a/shared-bindings/videocore/VideoBuffer.h b/shared-bindings/videocore/VideoBuffer.h new file mode 100644 index 00000000000..927169bc4b3 --- /dev/null +++ b/shared-bindings/videocore/VideoBuffer.h @@ -0,0 +1,11 @@ +// This file is part of the CircuitPython project: https://circuitpython.org +// +// SPDX-FileCopyrightText: Copyright (c) 2026 Scott Shawcroft for Adafruit Industries +// +// SPDX-License-Identifier: MIT + +#pragma once + +#include "shared-module/videocore/VideoBuffer.h" + +extern const mp_obj_type_t videocore_videobuffer_type; diff --git a/shared-bindings/videocore/__init__.c b/shared-bindings/videocore/__init__.c new file mode 100644 index 00000000000..83c8487941a --- /dev/null +++ b/shared-bindings/videocore/__init__.c @@ -0,0 +1,91 @@ +// This file is part of the CircuitPython project: https://circuitpython.org +// +// SPDX-FileCopyrightText: Copyright (c) 2026 Scott Shawcroft for Adafruit Industries +// +// SPDX-License-Identifier: MIT + +#include "py/obj.h" +#include "py/runtime.h" + +#include "shared-bindings/videocore/VideoBuffer.h" +#include "shared-bindings/videocore/__init__.h" + +//| """Support for playing video sources +//| +//| This module provides the video pipeline shared between video sources and +//| video outputs. It is the video equivalent of the `audiocore` module. +//| +//| Video sources, such as `mipicsi.Camera`, implement the internal videosource +//| protocol. A `VideoBuffer` tells a video source where to put its frames in a +//| framebuffer, such as `mipidsi.Display`, and can move it as needed. +//| +//| Stream a MIPI CSI camera live into the board's display (a MIPI DSI panel +//| on ESP32-P4-Function-EV, for example): +//| +//| .. code-block:: python +//| +//| import board +//| import displayio +//| import mipicsi +//| import sc2336 +//| import videocore +//| +//| display = board.DISPLAY() +//| framebuffer = display.framebuffer +//| sensor = sc2336.SC2336(board.I2C()) +//| sensor.mode = sc2336.Mode.HD_1280x720_RAW10_30FPS +//| # Trim the sensor output to the display size. +//| sensor.width = display.width +//| sensor.height = display.height +//| camera = mipicsi.Camera(sensor, output_format=mipicsi.PixelFormat.RGB565) +//| +//| video = videocore.VideoBuffer(framebuffer) +//| video.play(camera) +//| root = displayio.Group() +//| root.append(video) +//| display.root_group = root +//| +//| Capture a smaller part of the sensor's image into a `bytearray` and let +//| displayio place it in a Group, together with other layers. displayio bulk +//| copies new frames into the group's area of the display (no color +//| conversion): +//| +//| .. code-block:: python +//| +//| import board +//| import displayio +//| import mipicsi +//| import sc2336 +//| import videocore +//| +//| sensor = sc2336.SC2336(board.I2C()) +//| sensor.mode = sc2336.Mode.VGA_640x480_RAW10_50FPS +//| # Trim the sensor output to a 320x240 window. +//| sensor.width = 320 +//| sensor.height = 240 +//| camera = mipicsi.Camera(sensor, output_format=mipicsi.PixelFormat.RGB565) +//| +//| capture = videocore.VideoBuffer(bytearray(320 * 240 * 2)) +//| capture.play(camera) +//| +//| root = displayio.Group() +//| root.append(capture) +//| capture.x = 350 # position of the live video within the group +//| capture.y = 180 +//| board.DISPLAY().root_group = root # live video joins the displayio scene +//| """ +//| + +static const mp_rom_map_elem_t videocore_module_globals_table[] = { + { MP_ROM_QSTR(MP_QSTR___name__), MP_ROM_QSTR(MP_QSTR_videocore) }, + { MP_ROM_QSTR(MP_QSTR_VideoBuffer), MP_ROM_PTR(&videocore_videobuffer_type) }, +}; + +static MP_DEFINE_CONST_DICT(videocore_module_globals, videocore_module_globals_table); + +const mp_obj_module_t videocore_module = { + .base = { &mp_type_module }, + .globals = (mp_obj_dict_t *)&videocore_module_globals, +}; + +MP_REGISTER_MODULE(MP_QSTR_videocore, videocore_module); diff --git a/shared-bindings/videocore/__init__.h b/shared-bindings/videocore/__init__.h new file mode 100644 index 00000000000..d309fcc838b --- /dev/null +++ b/shared-bindings/videocore/__init__.h @@ -0,0 +1,9 @@ +// This file is part of the CircuitPython project: https://circuitpython.org +// +// SPDX-FileCopyrightText: Copyright (c) 2026 Scott Shawcroft for Adafruit Industries +// +// SPDX-License-Identifier: MIT + +#pragma once + +#include "shared-module/videocore/__init__.h" diff --git a/shared-module/displayio/Group.c b/shared-module/displayio/Group.c index 84ebffdab6d..2f96a0fd578 100644 --- a/shared-module/displayio/Group.c +++ b/shared-module/displayio/Group.c @@ -10,7 +10,7 @@ #include "py/objlist.h" #include "shared-bindings/displayio/TileGrid.h" -#if CIRCUITPY_VECTORIO +#if CIRCUITPY_VECTORIO || CIRCUITPY_VIDEOCORE #include "shared-bindings/vectorio/VectorShape.h" #endif @@ -52,11 +52,15 @@ void common_hal_displayio_group_set_hidden(displayio_group_t *self, bool hidden) displayio_group_set_hidden_by_parent(layer, hidden); continue; } - #if CIRCUITPY_VECTORIO + #if CIRCUITPY_VECTORIO || CIRCUITPY_VIDEOCORE const vectorio_draw_protocol_t *draw_protocol = mp_proto_get(MP_QSTR_protocol_draw, self->members->items[i]); if (draw_protocol != NULL) { layer = draw_protocol->draw_get_protocol_self(self->members->items[i]); - draw_protocol->draw_protocol_impl->draw_set_dirty(layer); + if (draw_protocol->draw_protocol_impl->draw_set_hidden != NULL) { + draw_protocol->draw_protocol_impl->draw_set_hidden(layer, hidden); + } else { + draw_protocol->draw_protocol_impl->draw_set_dirty(layer); + } continue; } #endif @@ -87,11 +91,15 @@ void displayio_group_set_hidden_by_parent(displayio_group_t *self, bool hidden) displayio_group_set_hidden_by_parent(layer, hidden); continue; } - #if CIRCUITPY_VECTORIO + #if CIRCUITPY_VECTORIO || CIRCUITPY_VIDEOCORE const vectorio_draw_protocol_t *draw_protocol = mp_proto_get(MP_QSTR_protocol_draw, self->members->items[i]); if (draw_protocol != NULL) { layer = draw_protocol->draw_get_protocol_self(self->members->items[i]); - draw_protocol->draw_protocol_impl->draw_set_dirty(layer); + if (draw_protocol->draw_protocol_impl->draw_set_hidden != NULL) { + draw_protocol->draw_protocol_impl->draw_set_hidden(layer, hidden); + } else { + draw_protocol->draw_protocol_impl->draw_set_dirty(layer); + } continue; } #endif @@ -141,16 +149,21 @@ bool displayio_group_get_previous_area(displayio_group_t *self, displayio_area_t } static void _update_child_transforms(displayio_group_t *self) { - if (!self->in_group) { - return; - } + // Draw protocol layers only act on a transform when we're attached to a + // display. TileGrids and nested Groups also need a transform to preserve + // their group membership, so pass the (zeroed-out when inactive) + // transform to them. + #if CIRCUITPY_VECTORIO || CIRCUITPY_VIDEOCORE + displayio_buffer_transform_t *draw_transform = + self->active ? &self->absolute_transform : NULL; + #endif for (size_t i = 0; i < self->members->len; i++) { mp_obj_t layer; - #if CIRCUITPY_VECTORIO + #if CIRCUITPY_VECTORIO || CIRCUITPY_VIDEOCORE const vectorio_draw_protocol_t *draw_protocol = mp_proto_get(MP_QSTR_protocol_draw, self->members->items[i]); if (draw_protocol != NULL) { layer = draw_protocol->draw_get_protocol_self(self->members->items[i]); - draw_protocol->draw_protocol_impl->draw_update_transform(layer, &self->absolute_transform); + draw_protocol->draw_protocol_impl->draw_update_transform(layer, draw_transform); continue; } #endif @@ -171,8 +184,8 @@ static void _update_child_transforms(displayio_group_t *self) { void displayio_group_update_transform(displayio_group_t *self, const displayio_buffer_transform_t *parent_transform) { - self->in_group = parent_transform != NULL; - if (self->in_group) { + self->active = parent_transform != NULL && parent_transform->scale != 0; + if (self->active) { int16_t x = self->x; int16_t y = self->y; if (parent_transform->transpose_xy) { @@ -188,6 +201,10 @@ void displayio_group_update_transform(displayio_group_t *self, self->absolute_transform.mirror_y = parent_transform->mirror_y; self->absolute_transform.scale = parent_transform->scale * self->scale; + } else { + // Clear the transform so inactive layers can't act on a stale one. + displayio_buffer_transform_t no_transform = {}; + self->absolute_transform = no_transform; } _update_child_transforms(self); } @@ -248,14 +265,15 @@ void common_hal_displayio_group_set_y(displayio_group_t *self, mp_int_t y) { static void _add_layer(displayio_group_t *self, mp_obj_t layer) { check_readonly(self); - #if CIRCUITPY_VECTORIO + #if CIRCUITPY_VECTORIO || CIRCUITPY_VIDEOCORE const vectorio_draw_protocol_t *draw_protocol = mp_proto_get(MP_QSTR_protocol_draw, layer); if (draw_protocol != NULL) { mp_obj_t protocol_self = draw_protocol->draw_get_protocol_self(layer); if (draw_protocol->draw_protocol_impl->draw_set_in_group(protocol_self, true)) { mp_raise_ValueError(MP_ERROR_TEXT("Layer already in a group")); } - draw_protocol->draw_protocol_impl->draw_update_transform(protocol_self, &self->absolute_transform); + draw_protocol->draw_protocol_impl->draw_update_transform( + protocol_self, self->active ? &self->absolute_transform : NULL); return; } #endif @@ -293,7 +311,7 @@ static void _remove_layer(displayio_group_t *self, size_t index) { mp_obj_t layer; displayio_area_t layer_area; bool rendered_last_frame = false; - #if CIRCUITPY_VECTORIO + #if CIRCUITPY_VECTORIO || CIRCUITPY_VIDEOCORE const vectorio_draw_protocol_t *draw_protocol = mp_proto_get(MP_QSTR_protocol_draw, self->members->items[index]); if (draw_protocol != NULL) { layer = draw_protocol->draw_get_protocol_self(self->members->items[index]); @@ -368,6 +386,7 @@ void displayio_group_construct(displayio_group_t *self, mp_obj_list_t *members, self->item_removed = false; self->scale = scale; self->in_group = false; + self->active = false; self->readonly = false; } @@ -377,7 +396,7 @@ bool displayio_group_fill_area(displayio_group_t *self, const _displayio_colorsp if (self->hidden == false) { for (int32_t i = self->members->len - 1; i >= 0; i--) { mp_obj_t layer; - #if CIRCUITPY_VECTORIO + #if CIRCUITPY_VECTORIO || CIRCUITPY_VIDEOCORE const vectorio_draw_protocol_t *draw_protocol = mp_proto_get(MP_QSTR_protocol_draw, self->members->items[i]); if (draw_protocol != NULL) { layer = draw_protocol->draw_get_protocol_self(self->members->items[i]); @@ -412,7 +431,7 @@ void displayio_group_finish_refresh(displayio_group_t *self) { self->item_removed = false; for (int32_t i = self->members->len - 1; i >= 0; i--) { mp_obj_t layer; - #if CIRCUITPY_VECTORIO + #if CIRCUITPY_VECTORIO || CIRCUITPY_VIDEOCORE const vectorio_draw_protocol_t *draw_protocol = mp_proto_get(MP_QSTR_protocol_draw, self->members->items[i]); if (draw_protocol != NULL) { layer = draw_protocol->draw_get_protocol_self(self->members->items[i]); @@ -443,7 +462,7 @@ static displayio_area_t *group_get_refresh_areas_impl(displayio_group_t *self, d for (int32_t i = self->members->len - 1; i >= 0; i--) { mp_obj_t layer; - #if CIRCUITPY_VECTORIO + #if CIRCUITPY_VECTORIO || CIRCUITPY_VIDEOCORE const vectorio_draw_protocol_t *draw_protocol = mp_proto_get(MP_QSTR_protocol_draw, self->members->items[i]); if (draw_protocol != NULL) { layer = draw_protocol->draw_get_protocol_self(self->members->items[i]); diff --git a/shared-module/displayio/Group.h b/shared-module/displayio/Group.h index 9ff6c7dca0f..d0a3bd353ec 100644 --- a/shared-module/displayio/Group.h +++ b/shared-module/displayio/Group.h @@ -24,10 +24,11 @@ typedef struct { uint16_t scale; bool item_removed : 1; bool in_group : 1; + bool active : 1; bool hidden : 1; bool hidden_by_parent : 1; bool readonly : 1; - uint8_t padding : 3; + uint8_t padding : 2; } displayio_group_t; void displayio_group_construct(displayio_group_t *self, mp_obj_list_t *members, uint32_t scale, mp_int_t x, mp_int_t y); diff --git a/shared-module/displayio/display_core.c b/shared-module/displayio/display_core.c index dfcddde7796..66837748eb3 100644 --- a/shared-module/displayio/display_core.c +++ b/shared-module/displayio/display_core.c @@ -112,6 +112,10 @@ bool displayio_display_core_set_root_group(displayio_display_core_t *self, displ return false; } if (self->current_group != NULL) { + // Deactivate the outgoing tree recursively. This clears its transforms + // and tells layers (such as a VideoBuffer) that they are no longer + // shown. + displayio_group_update_transform(self->current_group, NULL); self->current_group->in_group = false; } @@ -161,6 +165,9 @@ void displayio_display_core_finish_refresh(displayio_display_core_t *self) { void release_display_core(displayio_display_core_t *self) { if (self->current_group != NULL) { + // Deactivate the tree so layers stop acting on it (e.g. a VideoBuffer + // pauses its source). + displayio_group_update_transform(self->current_group, NULL); self->current_group->in_group = false; } } diff --git a/shared-module/framebufferio/FramebufferDisplay.c b/shared-module/framebufferio/FramebufferDisplay.c index 8116f4b0347..1cfb636420a 100644 --- a/shared-module/framebufferio/FramebufferDisplay.c +++ b/shared-module/framebufferio/FramebufferDisplay.c @@ -182,7 +182,23 @@ static bool _refresh_area(framebufferio_framebufferdisplay_obj_t *self, const di memset(mask, 0, mask_length * sizeof(mask[0])); memset(buffer, 0, buffer_size * sizeof(buffer[0])); - displayio_display_core_fill_area(&self->core, &subrectangle, mask, buffer); + bool covered = displayio_display_core_fill_area(&self->core, &subrectangle, mask, buffer); + + // A framebuffer-target VideoBuffer streams its pixels straight into + // the framebuffer, so when it is the only layer covering this + // subrectangle it returns full coverage without claiming any pixels. + // Nothing was composed into the scratch, so writing it back would + // erase the video; leave the framebuffer alone and just mark the rows. + bool write_back = true; + if (covered) { + write_back = false; + for (uint32_t m = 0; m < mask_length; m++) { + if (mask[m] != 0) { + write_back = true; + break; + } + } + } uint8_t *buf = (uint8_t *)self->bufinfo.buf, *endbuf = buf + self->bufinfo.len; (void)endbuf; // Hint to compiler that endbuf is "used" even if NDEBUG @@ -196,7 +212,9 @@ static bool _refresh_area(framebufferio_framebufferdisplay_obj_t *self, const di for (uint16_t i = subrectangle.y1; i < subrectangle.y2; i++) { assert(dest >= buf && dest < endbuf && dest + rowsize <= endbuf); MARK_ROW_DIRTY(i); - memcpy(dest, src, rowsize); + if (write_back) { + memcpy(dest, src, rowsize); + } dest += rowstride; src += rowsize; } diff --git a/shared-module/videocore/VideoBuffer.c b/shared-module/videocore/VideoBuffer.c new file mode 100644 index 00000000000..1095640b669 --- /dev/null +++ b/shared-module/videocore/VideoBuffer.c @@ -0,0 +1,828 @@ +// This file is part of the CircuitPython project: https://circuitpython.org +// +// SPDX-FileCopyrightText: Copyright (c) 2026 Scott Shawcroft for Adafruit Industries +// +// SPDX-License-Identifier: MIT + +#include + +#include "shared-module/videocore/VideoBuffer.h" + +#include "py/runtime.h" + +#include "shared-bindings/displayio/Group.h" +#include "shared-bindings/videocore/VideoBuffer.h" +#include "shared-bindings/videocore/__init__.h" +#include "shared-module/displayio/Group.h" +#include "shared-module/videocore/__init__.h" + +#if CIRCUITPY_FRAMEBUFFERIO +#include "shared-bindings/framebufferio/FramebufferDisplay.h" +#endif + +#define FB_GETTER(method, default_value) \ + (self->framebuffer_protocol->method \ + ? self->framebuffer_protocol->method(self->framebuffer) \ + : (default_value)) + +// Called from interrupt context when the source lands a frame. The frame +// landed in stream_base, the buffer the accepted destination gave (set when +// the last set_destination returned true), so keep it tiny. +static void videocore_videobuffer_on_frame(void *ctx, const void *frame) { + videocore_videobuffer_obj_t *self = ctx; + if (frame == self->stream_base) { + self->frame_ready = true; + return; + } + // It landed at a destination we have moved away from or paused: the + // frame that was being written at the time. Queue it for the refresh pass + // to repaint, since that area may already have been repainted. + const uint8_t head = self->stray_head; + if ((uint8_t)(head - self->stray_tail) >= VIDEOCORE_STRAY_FRAMES) { + self->stray_overflow = true; + return; + } + self->stray_frames[head % VIDEOCORE_STRAY_FRAMES] = frame; + __atomic_store_n(&self->stray_head, (uint8_t)(head + 1), __ATOMIC_RELEASE); +} + +// Union the screen area of a framebuffer-target frame that landed at frame +// into the dirty area. Framebuffer coordinates are screen coordinates (see +// compute_destination). A shift can start the frame mid-row, wrapping its +// rows; then cover the full width. +static void videocore_videobuffer_dirty_frame_at(videocore_videobuffer_obj_t *self, const void *frame) { + const size_t bytes_per_pixel = self->source_bits_per_pixel / 8; + if (self->framebuffer_protocol == NULL || bytes_per_pixel == 0) { + return; // Raw buffer frames are never shown where they land. + } + mp_buffer_info_t bufinfo; + self->framebuffer_protocol->get_bufinfo(self->framebuffer, &bufinfo); + const mp_int_t fb_width = FB_GETTER(get_width, 0); + const mp_int_t row_stride = FB_GETTER(get_row_stride, fb_width * self->source_bits_per_pixel / 8); + const mp_int_t first_pixel_offset = FB_GETTER(get_first_pixel_offset, 0); + const mp_int_t offset = (const uint8_t *)frame - (const uint8_t *)bufinfo.buf - first_pixel_offset; + if (bufinfo.buf == NULL || row_stride <= 0 || offset < 0 || offset >= (mp_int_t)bufinfo.len) { + return; + } + displayio_area_t area = { + .x1 = (offset % row_stride) / bytes_per_pixel, + .y1 = offset / row_stride, + }; + area.x2 = area.x1 + self->source_width; + area.y2 = area.y1 + self->source_height; + if (area.x2 > fb_width) { + area.x1 = 0; + area.x2 = fb_width; + area.y2 += 1; + } + displayio_area_union(&self->ephemeral_dirty_area, &area, &self->ephemeral_dirty_area); +} + +// Repaint where frames landed away from the current destination (see +// on_frame). Main context only. +static void videocore_videobuffer_dirty_stray_frames(videocore_videobuffer_obj_t *self) { + const uint8_t head = __atomic_load_n(&self->stray_head, __ATOMIC_ACQUIRE); + uint8_t tail = self->stray_tail; + while (tail != head) { + videocore_videobuffer_dirty_frame_at(self, self->stray_frames[tail % VIDEOCORE_STRAY_FRAMES]); + tail++; + } + __atomic_store_n(&self->stray_tail, tail, __ATOMIC_RELEASE); + if (self->stray_overflow) { + // Lost track of some: repaint the whole framebuffer. + self->stray_overflow = false; + if (self->framebuffer_protocol != NULL) { + displayio_area_t all = { + .x1 = 0, + .y1 = 0, + .x2 = FB_GETTER(get_width, 0), + .y2 = FB_GETTER(get_height, 0), + }; + displayio_area_union(&self->ephemeral_dirty_area, &all, &self->ephemeral_dirty_area); + } + } +} + +// Tear down playback state: clear our fields. +static void videocore_videobuffer_clear(videocore_videobuffer_obj_t *self); + +// Union the video's currently rendered rect into the dirty area so the next +// refresh repaints the pixels it leaves behind when it moves. Only meaningful +// while the video is actually shown: when it isn't, there are no pixels of +// ours to clear. Accumulates, so repeated moves before a refresh keep every +// vacated rect. +static void videocore_videobuffer_dirty_current_area(videocore_videobuffer_obj_t *self) { + if (self->absolute_transform.scale == 0 || self->hidden) { + return; + } + displayio_area_union( + &self->ephemeral_dirty_area, &self->current_area, &self->ephemeral_dirty_area); +} + +// Recompute the rendered rect (screen coordinates) of the video from the +// x/y position and the absolute transform. Video only supports scale 1 with +// no mirroring (validated in update_transform), so the rect is the frame +// size translated into place. +static void videocore_videobuffer_update_area(videocore_videobuffer_obj_t *self) { + const displayio_buffer_transform_t *t = &self->absolute_transform; + self->current_area.x1 = self->x + t->x; + self->current_area.y1 = self->y + t->y; + self->current_area.x2 = self->current_area.x1 + self->source_width; + self->current_area.y2 = self->current_area.y1 + self->source_height; + self->current_area.next = NULL; +} + +// Compute the destination for the source in the current target, honoring the +// current x/y position and the absolute transform's translation (a video in +// a group). Raises unless the whole frame fits in the target: the camera +// DMA's whole, contiguous frames with no stride padding, so rows can't wrap +// past the buffer and there is no cropping yet. The caller validates the +// fit (this runs from Python contexts); framebuffers in groups also move +// their destination from draw contexts with clamping (see _move_dest). +static void videocore_videobuffer_compute_destination( + videocore_videobuffer_obj_t *self, videocore_destination_t *dest) { + const size_t bytes_per_pixel = self->source_bits_per_pixel / 8; + const size_t row_bytes = (size_t)self->source_width * bytes_per_pixel; + + if (self->framebuffer_protocol == NULL) { + // Raw buffer mode: frames are written compactly from the start. + dest->base = self->bufinfo.buf; + dest->stride = row_bytes; + dest->row_bytes = row_bytes; + dest->height = self->source_height; + if (self->bufinfo.len < self->source_frame_bytes) { + mp_raise_ValueError(MP_ERROR_TEXT("Buffer too small")); + } + return; + } + + mp_buffer_info_t bufinfo; + self->framebuffer_protocol->get_bufinfo(self->framebuffer, &bufinfo); + if (bufinfo.buf == NULL) { + mp_raise_ValueError(MP_ERROR_TEXT("framebuffer went away")); + } + + const mp_int_t fb_width = FB_GETTER(get_width, 0); + const mp_int_t row_stride = FB_GETTER(get_row_stride, fb_width * self->source_bits_per_pixel / 8); + const mp_int_t first_pixel_offset = FB_GETTER(get_first_pixel_offset, 0); + + // The absolute position includes a containing group's translation. The + // shift may land the frame's rows mid-row: the image won't look right, + // but the contiguous DMA stays contiguous. Only the frame's start and + // end must remain within the buffer. + const displayio_buffer_transform_t *t = &self->absolute_transform; + const mp_int_t shift_x = self->x + t->x; + const mp_int_t shift_y = self->y + t->y; + if (self->x < 0 || self->y < 0 || shift_x < 0 || shift_y < 0 || + first_pixel_offset + shift_y * row_stride + shift_x * bytes_per_pixel + + (mp_int_t)self->source_frame_bytes > + bufinfo.len) { + mp_raise_ValueError(MP_ERROR_TEXT("video doesn't fit in framebuffer")); + } + + dest->base = (uint8_t *)bufinfo.buf + first_pixel_offset + + shift_y * row_stride + shift_x * bytes_per_pixel; + dest->stride = row_stride; + dest->row_bytes = row_bytes; + dest->height = self->source_height; +} + +// Move the framebuffer destination for a new absolute position, capping the +// shift so the frame stays completely within the framebuffer. Best effort: +// never raises, because this runs from displayio's refresh. +static void videocore_videobuffer_move_dest( + videocore_videobuffer_obj_t *self, mp_int_t shift_x, mp_int_t shift_y) { + if (!self->playing || self->paused) { + return; + } + mp_buffer_info_t bufinfo; + self->framebuffer_protocol->get_bufinfo(self->framebuffer, &bufinfo); + if (bufinfo.buf == NULL) { + return; + } + const size_t bytes_per_pixel = self->source_bits_per_pixel / 8; + const size_t row_bytes = (size_t)self->source_width * bytes_per_pixel; + const size_t frame_bytes = (size_t)self->source_height * row_bytes; + const mp_int_t fb_width = FB_GETTER(get_width, 0); + const mp_int_t row_stride = FB_GETTER(get_row_stride, fb_width * self->source_bits_per_pixel / 8); + const mp_int_t first_pixel_offset = FB_GETTER(get_first_pixel_offset, 0); + + if (shift_x < 0) { + shift_x = 0; + } + if (shift_y < 0) { + shift_y = 0; + } + // Cap the start and the end. Cap the rows first: enough rows must fit + // above the frame for it to fit at all. + const mp_int_t max_end = (mp_int_t)bufinfo.len - first_pixel_offset - (mp_int_t)frame_bytes; + if (shift_y > max_end / row_stride) { + shift_y = max_end / row_stride; + } + if (shift_y < 0) { + shift_y = 0; + } + const mp_int_t max_x = (max_end - shift_y * row_stride) / bytes_per_pixel; + if (shift_x > max_x) { + shift_x = max_x; + } + if (shift_x < 0) { + shift_x = 0; + } + + videocore_destination_t dest = { + .base = bufinfo.buf + first_pixel_offset + shift_y * row_stride + shift_x * bytes_per_pixel, + .stride = row_stride, + .row_bytes = row_bytes, + .height = self->source_height, + }; + // The result is ignored here: the source rejects a destination by + // returning false (e.g. a mid-row shift isn't DMA-aligned) and the + // stream continues where it was. on_frame points at the old destination + // until then, which keeps the last-known-good frame. + if (videosource_set_destination(self->source, &dest, videocore_videobuffer_on_frame, self)) { + self->stream_base = dest.base; + } +} + +void common_hal_videocore_videobuffer_construct(videocore_videobuffer_obj_t *self, mp_obj_t framebuffer) { + // The object has a finaliser, so initialise the fields it uses before any + // validation can raise. Otherwise a rejected framebuffer leaves source + // uninitialised and the later finaliser walks a garbage pointer. + self->source = mp_const_none; + self->stream_base = NULL; + self->playing = false; + self->framebuffer = framebuffer; + self->framebuffer_protocol = mp_proto_get(MP_QSTR_protocol_framebuffer, framebuffer); + if (self->framebuffer_protocol != NULL) { + if (self->framebuffer_protocol->get_bufinfo == NULL || + self->framebuffer_protocol->get_width == NULL || + self->framebuffer_protocol->get_height == NULL) { + mp_raise_ValueError(MP_ERROR_TEXT("framebuffer missing required functions")); + } + } else if (!mp_get_buffer(framebuffer, &self->bufinfo, MP_BUFFER_WRITE)) { + // Not a framebuffer and not a writable buffer such as a bytearray or + // memoryview. + mp_raise_TypeError_varg(MP_ERROR_TEXT("%q must be of type %q or %q, not %q"), + MP_QSTR_framebuffer, MP_QSTR_buffer, MP_QSTR_framebuffer, + mp_obj_get_type(framebuffer)->name); + } + + self->x = 0; + self->y = 0; + self->frame_ready = false; + self->stray_head = 0; + self->stray_tail = 0; + self->stray_overflow = false; + self->in_group = false; + self->hidden = false; + self->stale_size = false; + displayio_area_t empty = {}; + self->current_area = empty; + self->ephemeral_dirty_area = empty; + displayio_buffer_transform_t empty_transform = {}; + self->absolute_transform = empty_transform; + self->playing = false; + // Nothing is shown until a Group attached to a display updates our + // transform. + self->paused = true; +} + +// Start the source's hardware and capture its frame format. It is not pointed +// at a destination yet, so it does not write frames: the caller either starts +// streaming or leaves it paused. When paused is true the source comes up +// paused, so no frame can slip out between starting and pausing. The source +// may reconfigure itself (e.g. a sensor picks its mode), which fixes what +// get_* report from here on. Runs from Python contexts (play) so it may raise. +static void videocore_videobuffer_start_source(videocore_videobuffer_obj_t *self, bool paused) { + mp_obj_t source = self->source; + const videosource_p_t *protocol = videosource_check(source); + + videosource_play(source, paused); + + mp_int_t bits_per_pixel = protocol->get_bits_per_pixel(source); + if (bits_per_pixel % 8 != 0) { + mp_raise_ValueError(MP_ERROR_TEXT("video format must be byte aligned")); + } + mp_int_t width = protocol->get_width(source); + mp_int_t height = protocol->get_height(source); + size_t frame_bytes = (size_t)((width * height * bits_per_pixel) / 8); + + if (self->framebuffer_protocol != NULL) { + mp_int_t color_depth = FB_GETTER(get_color_depth, 16); + if (color_depth % 8 != 0 || bits_per_pixel != color_depth) { + mp_raise_ValueError(MP_ERROR_TEXT("video format doesn't match framebuffer")); + } + } + + self->source_width = width; + self->source_height = height; + self->source_bits_per_pixel = bits_per_pixel; + self->source_frame_bytes = frame_bytes; + + if (self->framebuffer_protocol == NULL && (mp_int_t)self->bufinfo.len < (mp_int_t)frame_bytes) { + mp_raise_ValueError(MP_ERROR_TEXT("Buffer too small")); + } +} + +// Point a started source at the current destination and let it stream. Runs +// from Python contexts (play and draw_update_transform) so it may raise. +static void videocore_videobuffer_start_stream(videocore_videobuffer_obj_t *self) { + videocore_destination_t dest; + videocore_videobuffer_compute_destination(self, &dest); + + // Tell the source where to put its pixels. The source streams from now + // on; on_frame tells us when a new frame has landed. It is the + // displayio group tree (and this object's Python references) that keep + // the VideoBuffer alive; a GC'd buffer's finaliser stops the stream. + if (!videosource_set_destination(self->source, &dest, videocore_videobuffer_on_frame, self)) { + mp_raise_ValueError(MP_ERROR_TEXT("video source rejected the destination")); + } + self->stream_base = dest.base; + self->paused = false; +} + +// Un-show a framebuffer target by pausing its source. The play session and +// the source's hardware stay alive, but no frames are written to the target. +// Best effort, like the finaliser's stop: this runs from a group operation, so +// it must not raise. +static void videocore_videobuffer_pause_stream(videocore_videobuffer_obj_t *self) { + if (self->paused || self->source == mp_const_none) { + return; // Already paused, or nothing has been started yet. + } + // Drop the destination before pausing the source: the source finishes its + // pause at the next frame boundary, so the frame being written still + // lands after this, and on_frame queues it as a stray to repaint over. + self->stream_base = NULL; + self->paused = true; + nlr_buf_t nlr; + if (nlr_push(&nlr) == 0) { + videosource_pause(self->source); + nlr_pop(); + } + self->frame_ready = false; +} + +// Re-show a framebuffer target by pointing its paused source at the current +// destination and resuming it. Position changes while hidden are applied here; +// the source's format was fixed at play(), so reject a changed source rather +// than resuming into a wrong-sized frame. +static void videocore_videobuffer_resume_stream(videocore_videobuffer_obj_t *self) { + const videosource_p_t *protocol = videosource_check(self->source); + if (protocol->get_width(self->source) != self->source_width || + protocol->get_height(self->source) != self->source_height || + protocol->get_bits_per_pixel(self->source) != self->source_bits_per_pixel) { + self->stale_size = true; + return; + } + + videocore_destination_t dest; + videocore_videobuffer_compute_destination(self, &dest); + if (!videosource_set_destination(self->source, &dest, videocore_videobuffer_on_frame, self)) { + mp_raise_ValueError(MP_ERROR_TEXT("video source rejected the destination")); + } + videosource_resume(self->source); + self->stream_base = dest.base; + self->paused = false; + // Wait for the first frame at the new destination. + self->frame_ready = false; +} + +// Start, resume or pause a framebuffer target's source to match whether it is +// shown (on a display and not hidden). It streams only while shown; while +// hidden the source stays active but paused, so showing it again is quick. +// Plain buffer targets stream regardless; displayio renders them wherever they +// are. +static void videocore_videobuffer_update_stream(videocore_videobuffer_obj_t *self) { + if (self->framebuffer_protocol == NULL) { + return; // Plain buffer targets always stream. + } + const bool shown = self->absolute_transform.scale != 0 && !self->hidden; + if (!self->playing) { + // Remember whether the target is shown so play() can start the source + // in the right state. + self->paused = !shown; + return; + } + if (shown && !self->stale_size) { + if (self->paused) { + videocore_videobuffer_resume_stream(self); + } + } else if (!shown && !self->paused) { + videocore_videobuffer_pause_stream(self); + } +} + +void common_hal_videocore_videobuffer_play(videocore_videobuffer_obj_t *self, mp_obj_t source) { + if (self->playing) { + mp_raise_RuntimeError(MP_ERROR_TEXT("already playing")); + } + videosource_check(source); + self->source = source; + self->stale_size = false; + + // A framebuffer target streams only while shown: DMA'ing into a buffer + // displayio isn't presenting would be meaningless. Start the source now + // either way, but start it paused when it isn't shown. Passing paused to + // play() (rather than pausing afterwards) keeps it from writing a frame + // in between. Plain buffer targets start streaming immediately. + const bool start_paused = self->framebuffer_protocol != NULL && self->paused; + videocore_videobuffer_start_source(self, start_paused); + self->playing = true; + if (start_paused) { + // No destination is set until the VideoBuffer is shown, so the source + // has nowhere to put a frame while paused. + self->frame_ready = false; + self->stream_base = NULL; + } else { + videocore_videobuffer_start_stream(self); + } +} + +void common_hal_videocore_videobuffer_set_position(videocore_videobuffer_obj_t *self, mp_int_t x, mp_int_t y) { + if (self->framebuffer_protocol != NULL && x != 0) { + // The source DMAs whole, contiguous frames with no stride, so every + // frame row must land on a framebuffer row: the video's x must be 0. + mp_raise_ValueError(MP_ERROR_TEXT("video in a framebuffer must be at x == 0")); + } + // Repaint where the video used to be. Union (rather than overwrite) so a + // move before the previous one was refreshed isn't lost. + videocore_videobuffer_dirty_current_area(self); + self->x = x; + self->y = y; + videocore_videobuffer_update_area(self); + if (!self->playing || self->paused || self->framebuffer_protocol == NULL) { + // Raw buffer targets are rendered by displayio; their capture + // destination doesn't move with the position. A paused framebuffer + // target has no destination to move yet. + return; + } + // The source's frame size was fixed when play() was called. Reject + // sources that changed it (e.g. a sensor mode change) rather than + // blitting at stale geometry. + const videosource_p_t *protocol = videosource_check(self->source); + mp_int_t width = protocol->get_width(self->source); + mp_int_t height = protocol->get_height(self->source); + mp_int_t bits_per_pixel = protocol->get_bits_per_pixel(self->source); + if (width != self->source_width || height != self->source_height || + bits_per_pixel != self->source_bits_per_pixel) { + mp_raise_ValueError(MP_ERROR_TEXT("video size changed since play; stop and play again")); + } + // Move the destination between frames; the source picks up the change on + // its next frame. The result may be ignored here (the source rejects a + // destination by returning false; the stream continues where it was). + videocore_destination_t dest; + videocore_videobuffer_compute_destination(self, &dest); + if (videosource_set_destination(self->source, &dest, videocore_videobuffer_on_frame, self)) { + self->stream_base = dest.base; + } +} + +static void videocore_videobuffer_clear(videocore_videobuffer_obj_t *self) { + self->playing = false; + self->frame_ready = false; + self->stream_base = NULL; + self->source = mp_const_none; +} + +void common_hal_videocore_videobuffer_stop(videocore_videobuffer_obj_t *self) { + // Stop the source; the camera also stops the sensor so its settings can + // be changed. That may raise: surface it, after cleaning up, since the + // stream did stop. + if (self->source != mp_const_none) { + nlr_buf_t nlr; + if (nlr_push(&nlr) == 0) { + videosource_stop(self->source); + nlr_pop(); + } else { + videocore_videobuffer_clear(self); + nlr_raise(nlr.ret_val); + } + } + videocore_videobuffer_clear(self); +} + +// Stop without raising: runs from background contexts (reload). A failed +// stop is best-effort; the source's own finaliser stops it as well. +void videocore_videobuffer_stop_quiet(videocore_videobuffer_obj_t *self) { + nlr_buf_t nlr; + if (nlr_push(&nlr) == 0) { + if (self->source != mp_const_none) { + videosource_stop(self->source); + } + nlr_pop(); + } + videocore_videobuffer_clear(self); +} + +bool common_hal_videocore_videobuffer_get_playing(videocore_videobuffer_obj_t *self) { + return self->playing; +} + +mp_obj_t common_hal_videocore_videobuffer_get_source(videocore_videobuffer_obj_t *self) { + return self->source; +} + +// Draw protocol implementation: a VideoBuffer holding a plain buffer gets +// bulk copied into the display by displayio's rendering; a framebuffer +// target's pixels are streamed into its framebuffer directly by the source, +// so displayio's fill reads them back into the scratch buffer it composes in +// (the refresh writes that scratch buffer back over the framebuffer). + +static mp_obj_t videocore_videobuffer_draw_protocol_self(mp_obj_t protocol_container) { + return protocol_container; +} + +static bool videocore_videobuffer_draw_fill_area(mp_obj_t self_in, const _displayio_colorspace_t *colorspace, const displayio_area_t *area, uint32_t *mask, uint32_t *buffer) { + videocore_videobuffer_obj_t *self = MP_OBJ_TO_PTR(self_in); + // buffer isn't necessarily the framebuffer. It may be a scratch buffer. + if (!self->playing || self->stream_base == NULL || self->hidden) { + return false; + } + + displayio_area_t overlap; + if (!displayio_area_compute_overlap(area, &self->current_area, &overlap)) { + return false; + } + const bool full_coverage = displayio_area_equal(area, &overlap); + const uint16_t span_px = displayio_area_width(area); + const size_t overlap_width = displayio_area_width(&overlap); + + if (self->framebuffer_protocol != NULL) { + // Framebuffer target: the source streamed the pixels straight into the + // framebuffer, so there is nothing to copy *into* the framebuffer (the + // DMA did that). If the video is the only thing covering this area, + // claiming nothing signals the driver to skip its write-back and + // leave those pixels alone. Otherwise displayio composes into a + // scratch buffer and writes it back over the framebuffer, so read the + // video's pixels into the scratch (where a higher layer hasn't already + // drawn) and claim them so lower layers don't draw over them. + const size_t bytes_per_pixel = self->source_bits_per_pixel / 8; + const mp_int_t fb_width = FB_GETTER(get_width, 0); + const mp_int_t row_stride = FB_GETTER(get_row_stride, fb_width * self->source_bits_per_pixel / 8); + const uint8_t *frame = (const uint8_t *)self->stream_base; + uint8_t *out = (uint8_t *)buffer; + + // Check whole words. This may include pixels between overlap rows or + // just outside the overlap, which can only make us take the slower + // per-pixel path below; it never misses a claimed pixel. + const size_t first_px = (size_t)(overlap.y1 - area->y1) * span_px + + (overlap.x1 - area->x1); + const size_t end_px = (size_t)(overlap.y2 - 1 - area->y1) * span_px + + (overlap.x2 - area->x1); + bool masked = false; + for (size_t word = first_px / 32; word <= (end_px - 1) / 32; word++) { + if (mask[word] != 0) { + masked = true; + break; + } + } + if (full_coverage && !masked) { + // Sole coverer: nothing was composed, so the driver skips the copy. + return true; + } + + for (int16_t y = overlap.y1; y < overlap.y2; y++) { + const size_t px = (size_t)(y - area->y1) * span_px + (overlap.x1 - area->x1); + const int16_t fy = y - self->current_area.y1; + const uint8_t *row = frame + (size_t)fy * row_stride + + (size_t)(overlap.x1 - self->current_area.x1) * bytes_per_pixel; + size_t i = 0; + while (i < overlap_width) { + // Skip pixels a higher layer already claimed or drew. + while (i < overlap_width && (mask[(px + i) / 32] & (1u << ((px + i) % 32)))) { + i++; + } + const size_t start = i; + while (i < overlap_width && !(mask[(px + i) / 32] & (1u << ((px + i) % 32)))) { + mask[(px + i) / 32] |= 1u << ((px + i) % 32); + i++; + } + if (i > start) { + memcpy( + out + (px + start) * bytes_per_pixel, + row + start * bytes_per_pixel, + (i - start) * bytes_per_pixel); + } + } + } + return full_coverage; + } + + // Raw buffer target: bulk copy, no color conversion: the display depth + // must match the video's pixel format byte for byte. + if (colorspace->depth != self->source_bits_per_pixel || colorspace->depth != 16) { + return false; + } + + // Opaque: claim the pixels. + for (int16_t y = overlap.y1; y < overlap.y2; y++) { + const size_t px = (size_t)(y - area->y1) * span_px + (overlap.x1 - area->x1); + for (size_t i = 0; i < overlap_width; i++) { + mask[(px + i) / 32] |= 1u << ((px + i) % 32); + } + } + + const size_t bytes_per_pixel = self->source_bits_per_pixel / 8; + const size_t row_bytes = (size_t)self->source_width * bytes_per_pixel; + const uint8_t *frame = (const uint8_t *)self->stream_base; + + for (int16_t y = overlap.y1; y < overlap.y2; y++) { + const size_t px = (size_t)(y - area->y1) * span_px + (overlap.x1 - area->x1); + const int16_t fy = y - self->current_area.y1; + memcpy( + (uint16_t *)buffer + px, + frame + (size_t)fy * row_bytes + (size_t)(overlap.x1 - self->current_area.x1) * bytes_per_pixel, + overlap_width * bytes_per_pixel); + } + return full_coverage; +} + +static bool videocore_videobuffer_draw_get_dirty_area(mp_obj_t self_in, displayio_area_t *out_area) { + videocore_videobuffer_obj_t *self = MP_OBJ_TO_PTR(self_in); + out_area->x1 = out_area->x2; + displayio_area_union(&self->ephemeral_dirty_area, &self->current_area, out_area); + return true; +} + +static void videocore_videobuffer_draw_update_transform(mp_obj_t self_in, displayio_buffer_transform_t *group_transform) { + videocore_videobuffer_obj_t *self = MP_OBJ_TO_PTR(self_in); + // Remember where the video was so a move can repaint the pixels it + // vacates. The area is only valid while the video is shown; a layer being + // attached or detached is handled by the display's full refresh. + displayio_area_t previous_area = self->current_area; + const bool was_shown = self->absolute_transform.scale != 0 && !self->hidden; + // A group that isn't attached to a display has no transform, and its + // update_transform clears the absolute transform we receive. A real + // (shown) transform always has scale >= 1. + if (group_transform == NULL || group_transform->scale == 0) { + displayio_buffer_transform_t no_transform = {}; + self->absolute_transform = no_transform; + } else { + // Validate the transform: video frames are bulk copied (raw targets) + // or DMA'd contiguously (framebuffer targets), so only translation + // moves. + if (group_transform->scale != 0 && + (group_transform->scale != 1 || + group_transform->mirror_x || group_transform->mirror_y || + group_transform->transpose_xy || + group_transform->dx != 1 || group_transform->dy != 1)) { + mp_raise_ValueError(MP_ERROR_TEXT("VideoBuffer does not support transforms")); + } + self->absolute_transform = *group_transform; + } + + // A framebuffer target starts or stops with whether it is shown (the + // absolute transform is known now). This runs from Python contexts, so a + // (re)start may raise. + videocore_videobuffer_update_stream(self); + + videocore_videobuffer_update_area(self); + + // Moved by a containing Group (or a transform change): dirty the rect it + // used to occupy so the refresh repaints those pixels instead of leaving + // the previous frame there. + if (was_shown && self->absolute_transform.scale != 0 && + !displayio_area_equal(&previous_area, &self->current_area)) { + displayio_area_union( + &self->ephemeral_dirty_area, &previous_area, &self->ephemeral_dirty_area); + } + + // Framebuffer targets stream straight into the buffer: move the + // destination along with the shift. Best effort, since this can't raise: + // the shift may land the start mid-row, which won't look right but is + // obvious; the frame is capped to stay completely within the buffer. + if (self->framebuffer_protocol != NULL) { + videocore_videobuffer_move_dest( + self, + self->x + self->absolute_transform.x, + self->y + self->absolute_transform.y); + } +} + +static void videocore_videobuffer_draw_finish_refresh(mp_obj_t self_in) { + videocore_videobuffer_obj_t *self = MP_OBJ_TO_PTR(self_in); + // The frame (or move) rendered; stop flagging it. + self->frame_ready = false; + displayio_area_t empty = {}; + self->ephemeral_dirty_area = empty; +} + +static displayio_area_t *videocore_videobuffer_draw_get_refresh_areas(mp_obj_t self_in, displayio_area_t *tail) { + videocore_videobuffer_obj_t *self = MP_OBJ_TO_PTR(self_in); + // Frames that landed after the video moved or paused, possibly over the + // repaint of the area they landed in: repaint those areas again. + videocore_videobuffer_dirty_stray_frames(self); + // Always repaint the area the video vacated, even once it is hidden or + // stopped, so its last frame doesn't stay on the target. + if (!displayio_area_empty(&self->ephemeral_dirty_area)) { + // The video moved or was hidden: repaint with the lower layers. + self->ephemeral_dirty_area.next = tail; + tail = &self->ephemeral_dirty_area; + } + if (!self->playing || self->paused || self->hidden || self->stale_size) { + return tail; + } + if (self->frame_ready) { + // New frame: bulk copy it in. + self->current_area.next = tail; + tail = &self->current_area; + } + return tail; +} + +static void videocore_videobuffer_draw_set_dirty(mp_obj_t self_in) { + (void)self_in; +} + +static void videocore_videobuffer_draw_set_hidden(mp_obj_t self_in, bool hidden) { + videocore_videobuffer_obj_t *self = MP_OBJ_TO_PTR(self_in); + if (self->hidden == hidden) { + return; + } + self->hidden = hidden; + // Repaint what the video showed or vacated. + self->ephemeral_dirty_area = self->current_area; + if (!hidden && self->playing) { + // The source normally can't change while the sensor stays streaming, + // but validate its sizing again in case it did: if it no longer + // matches, don't raise; simply stop showing the stream until stop and + // play again. + const videosource_p_t *protocol = mp_proto_get(MP_QSTR_protocol_videosource, self->source); + if (protocol != NULL && + (protocol->get_width(self->source) != self->source_width || + protocol->get_height(self->source) != self->source_height || + protocol->get_bits_per_pixel(self->source) != self->source_bits_per_pixel)) { + self->stale_size = true; + } + } + // Hiding or unhiding a framebuffer target pauses or resumes its source. + videocore_videobuffer_update_stream(self); +} + +static bool videocore_videobuffer_draw_set_in_group(mp_obj_t self_in, bool in_group) { + videocore_videobuffer_obj_t *self = MP_OBJ_TO_PTR(self_in); + bool was_in_group = self->in_group; + self->in_group = in_group; + return was_in_group; +} + +static vectorio_draw_protocol_impl_t videocore_videobuffer_draw_protocol_impl = { + .draw_fill_area = videocore_videobuffer_draw_fill_area, + .draw_get_dirty_area = videocore_videobuffer_draw_get_dirty_area, + .draw_update_transform = videocore_videobuffer_draw_update_transform, + .draw_finish_refresh = videocore_videobuffer_draw_finish_refresh, + .draw_get_refresh_areas = videocore_videobuffer_draw_get_refresh_areas, + .draw_set_dirty = videocore_videobuffer_draw_set_dirty, + .draw_set_hidden = videocore_videobuffer_draw_set_hidden, + .draw_set_in_group = videocore_videobuffer_draw_set_in_group, +}; + +const vectorio_draw_protocol_t videocore_videobuffer_draw_protocol = { + MP_PROTO_IMPLEMENT(MP_QSTR_protocol_draw) + .draw_get_protocol_self = videocore_videobuffer_draw_protocol_self, + .draw_protocol_impl = &videocore_videobuffer_draw_protocol_impl, +}; + +// Root group validation: a VideoBuffer that streams straight into a +// framebuffer bypasses displayio rendering, so it may only be shown on a +// display that renders that same framebuffer. Plain buffer targets are drawn +// by displayio and are fine anywhere. Walks the group tree and raises when a +// VideoBuffer doesn't match the display it would render on. +static void videocore_validate_member(mp_obj_t member, mp_obj_t display) { + displayio_group_t *group = mp_obj_cast_to_native_base(member, &displayio_group_type); + if (group != MP_OBJ_NULL) { + for (size_t i = 0; i < group->members->len; i++) { + videocore_validate_member(group->members->items[i], display); + } + return; + } + const vectorio_draw_protocol_t *draw_protocol = mp_proto_get(MP_QSTR_protocol_draw, member); + if (draw_protocol == NULL) { + return; + } + mp_obj_t layer = draw_protocol->draw_get_protocol_self(member); + if (mp_obj_get_type(layer) != &videocore_videobuffer_type) { + return; + } + videocore_videobuffer_obj_t *self = MP_OBJ_TO_PTR(layer); + if (self->framebuffer_protocol == NULL || display == self->framebuffer) { + return; + } + #if CIRCUITPY_FRAMEBUFFERIO + // A FramebufferDisplay renders the framebuffer it wraps. That is the only + // kind of display a framebuffer-target VideoBuffer can be shown on. + mp_obj_t native_display = mp_obj_cast_to_native_base(display, &framebufferio_framebufferdisplay_type); + if (native_display != MP_OBJ_NULL) { + framebufferio_framebufferdisplay_obj_t *fd = MP_OBJ_TO_PTR(native_display); + if (fd->framebuffer == self->framebuffer) { + return; + } + } + #endif + mp_raise_ValueError(MP_ERROR_TEXT("VideoBuffer is playing into a different framebuffer")); +} + +void videocore_validate_root_group(mp_obj_t root_group, mp_obj_t display) { + videocore_validate_member(root_group, display); +} diff --git a/shared-module/videocore/VideoBuffer.h b/shared-module/videocore/VideoBuffer.h new file mode 100644 index 00000000000..3a21b9b6d54 --- /dev/null +++ b/shared-module/videocore/VideoBuffer.h @@ -0,0 +1,96 @@ +// This file is part of the CircuitPython project: https://circuitpython.org +// +// SPDX-FileCopyrightText: Copyright (c) 2026 Scott Shawcroft for Adafruit Industries +// +// SPDX-License-Identifier: MIT + +#pragma once + +#include "py/obj.h" + +#include "shared-bindings/vectorio/__init__.h" // draw protocol +#include "shared-module/displayio/area.h" + +// How many frames that landed away from the current destination (see +// stray_frames) can wait for a refresh to repaint them. One lands per move or +// pause; more pile up only when the video moves faster than the display +// refreshes, and an overflow repaints the whole framebuffer instead. +#define VIDEOCORE_STRAY_FRAMES 4 +#include "shared-module/framebufferio/FramebufferDisplay.h" // framebuffer protocol +#include "shared-module/videocore/__init__.h" + +typedef struct { + mp_obj_base_t base; + // Object frames are streamed into: an object implementing the framebuffer + // protocol, such as mipidsi.Display, or a writable buffer such as a + // bytearray or memoryview. + mp_obj_t framebuffer; + const framebuffer_p_t *framebuffer_protocol; + // Target buffer info in raw buffer mode (framebuffer_protocol == NULL). + mp_buffer_info_t bufinfo; + // The video source being played, or None. + mp_obj_t source; + // Destination position of the video within the target. + mp_int_t x; + mp_int_t y; + mp_int_t source_width; + mp_int_t source_height; + mp_int_t source_bits_per_pixel; + size_t source_frame_bytes; + // Where the source streams frames into: the requested destination, or a + // later, hardware-aligned position inside it. "Later in the buffer" only + // happens for plain-buffer targets; set by play() and the x/y + // properties. + const void *stream_base; + // Set from interrupt context when the source lands a frame in the + // destination; cleared by the refresh pass once the new frame has been + // rendered into the display. + volatile bool frame_ready; + // Frames that landed somewhere other than stream_base: the frame being + // written when the destination changes, or when the source is paused, + // still lands at the previous destination, possibly after the refresh + // that repainted the vacated area. on_frame queues where they landed + // (interrupt context, the only producer) and the refresh pass repaints + // those areas with the lower layers (the only consumer). + const void *volatile stray_frames[VIDEOCORE_STRAY_FRAMES]; + volatile uint8_t stray_head; + volatile uint8_t stray_tail; + volatile bool stray_overflow; + // displayio draw protocol state: the rect currently rendered (in screen + // coordinates) and the rect that changed since the last refresh. + displayio_area_t current_area; + displayio_area_t ephemeral_dirty_area; + displayio_buffer_transform_t absolute_transform; + bool in_group; + // Set by the group when hidden; while hidden the video doesn't show and + // a framebuffer target's source is paused (it is revalidated on unhide). + bool hidden; + // True when the source's sizing no longer matches what play() captured: + // the video won't show until stop and play again. + bool stale_size; + bool playing; + // True when the framebuffer target isn't shown (not attached to a display + // or hidden), so the source should not stream to the target. play() passes + // this to the source so it starts paused; otherwise showing/hiding the + // target pauses and resumes the source. It also keeps this state while + // stopped, so a later play() starts correctly. Plain buffer targets are + // never paused. + bool paused; +} videocore_videobuffer_obj_t; + +void common_hal_videocore_videobuffer_construct(videocore_videobuffer_obj_t *self, mp_obj_t framebuffer); +void common_hal_videocore_videobuffer_play(videocore_videobuffer_obj_t *self, mp_obj_t source); +void common_hal_videocore_videobuffer_set_position(videocore_videobuffer_obj_t *self, mp_int_t x, mp_int_t y); +void common_hal_videocore_videobuffer_stop(videocore_videobuffer_obj_t *self); +// Stop without raising (from the finaliser). A failed stop is best-effort; +// the source's own finaliser stops it as well. +void videocore_videobuffer_stop_quiet(videocore_videobuffer_obj_t *self); +bool common_hal_videocore_videobuffer_get_playing(videocore_videobuffer_obj_t *self); +mp_obj_t common_hal_videocore_videobuffer_get_source(videocore_videobuffer_obj_t *self); + +// The draw protocol implementation that lets a VideoBuffer be added to a +// displayio Group: plain-buffer targets get bulk copied into the framebuffer +// by displayio's rendering; framebuffer targets claim their area (a no-op +// fill: the source streams their pixels straight into the framebuffer) so +// displayio's refresh pushes the video's rows to the framebuffer. +extern const vectorio_draw_protocol_t videocore_videobuffer_draw_protocol; diff --git a/shared-module/videocore/__init__.c b/shared-module/videocore/__init__.c new file mode 100644 index 00000000000..6f45f783688 --- /dev/null +++ b/shared-module/videocore/__init__.c @@ -0,0 +1,56 @@ +// This file is part of the CircuitPython project: https://circuitpython.org +// +// SPDX-FileCopyrightText: Copyright (c) 2026 Scott Shawcroft for Adafruit Industries +// +// SPDX-License-Identifier: MIT + +#include "shared-module/videocore/__init__.h" +#include "shared-module/videocore/VideoBuffer.h" + +// Playback, presentation and garbage collection are tied to the displayio +// group the VideoBuffer is in: displayio's refresh drives frame presentation +// through the draw protocol, and the group (or a Python reference) keeps the +// object alive; its finaliser stops the stream. + +const videosource_p_t *videosource_check(mp_obj_t obj) { + return (const videosource_p_t *)mp_proto_get_or_throw(MP_QSTR_protocol_videosource, obj); +} + +void videosource_play(mp_obj_t source, bool paused) { + const videosource_p_t *protocol = videosource_check(source); + if (protocol->play != NULL) { + protocol->play(source, paused); + } +} + +void videosource_stop(mp_obj_t source) { + const videosource_p_t *protocol = videosource_check(source); + if (protocol->stop != NULL) { + protocol->stop(source); + } +} + +void videosource_pause(mp_obj_t source) { + videosource_check(source)->pause(source); +} + +void videosource_resume(mp_obj_t source) { + videosource_check(source)->resume(source); +} + +bool videosource_set_destination(mp_obj_t source, const videocore_destination_t *dest, + videocore_on_frame_fun on_frame, void *ctx) { + return videosource_check(source)->set_destination(source, dest, on_frame, ctx); +} + +mp_int_t videosource_get_width(mp_obj_t source) { + return videosource_check(source)->get_width(source); +} + +mp_int_t videosource_get_height(mp_obj_t source) { + return videosource_check(source)->get_height(source); +} + +mp_int_t videosource_get_bits_per_pixel(mp_obj_t source) { + return videosource_check(source)->get_bits_per_pixel(source); +} diff --git a/shared-module/videocore/__init__.h b/shared-module/videocore/__init__.h new file mode 100644 index 00000000000..86eddae7c6e --- /dev/null +++ b/shared-module/videocore/__init__.h @@ -0,0 +1,120 @@ +// This file is part of the CircuitPython project: https://circuitpython.org +// +// SPDX-FileCopyrightText: Copyright (c) 2026 Scott Shawcroft for Adafruit Industries +// +// SPDX-License-Identifier: MIT + +#pragma once + +#include +#include +#include + +#include "py/obj.h" +#include "py/proto.h" + +// Destination for a video stream: where to put pixels, as a pointer to the +// first pixel and the destination row stride. row_bytes/height describe the +// source frame; the source assumes the destination holds the whole frame +// (the ESP CSI DMA is contiguous-only), so the rect is validated before +// streaming starts and when the position changes. +typedef struct { + // First pixel of the destination. + uint8_t *base; + // Destination row stride, in bytes. + size_t stride; + // Source frame row size, in bytes. + size_t row_bytes; + // Source frame height, in rows. + size_t height; +} videocore_destination_t; + +// Called (in interrupt context) when a frame has landed. frame is the +// destination base it landed at: usually the latest accepted destination, but +// a frame already being written when the destination changed (or when the +// source was paused) still lands at the previous one. +typedef void (*videocore_on_frame_fun)(void *ctx, const void *frame); + +// Protocol implemented by video streamers such as mipicsi.Camera. A source +// streams continuously while playing (unless paused); the target sets (and can +// change) where the pixels go by calling set_destination() at any time, +// including between frames. +// +// This is the video equivalent of the audiosample protocol in the audiocore +// module, inverted: instead of the target pulling frames, the source writes +// where it is told to. +typedef struct _videosource_p_t { + MP_PROTOCOL_HEAD // MP_QSTR_protocol_videosource + + // Create whatever streaming hardware the source needs and start it (e.g. + // start a sensor streaming). Called once before the first + // set_destination of a stream, from Python (VideoBuffer.play), so it may + // raise. Raises when the source is already streaming for someone else. + // May be NULL for sources that need no preparation. The source must not + // write frames before set_destination(). When paused is true, the source's + // hardware must come up active but not writing frames, as if pause() had + // been called before it started; this avoids a play-then-pause race when + // the target isn't shown yet. + void (*play)(mp_obj_t self, bool paused); + + // Pause streaming without stopping the camera or releasing its hardware: + // e.g. stop the CSI controller's DMA while the sensor keeps streaming. + // No frames are written until resume(). Called when a framebuffer target + // stops being shown. set_destination() while paused updates where resume() + // will stream. Must not raise: it runs from displayio group operations. + void (*pause)(mp_obj_t self); + + // Resume streaming after pause() into the most recent destination. Called + // from Python contexts (when the framebuffer target is shown again), so it + // may raise; the source must stay paused if it does. + void (*resume)(mp_obj_t self); + + // Direct future frames into dest. The destination may be changed + // between frames; there is no NULL/stop value: stopping is stop()'s + // job. Never raises: returns whether the stream accepted the + // destination (the caller may ignore the result). on_frame(ctx, frame) + // is called from interrupt context whenever a frame has landed, with the + // destination base it landed at (the source never relocates frames + // elsewhere). A change takes effect from the next frame: the frame being + // written still lands at the previous destination. + bool (*set_destination)(mp_obj_t self, const videocore_destination_t *dest, + videocore_on_frame_fun on_frame, void *ctx); + + // Stop streaming and release the streaming hardware (e.g. stop the + // sensor so its settings can be changed). Called once per stream, from + // Python (VideoBuffer.stop), so it may raise; exceptions there are + // clearer than implicit stops. Only called when play() succeeded. + void (*stop)(mp_obj_t self); + + // Width of the frames the source streams, in pixels. + mp_int_t (*get_width)(mp_obj_t self); + // Height of the frames the source streams, in lines. + mp_int_t (*get_height)(mp_obj_t self); + // Bits per pixel of the frames the source streams. + mp_int_t (*get_bits_per_pixel)(mp_obj_t self); +} videosource_p_t; + +// Protocol function helpers. videosource_play/stop are no-ops for sources +// whose protocol doesn't implement them. pause/resume are required: they +// keep a source's hardware alive while a framebuffer target is hidden. +void videosource_play(mp_obj_t source, bool paused); +void videosource_stop(mp_obj_t source); +void videosource_pause(mp_obj_t source); +void videosource_resume(mp_obj_t source); + +// Return the videosource protocol of obj or raise TypeError. +const videosource_p_t *videosource_check(mp_obj_t obj); + +// Dispatch helpers mirroring the audiosample_* functions. videosource_play +// is a no-op for sources whose protocol has no play. +bool videosource_set_destination(mp_obj_t source, const videocore_destination_t *dest, + videocore_on_frame_fun on_frame, void *ctx); +mp_int_t videosource_get_width(mp_obj_t source); +mp_int_t videosource_get_height(mp_obj_t source); +mp_int_t videosource_get_bits_per_pixel(mp_obj_t source); + +// Raise when the group tree contains a VideoBuffer that streams into a +// framebuffer other than the one this display renders. Called when a group +// is made a display's root group. Implemented in shared-module/videocore/ +// VideoBuffer.c. +void videocore_validate_root_group(mp_obj_t root_group, mp_obj_t display);