From 2edb7f1acfb6ad5d84cf2757a9f71823ba8949f7 Mon Sep 17 00:00:00 2001 From: Scott Shawcroft Date: Wed, 7 Oct 2026 10:34:14 -0700 Subject: [PATCH 1/2] Add initial video pipeline support Start with `mipicsi` and a `videocore` object for storing a framebuffer. With this you can capture camera input into a buffer or directly into another DSI display's framebuffer. Capturing into the buffer and shown with displayio will be much slower than copying straight into the framebuffer. It is more flexible with size and position though. --- locale/circuitpython.pot | 64 +- ports/espressif/CMakeLists.txt | 17 +- ports/espressif/Makefile | 23 +- .../espressif_esp32p4_function_ev/sdkconfig | 24 + ports/espressif/common-hal/mipicsi/Camera.c | 468 +++++++++++++++ ports/espressif/common-hal/mipicsi/Camera.h | 89 +++ ports/espressif/common-hal/mipicsi/__init__.c | 7 + ports/espressif/common-hal/mipidsi/Bus.c | 45 +- ports/espressif/common-hal/mipidsi/Bus.h | 9 + ports/espressif/common-hal/mipidsi/Display.c | 13 +- ports/espressif/mpconfigport.mk | 7 + ports/espressif/supervisor/port.c | 6 +- py/circuitpy_defns.mk | 12 + py/circuitpy_mpconfig.mk | 9 + shared-bindings/adafruit_pixelbuf/PixelBuf.c | 8 +- shared-bindings/busdisplay/BusDisplay.c | 8 + .../framebufferio/FramebufferDisplay.c | 8 + shared-bindings/mipicsi/BayerOrder.c | 50 ++ shared-bindings/mipicsi/Camera.c | 231 ++++++++ shared-bindings/mipicsi/Camera.h | 60 ++ shared-bindings/mipicsi/PixelFormat.c | 66 +++ shared-bindings/mipicsi/__init__.c | 46 ++ shared-bindings/mipicsi/__init__.h | 33 ++ shared-bindings/vectorio/__init__.h | 5 + shared-bindings/videocore/VideoBuffer.c | 241 ++++++++ shared-bindings/videocore/VideoBuffer.h | 11 + shared-bindings/videocore/__init__.c | 86 +++ shared-bindings/videocore/__init__.h | 9 + shared-module/displayio/Group.c | 30 +- shared-module/videocore/VideoBuffer.c | 547 ++++++++++++++++++ shared-module/videocore/VideoBuffer.h | 73 +++ shared-module/videocore/__init__.c | 48 ++ shared-module/videocore/__init__.h | 96 +++ 33 files changed, 2424 insertions(+), 25 deletions(-) create mode 100644 ports/espressif/common-hal/mipicsi/Camera.c create mode 100644 ports/espressif/common-hal/mipicsi/Camera.h create mode 100644 ports/espressif/common-hal/mipicsi/__init__.c create mode 100644 shared-bindings/mipicsi/BayerOrder.c create mode 100644 shared-bindings/mipicsi/Camera.c create mode 100644 shared-bindings/mipicsi/Camera.h create mode 100644 shared-bindings/mipicsi/PixelFormat.c create mode 100644 shared-bindings/mipicsi/__init__.c create mode 100644 shared-bindings/mipicsi/__init__.h create mode 100644 shared-bindings/videocore/VideoBuffer.c create mode 100644 shared-bindings/videocore/VideoBuffer.h create mode 100644 shared-bindings/videocore/__init__.c create mode 100644 shared-bindings/videocore/__init__.h create mode 100644 shared-module/videocore/VideoBuffer.c create mode 100644 shared-module/videocore/VideoBuffer.h create mode 100644 shared-module/videocore/__init__.c create mode 100644 shared-module/videocore/__init__.h diff --git a/locale/circuitpython.pot b/locale/circuitpython.pot index c03d1bd0ae4..97b798f7f5f 100644 --- a/locale/circuitpython.pot +++ b/locale/circuitpython.pot @@ -974,7 +974,6 @@ msgstr "" #: ports/atmel-samd/common-hal/pulseio/PulseIn.c #: ports/atmel-samd/common-hal/rotaryio/IncrementalEncoder.c #: ports/cxd56/common-hal/pulseio/PulseIn.c -#: ports/espressif/common-hal/mii/RMII.c #: ports/raspberrypi/common-hal/picodvi/Framebuffer_RP2350.c #: ports/zephyr-cp/common-hal/busio/I2C.c #: ports/zephyr-cp/common-hal/busio/SPI.c @@ -1182,6 +1181,7 @@ msgid "Another send is already active" msgstr "" #: ports/atmel-samd/common-hal/sdioio/SDCard.c +#: ports/espressif/common-hal/mii/Ethernet.c #: ports/espressif/common-hal/qspibus/QSPIBus.c shared-bindings/emmcio/EMMC.c #: shared-bindings/socketpool/SocketPool.c msgid "%q failure: %d" @@ -1219,6 +1219,7 @@ msgstr "" #: shared-bindings/busdisplay/BusDisplay.c #: shared-bindings/framebufferio/FramebufferDisplay.c #: shared-bindings/struct/__init__.c shared-module/struct/__init__.c +#: shared-module/videocore/VideoBuffer.c msgid "Buffer too small" msgstr "" @@ -1517,7 +1518,8 @@ msgid "%q must be 8, 16, 24, or 32" msgstr "" #: ports/espressif/common-hal/audiobusio/__init__.c -#: ports/espressif/common-hal/audioi2sin/I2SIn.c shared-bindings/emmcio/EMMC.c +#: ports/espressif/common-hal/audioi2sin/I2SIn.c +#: ports/espressif/common-hal/mii/Ethernet.c shared-bindings/emmcio/EMMC.c msgid "Peripheral in use" msgstr "" @@ -1563,6 +1565,8 @@ msgid "Can't construct AudioOut because continuous channel already open" msgstr "" #: ports/espressif/common-hal/audioio/AudioOut.c +#: ports/espressif/common-hal/mipicsi/Camera.c +#: shared-module/videocore/VideoBuffer.c msgid "already playing" msgstr "" @@ -1751,6 +1755,18 @@ msgstr "" msgid "Address range not allowed" msgstr "" +#: ports/espressif/common-hal/mipicsi/Camera.c +msgid "pixel_format must be RAW8, RAW10 or RAW12 when output_format differs" +msgstr "" + +#: ports/espressif/common-hal/mipicsi/Camera.c +msgid "output_format must be RGB565 or RGB888" +msgstr "" + +#: ports/espressif/common-hal/mipicsi/Camera.c +msgid "ISP conversion not supported on this chip" +msgstr "" + #: ports/espressif/common-hal/nvm/ByteArray.c msgid "NVS Error" msgstr "" @@ -4771,6 +4787,50 @@ msgstr "" msgid "Polygon needs at least 3 points" msgstr "" +#: shared-module/videocore/VideoBuffer.c +msgid "framebuffer went away" +msgstr "" + +#: shared-module/videocore/VideoBuffer.c +msgid "video doesn't fit in framebuffer" +msgstr "" + +#: shared-module/videocore/VideoBuffer.c +msgid "framebuffer missing required functions" +msgstr "" + +#: shared-module/videocore/VideoBuffer.c +msgid "expected FrameBuffer or %q" +msgstr "" + +#: shared-module/videocore/VideoBuffer.c +msgid "video format must be byte aligned" +msgstr "" + +#: shared-module/videocore/VideoBuffer.c +msgid "video format doesn't match framebuffer" +msgstr "" + +#: shared-module/videocore/VideoBuffer.c +msgid "video source rejected the destination" +msgstr "" + +#: shared-module/videocore/VideoBuffer.c +msgid "video in a framebuffer must be at x == 0" +msgstr "" + +#: shared-module/videocore/VideoBuffer.c +msgid "video size changed since play; stop and play again" +msgstr "" + +#: shared-module/videocore/VideoBuffer.c +msgid "VideoBuffer does not support transforms" +msgstr "" + +#: shared-module/videocore/VideoBuffer.c +msgid "VideoBuffer is playing into a different framebuffer" +msgstr "" + #: supervisor/shared/bluetooth/bluetooth.c msgid "Reconnecting" msgstr "" diff --git a/ports/espressif/CMakeLists.txt b/ports/espressif/CMakeLists.txt index 608d4f4c7b3..979afdad548 100644 --- a/ports/espressif/CMakeLists.txt +++ b/ports/espressif/CMakeLists.txt @@ -5,7 +5,22 @@ cmake_minimum_required(VERSION 3.16) set(ENV{IDF_PATH} ${CMAKE_SOURCE_DIR}/esp-idf) # The component list here determines what options we get in menuconfig and what the ninja file can build. -set(COMPONENTS bt driver esp_driver_dac esp_driver_gpio esp_driver_gptimer esp_driver_i2c esp_driver_i2s esp_driver_ledc esp_driver_pcnt esp_driver_rmt esp_driver_sdmmc esp_driver_spi esp_driver_touch_sens esp_driver_tsens esp_driver_uart esp_eth esp-tls esp_adc esp_adc_cal esp_event esp_netif esp_psram esp_security esp_wifi esptool_py freertos log lwip main mbedtls mdns pthread soc ulp wpa_supplicant esp-camera esp_lcd vfs esp_vfs_console esp_stdio sdmmc usb) +# Keep this split into separate set()/list() calls so additions merge easily. +# Core components +set(COMPONENTS bt driver esptool_py freertos log main mbedtls pthread soc) +# Peripheral drivers +list(APPEND COMPONENTS esp_driver_dac esp_driver_gpio esp_driver_gptimer esp_driver_i2c esp_driver_i2s esp_driver_ledc esp_driver_pcnt esp_driver_rmt esp_driver_sdmmc esp_driver_spi esp_driver_touch_sens esp_driver_tsens esp_driver_uart) +# Networking +list(APPEND COMPONENTS esp_eth esp-tls esp_adc esp_adc_cal esp_event esp_netif esp_psram esp_security esp_wifi lwip mdns wpa_supplicant) +# Misc +list(APPEND COMPONENTS ulp vfs esp_vfs_console esp_stdio sdmmc usb) +# Display and camera +list(APPEND COMPONENTS esp-camera esp_lcd) + +if(CIRCUITPY_MIPICSI) + list(APPEND COMPONENTS esp_driver_cam esp_hal_cam) +endif() + set(EXTRA_COMPONENT_DIRS "esp-protocols/components/mdns" "esp-camera") include($ENV{IDF_PATH}/tools/cmake/project.cmake) diff --git a/ports/espressif/Makefile b/ports/espressif/Makefile index 96a71b177b4..e4f93259bce 100644 --- a/ports/espressif/Makefile +++ b/ports/espressif/Makefile @@ -226,6 +226,13 @@ REGISTRATION_FUNCTIONS = \ -u esp_security_init_include_impl \ -u mbedtls_psa_crypto_init_include_impl +# Video DMA destinations come from the Python heap (user bytearrays), and +# the cache line is 64 bytes on the P4, so make GC blocks 64 bytes as well. +# supervisor/port.c aligns the PSRAM heap areas to the same value. +ifeq ($(IDF_TARGET),esp32p4) +CFLAGS += -DMICROPY_BYTES_PER_GC_BLOCK=64 +endif + # RISC-V picolibc ships optimized memcpy/memset/etc. that need forcing # (gated by CONFIG_LIBC_OPTIMIZED_MISALIGNED_ACCESS in IDF's esp_libc CMake). ifeq ($(IDF_TARGET_ARCH),riscv) @@ -504,9 +511,12 @@ endif CHIP_COMPONENTS = \ + esp_driver_cam \ + esp_driver_isp \ esp_driver_touch_sens \ esp_driver_tsens \ esp_driver_usb_serial_jtag \ + esp_hal_cam \ esp_hal_dma \ esp_hal_emac \ esp_hal_i2s \ @@ -735,6 +745,14 @@ CFLAGS += \ -isystem esp-idf/components/esp_lcd/dsi/include endif +ifneq ($(CIRCUITPY_MIPICSI),0) +CFLAGS += \ + -isystem esp-idf/components/esp_driver_cam/include \ + -isystem esp-idf/components/esp_driver_cam/csi/include \ + -isystem esp-idf/components/esp_driver_isp/include \ + -isystem esp-idf/components/esp_hal_cam/include +endif + ifneq ($(CIRCUITPY_ESPCAMERA),0) SRC_CAMERA := \ $(wildcard common-hal/espcamera/*.c) \ @@ -935,7 +953,7 @@ do-sdkconfig: $(BUILD)/esp-idf/config/sdkconfig.h QSTR_GLOBAL_REQUIREMENTS += $(BUILD)/esp-idf/config/sdkconfig.h $(BUILD)/esp-idf/config/sdkconfig.h: boards/$(BOARD)/sdkconfig boards/$(BOARD)/mpconfigboard.mk CMakeLists.txt | $(BUILD)/esp-idf $(STEPECHO) "Create $@" - $(Q)env IDF_PATH=$(IDF_PATH) IDF_COMPONENT_MANAGER=0 cmake -S . -B $(BUILD)/esp-idf -DSDKCONFIG=$(BUILD)/esp-idf/sdkconfig -DSDKCONFIG_DEFAULTS="$(SDKCONFIGS)" -DCMAKE_TOOLCHAIN_FILE=$(IDF_PATH)/tools/cmake/toolchain-$(IDF_TARGET).cmake -DIDF_TARGET=$(IDF_TARGET) -GNinja + $(Q)env IDF_PATH=$(IDF_PATH) IDF_COMPONENT_MANAGER=0 cmake -S . -B $(BUILD)/esp-idf -DSDKCONFIG=$(BUILD)/esp-idf/sdkconfig -DSDKCONFIG_DEFAULTS="$(SDKCONFIGS)" -DCMAKE_TOOLCHAIN_FILE=$(IDF_PATH)/tools/cmake/toolchain-$(IDF_TARGET).cmake -DIDF_TARGET=$(IDF_TARGET) -DCIRCUITPY_MIPICSI=$(CIRCUITPY_MIPICSI) -GNinja $(Q)$(PYTHON) tools/check-sdkconfig.py \ CIRCUITPY_DUALBANK=$(CIRCUITPY_DUALBANK) \ CIRCUITPY_STORAGE_EXTEND=$(CIRCUITPY_STORAGE_EXTEND) \ @@ -1060,6 +1078,9 @@ endif ifneq ($(CIRCUITPY_MIPIDSI),0) ESP_IDF_COMPONENTS_LINK += esp_lcd endif +ifneq ($(CIRCUITPY_MIPICSI),0) + ESP_IDF_COMPONENTS_LINK += esp_driver_cam esp_driver_isp esp_hal_cam +endif ifneq ($(CIRCUITPY_PARALLELDISPLAYBUS),0) ESP_IDF_COMPONENTS_LINK += esp_lcd endif diff --git a/ports/espressif/boards/espressif_esp32p4_function_ev/sdkconfig b/ports/espressif/boards/espressif_esp32p4_function_ev/sdkconfig index fb8986613ff..8ad98a7b081 100644 --- a/ports/espressif/boards/espressif_esp32p4_function_ev/sdkconfig +++ b/ports/espressif/boards/espressif_esp32p4_function_ev/sdkconfig @@ -1,3 +1,27 @@ +# +# Espressif IoT Development Framework Configuration +# +# Bootloader +# +CONFIG_BOOTLOADER_LOG_LEVEL_WARN=y +CONFIG_BOOTLOADER_LOG_LEVEL=2 +# end of Bootloader + +# This board ships with pre-v3 (eco2) silicon: link the 2nd stage bootloader +# into the lower memory layout used by revisions 0.x and 1.x. CONFIG_ESP32P4_SELECTS_REV_LESS_V3=y +# This board's silicon is rev v1.3; match the working BSP-demo build (REV_MIN +# v1.0) so IDF compiles the same errata/clock handling as the reference stack. CONFIG_ESP32P4_REV_MIN_100=y CONFIG_ETH_USE_ESP32_EMAC=y + +# ESP-STDIO: send the ESP-IDF console to the USB-Serial-JTAG port so bootloader +# logs are visible on the board's serial connector. +# +CONFIG_ESP_CONSOLE_USB_SERIAL_JTAG=y +CONFIG_ESP_CONSOLE_USB_SERIAL_JTAG_ENABLED=y +CONFIG_ESP_CONSOLE_UART_NUM=-1 +CONFIG_ESP_CONSOLE_ROM_SERIAL_PORT_NUM=6 +# end of ESP-STDIO + +# CONFIG_ESP_SYSTEM_MEMPROT is not set diff --git a/ports/espressif/common-hal/mipicsi/Camera.c b/ports/espressif/common-hal/mipicsi/Camera.c new file mode 100644 index 00000000000..368061e53e7 --- /dev/null +++ b/ports/espressif/common-hal/mipicsi/Camera.c @@ -0,0 +1,468 @@ +// This file is part of the CircuitPython project: https://circuitpython.org +// +// SPDX-FileCopyrightText: Copyright (c) 2025 Jeff Epler for Adafruit Industries +// +// SPDX-License-Identifier: MIT + +#include "py/mperrno.h" +#include "py/mphal.h" +#include "py/nlr.h" +#include "py/runtime.h" + +#include "common-hal/mipicsi/Camera.h" +#include "shared-bindings/mipicsi/Camera.h" +#include "shared-bindings/mipicsi/__init__.h" +#include "bindings/espidf/__init__.h" + +#include "common-hal/mipidsi/Bus.h" + +#include +#include "esp_attr.h" +#include "esp_cache.h" +#include "esp_cam_ctlr.h" +#include "esp_private/esp_cache_private.h" +#include "esp_cam_ctlr_csi.h" +#include "hal/color_hal.h" + +// MIPI CSI receives data from a fixed, dedicated set of pins, so no pin +// configuration or claiming is done here. + +#define ISP_CLK_HZ (80 * 1000 * 1000) + +static cam_ctlr_color_t mipicsi_color_for(mipicsi_pixel_format_t format) { + switch (format) { + case MIPICSI_FORMAT_RAW8: + return CAM_CTLR_COLOR_RAW8; + case MIPICSI_FORMAT_RAW10: + return CAM_CTLR_COLOR_RAW10; + case MIPICSI_FORMAT_RAW12: + return CAM_CTLR_COLOR_RAW12; + case MIPICSI_FORMAT_RGB888: + return CAM_CTLR_COLOR_RGB888; + case MIPICSI_FORMAT_YUV422: + return CAM_CTLR_COLOR_YUV422_YUYV; + case MIPICSI_FORMAT_GRAYSCALE: + return CAM_CTLR_COLOR_GRAY8; + case MIPICSI_FORMAT_RGB565: + default: + return CAM_CTLR_COLOR_RGB565; + } +} + +static mp_int_t mipicsi_bits_per_pixel_for(mipicsi_pixel_format_t format) { + switch (format) { + case MIPICSI_FORMAT_RAW10: + return 10; + case MIPICSI_FORMAT_RAW12: + return 12; + case MIPICSI_FORMAT_RGB888: + return 24; + case MIPICSI_FORMAT_RGB565: + case MIPICSI_FORMAT_YUV422: + return 16; + case MIPICSI_FORMAT_RAW8: + case MIPICSI_FORMAT_GRAYSCALE: + default: + return 8; + } +} + +#if SOC_ISP_SUPPORTED +// The ISP color types share their FOURCC values with the CAM_CTLR color +// types, except for GRAYSCALE (no ISP grey mode) and YUV422 (the ISP input is +// UYVY, the CAM_CTLR YUYV). Only sensor RAW formats convert here. +static isp_color_t mipicsi_isp_color_for(mipicsi_pixel_format_t format) { + switch (format) { + case MIPICSI_FORMAT_RAW8: + return ISP_COLOR_RAW8; + case MIPICSI_FORMAT_RAW10: + return ISP_COLOR_RAW10; + case MIPICSI_FORMAT_RAW12: + return ISP_COLOR_RAW12; + case MIPICSI_FORMAT_RGB565: + return ISP_COLOR_RGB565; + case MIPICSI_FORMAT_RGB888: + return ISP_COLOR_RGB888; + case MIPICSI_FORMAT_YUV422: + case MIPICSI_FORMAT_GRAYSCALE: + default: + return 0; + } +} +#endif + +// Called by the driver (in ISR context) when the CSI host detects an error +// (PHY, packet, frame or CRC problems). Only installed for diagnostics: the +// counts are printed to the ROM console so they show up on the serial console. +static IRAM_ATTR bool mipicsi_on_error( + esp_cam_ctlr_handle_t handle, const esp_cam_ctlr_error_event_data_t *edata, void *user_data) { + mipicsi_camera_obj_t *self = user_data; + self->error_count++; + self->error_events |= edata->csi_host_err_evts; + if (self->error_count <= 3) { + esp_rom_printf("mipicsi: csi error #%lu events 0x%lx\n", + (unsigned long)self->error_count, (unsigned long)edata->csi_host_err_evts); + } + return false; +} + +// Called by the driver (in ISR context) to get the next buffer to DMA into. +static IRAM_ATTR bool mipicsi_on_get_new_trans( + esp_cam_ctlr_handle_t handle, esp_cam_ctlr_trans_t *trans, void *user_data) { + mipicsi_camera_obj_t *self = user_data; + + // Direct streaming: DMA the whole frame into the current destination. + // The videobuffer can change the destination between frames; the change + // is picked up here on the next frame. + // + // The controller only runs while a destination is set, so there is + // always somewhere to DMA into here; set_destination(NULL) stops the + // controller before clearing the destination. With the driver's backup + // buffer disabled, no buffer to hand over is unrecoverable (the driver + // asserts, or DMAs to address 0 in release builds), so there is no + // "give NULL" path. + trans->buffer = (void *)self->dest_base; + trans->buflen = self->buffer_len; + return false; +} + +// Called by the driver (in ISR context) when a frame has been written. +static IRAM_ATTR bool mipicsi_on_trans_finished( + esp_cam_ctlr_handle_t handle, esp_cam_ctlr_trans_t *trans, void *user_data) { + mipicsi_camera_obj_t *self = user_data; + + if (self->dest_base != NULL && self->dest_on_frame != NULL) { + self->dest_on_frame(self->dest_ctx); + } + return false; +} + +void common_hal_mipicsi_camera_construct( + mipicsi_camera_obj_t *self, + mipicsi_pixel_format_t output_format) { + + self->width = 1; + self->height = 1; + self->pixel_format = output_format; + self->output_format = output_format; + self->dest_base = NULL; + self->dest_on_frame = NULL; + self->dest_ctx = NULL; + self->ctlr_started = false; + self->error_count = 0; + self->error_events = 0; + self->sensor = mp_const_none; + self->phy_ldo = NULL; + self->isp_proc = NULL; + self->spinlock = (portMUX_TYPE)portMUX_INITIALIZER_UNLOCKED; + // No receiver hardware yet: it is created in the videosample play step, + // where the sensor's current mode is known. +} + +// Videosample protocol play step: create the LDO, ISP and CSI controller for +// the given (freshly read) sensor parameters and start the sensor streaming. +// Raises when the camera is already streaming for someone else. Not a user +// facing method: it is driven by the videosample protocol (from +// VideoBuffer.play), so it may raise. +void common_hal_mipicsi_camera_videosample_play( + mipicsi_camera_obj_t *self, + const mipicsi_sensor_params_t *params) { + + if (self->handle != NULL) { + mp_raise_RuntimeError(MP_ERROR_TEXT("already playing")); + } + + // Adopt what we need from the sensor's current mode; it may have + // changed since the Camera was made (sensors are free to reconfigure + // while they aren't being played). + self->width = params->width; + self->height = params->height; + self->bayer_order = params->bayer_order; + self->data_lanes = params->data_lanes; + self->lane_bit_rate_mbps = params->lane_bit_rate_mbps; + + const cam_ctlr_color_t color = mipicsi_color_for(self->pixel_format); + + #if SOC_ISP_SUPPORTED + if (self->output_format != self->pixel_format) { + // The sensor transmits in a different format than the one requested. + // On gen-1 P4 silicon the CSI receiver cannot convert, so route it + // through the ISP processor instead, mirroring the IDF + // esp_driver_cam csi test. + const isp_color_t isp_in = mipicsi_isp_color_for(self->pixel_format); + const isp_color_t isp_out = mipicsi_isp_color_for(self->output_format); + if (isp_in == 0 || (isp_in != ISP_COLOR_RAW8 && isp_in != ISP_COLOR_RAW10 && isp_in != ISP_COLOR_RAW12)) { + mp_raise_ValueError(MP_ERROR_TEXT("pixel_format must be RAW8, RAW10 or RAW12 when output_format differs")); + } + if (isp_out != ISP_COLOR_RGB565 && isp_out != ISP_COLOR_RGB888) { + mp_raise_ValueError(MP_ERROR_TEXT("output_format must be RGB565 or RGB888")); + } + esp_isp_processor_cfg_t isp_config = { + .clk_hz = ISP_CLK_HZ, + .input_data_source = ISP_INPUT_DATA_SOURCE_CSI, + .input_data_color_type = isp_in, + .output_data_color_type = isp_out, + .has_line_start_packet = false, + .has_line_end_packet = false, + .h_res = self->width, + .v_res = self->height, + .bayer_order = (color_raw_element_order_t)self->bayer_order, + }; + CHECK_ESP_RESULT(esp_isp_new_processor(&isp_config, &self->isp_proc)); + } + #else + if (self->output_format != self->pixel_format) { + mp_raise_NotImplementedError(MP_ERROR_TEXT("ISP conversion not supported on this chip")); + } + #endif + + // Power the MIPI D-PHY. The CSI and DSI PHYs share LDO channel 3; the + // shared reference count (see mipidsi_phy_ldo_acquire) keeps the rail up + // while either the camera or a mipidsi.Bus is alive. The sensor's I2C + // control interface is on a different rail and works without this. + self->phy_ldo = mipidsi_phy_ldo_acquire(); + + // The received images are not converted, so the input and output color + // types are the same and color conversion is bypassed. + esp_cam_ctlr_csi_config_t csi_config = { + .ctlr_id = 0, + .h_res = self->width, + .v_res = self->height, + .data_lane_num = self->data_lanes, + .lane_bit_rate_mbps = self->lane_bit_rate_mbps, + .input_data_color_type = color, + .output_data_color_type = color, + // One transaction queue item; frames go straight to the current + // destination. The controller is only started while a destination + // is set, so the driver's backup buffer is never needed and its + // frame-sized allocation is skipped. + .queue_items = 1, + .bk_buffer_dis = true, + }; + + esp_err_t result = esp_cam_new_csi_ctlr(&csi_config, &self->handle); + if (result != ESP_OK) { + mipidsi_phy_ldo_release(); + self->phy_ldo = NULL; + raise_esp_error(result); + } + + // Exact frame size. Destinations are written directly (no cache-line + // padding is ever DMA'd past the last row), so they only need to be wide + // enough to hold the whole frame; the DSI framebuffer always is. When + // the ISP is in the pipeline, the CSI receiver hands its converted + // output to us. + self->buffer_len = ((size_t)self->width * self->height * mipicsi_bits_per_pixel_for(self->output_format) + 7) / 8; + + esp_cam_ctlr_evt_cbs_t cbs = { + .on_get_new_trans = mipicsi_on_get_new_trans, + .on_trans_finished = mipicsi_on_trans_finished, + .on_error = mipicsi_on_error, + }; + result = esp_cam_ctlr_register_event_callbacks(self->handle, &cbs, self); + if (result != ESP_OK) { + common_hal_mipicsi_camera_deinit(self); + self->sensor = mp_const_none; + raise_esp_error(result); + } + + result = esp_cam_ctlr_enable(self->handle); + if (result != ESP_OK) { + common_hal_mipicsi_camera_deinit(self); + self->sensor = mp_const_none; + raise_esp_error(result); + } + #if SOC_ISP_SUPPORTED + if (self->isp_proc) { + result = esp_isp_enable(self->isp_proc); + if (result != ESP_OK) { + common_hal_mipicsi_camera_deinit(self); + self->sensor = mp_const_none; + raise_esp_error(result); + } + } + #endif + + // Start the sensor streaming so frames exist to capture once a + // destination is set. Sensors' _start() is expected to be idempotent. + // The camera never calls the sensor's stop(): when a stream ends (or on + // reload) only the CSI receiver and ISP are torn down; the sensor stops + // through its own finaliser (deinit/__del__). + if (self->sensor != mp_const_none) { + nlr_buf_t nlr; + if (nlr_push(&nlr) == 0) { + mp_obj_t start_method[2]; + mp_load_method(self->sensor, MP_QSTR__start, start_method); + mp_call_method_n_kw(0, 0, start_method); + nlr_pop(); + } else { + common_hal_mipicsi_camera_deinit(self); + self->sensor = mp_const_none; + nlr_raise(nlr.ret_val); + } + } + + // Don't start the controller yet: it only runs while a destination is + // set (see common_hal_mipicsi_camera_set_destination), so no frame is + // ever captured without a place to put it. +} + +// Tear down the receiver hardware quietly: this runs from Python (a +// VideoBuffer stopped) and from background contexts (destination dropped on +// reload), so it must not raise. The sensor is left alone: its own finaliser +// stops it. Only the CSI receiver and ISP turn off. +static void mipicsi_camera_teardown(mipicsi_camera_obj_t *self) { + if (self->ctlr_started) { + esp_cam_ctlr_stop(self->handle); + self->ctlr_started = false; + } + portENTER_CRITICAL(&self->spinlock); + self->dest_base = NULL; + self->dest_on_frame = NULL; + self->dest_ctx = NULL; + portEXIT_CRITICAL(&self->spinlock); + + nlr_buf_t nlr; + if (nlr_push(&nlr) == 0) { + if (self->handle != NULL) { + esp_cam_ctlr_disable(self->handle); + esp_cam_ctlr_del(self->handle); + self->handle = NULL; + } + #if SOC_ISP_SUPPORTED + if (self->isp_proc) { + esp_isp_disable(self->isp_proc); + esp_isp_del_processor(self->isp_proc); + self->isp_proc = NULL; + } + #endif + mipidsi_phy_ldo_release(); + self->phy_ldo = NULL; + nlr_pop(); + } else { + // Nothing to do about a failed teardown, not even a partially + // released one; swallow the exception and continue. + } +} + +void common_hal_mipicsi_camera_set_sensor(mipicsi_camera_obj_t *self, mp_obj_t sensor) { + self->sensor = sensor; +} + +void common_hal_mipicsi_camera_deinit(mipicsi_camera_obj_t *self) { + if (self->handle == NULL) { + // Not currently playing: nothing to tear down. + return; + } + + // Only turn off the CSI receiver and ISP here (including from background + // contexts such as reload): the sensor stops itself through its own + // finaliser (deinit/__del__). + mipicsi_camera_teardown(self); + self->sensor = mp_const_none; +} +// Videosample protocol stop step: tear down the receiver hardware and stop +// the sensor so its settings can be changed. Not a user facing method: it is +// driven by the videosample protocol (from VideoBuffer.stop), so it may +// raise; exceptions from the sensor's stop() propagate. +void common_hal_mipicsi_camera_videosample_stop(mipicsi_camera_obj_t *self) { + if (self->handle == NULL) { + return; // Not playing. + } + + // The receiver first: after the CSI controller stops, no more frames are + // captured, whatever the sensor does. + mipicsi_camera_teardown(self); + + // Stop the sensor (e.g. the sc2336 stops streaming and unlocks its + // mode). Sensors' _stop() is expected to be idempotent. + if (self->sensor != mp_const_none) { + mp_obj_t stop_method[2]; + mp_load_method(self->sensor, MP_QSTR__stop, stop_method); + mp_call_method_n_kw(0, 0, stop_method); + } +} + +bool common_hal_mipicsi_camera_deinited(mipicsi_camera_obj_t *self) { + return self->handle == NULL; +} + +bool common_hal_mipicsi_camera_set_destination( + mipicsi_camera_obj_t *self, + uint8_t *base, + size_t stride, + void (*on_frame)(void *ctx), + void *ctx) { + const size_t row_bytes = ((size_t)self->width * mipicsi_bits_per_pixel_for(self->output_format) + 7) / 8; + const size_t frame_bytes = (size_t)self->height * row_bytes; + + // There is no stop value on the destination: stopping is the videosample + // stop step's job. The protocol never passes NULL, + // so this guard is only about internal misuse. + assert(base != NULL); + + // DMA destinations must be cache-line aligned: the driver syncs the + // buffer on every frame with esp_cache_msync, which rejects anything + // else. PSRAM heap areas (and with 64-byte GC blocks, every GC + // allocation in them, e.g. bytearrays) are allocated 64-byte aligned on + // this chip, so this is a safety net. + size_t cache_line = 0; + esp_cache_get_alignment(MALLOC_CAP_SPIRAM, &cache_line); + if (cache_line == 0) { + cache_line = 64; + } + if ((uintptr_t)base % cache_line != 0) { + return false; + } + + // The destination is assumed to hold the frame rows with no stride + // padding: the CSI link-list DMA only does contiguous transfers in this + // IDF. The videobuffer validates the fit before getting here; the whole + // (padded) frame must lie within the destination: rows can't wrap past + // its end, so there is no cropping or overhang. + if (stride != row_bytes) { + return false; + } + + // Size handed to the driver: the frame, cache-line padded at the end so + // the driver's per-frame sync is happy. + const size_t buffer_len = (frame_bytes + cache_line - 1) / cache_line * cache_line; + + portENTER_CRITICAL(&self->spinlock); + self->dest_base = base; + self->dest_on_frame = on_frame; + self->dest_ctx = ctx; + self->buffer_len = buffer_len; + portEXIT_CRITICAL(&self->spinlock); + + // The sensor streaming and the receiver hardware exist from play(); + // here only the controller runs or doesn't. + if (!self->ctlr_started) { + esp_err_t result = esp_cam_ctlr_start(self->handle); + if (result != ESP_OK) { + return false; + } + self->ctlr_started = true; + } + return true; +} + +mipicsi_pixel_format_t common_hal_mipicsi_camera_get_pixel_format(mipicsi_camera_obj_t *self) { + return self->pixel_format; +} + +mipicsi_pixel_format_t common_hal_mipicsi_camera_get_output_format(mipicsi_camera_obj_t *self) { + return self->output_format; +} + +mp_int_t common_hal_mipicsi_camera_get_width(mipicsi_camera_obj_t *self) { + return self->width; +} + +mp_int_t common_hal_mipicsi_camera_get_height(mipicsi_camera_obj_t *self) { + return self->height; +} + +mp_int_t common_hal_mipicsi_camera_get_bits_per_pixel(mipicsi_camera_obj_t *self) { + return mipicsi_bits_per_pixel_for(self->output_format); +} diff --git a/ports/espressif/common-hal/mipicsi/Camera.h b/ports/espressif/common-hal/mipicsi/Camera.h new file mode 100644 index 00000000000..b3e0d269e5d --- /dev/null +++ b/ports/espressif/common-hal/mipicsi/Camera.h @@ -0,0 +1,89 @@ +// This file is part of the CircuitPython project: https://circuitpython.org +// +// SPDX-FileCopyrightText: Copyright (c) 2025 Jeff Epler for Adafruit Industries +// +// SPDX-License-Identifier: MIT + +#pragma once + +#include + +#include "py/obj.h" + +#include "freertos/FreeRTOS.h" +#include "freertos/portmacro.h" + +#include "esp_cam_ctlr.h" +#include "esp_ldo_regulator.h" + +#if SOC_ISP_SUPPORTED +#include "driver/isp.h" +#endif + +#include "shared-bindings/mipicsi/__init__.h" + +typedef struct mipicsi_camera_obj mipicsi_camera_obj_t; + +// Sensor parameters used to build the receiver. Read off the sensor object +// when a stream starts (the sensor is free to reconfigure between streams). +typedef struct { + mp_int_t width; + mp_int_t height; + mp_int_t bayer_order; + mp_int_t data_lanes; + mp_int_t lane_bit_rate_mbps; +} mipicsi_sensor_params_t; + +struct mipicsi_camera_obj { + mp_obj_base_t base; + esp_cam_ctlr_handle_t handle; + + // Power for the MIPI D-PHY, shared with the DSI transmitter. Acquired + // when a stream starts and released when it (or the camera) ends. + esp_ldo_channel_handle_t phy_ldo; + + // Camera sensor configured to transmit to this receiver, or None. + mp_obj_t sensor; + + // ISP processor used to convert sensor frames to output_format, only + // present when the CSI receiver itself cannot convert (e.g. gen-1 P4). + mipicsi_pixel_format_t output_format; + #if SOC_ISP_SUPPORTED + isp_proc_handle_t isp_proc; + #endif + + // The receiver hardware (LDO, ISP, CSI controller) is created when a + // stream starts (see play) and torn down when it ends. The sensor's + // parameters (width, height, data_lanes, lane_bit_rate_mbps, bayer_order) + // come from the sensor at play time as well; pixel_format only when it + // wasn't overridden at construction. + // Streaming destination, set by videocore.VideoBuffer. While base is + // NULL the CSI controller is stopped: no capture buffers are used at + // all. The destination is assumed to be a whole frame: DMA'd straight + // into base, stride equal to the row size. on_frame(ctx) is called from + // ISR context each time a frame lands in base. + // The DMA destination: set_destination requires it to be cache-line + // aligned (the driver's per-frame cache sync needs that) and reports a + // rejected destination with a false return, rather than shifting frames + // elsewhere. The frame must be contiguous. + volatile void *dest_base; + void (*dest_on_frame)(void *ctx); + void *dest_ctx; + bool ctlr_started; + + // Diagnostics from the CSI host error interrupt. + volatile uint32_t error_count; + volatile uint32_t error_events; + + portMUX_TYPE spinlock; + + mp_int_t width; + mp_int_t height; + mipicsi_pixel_format_t pixel_format; + mp_int_t data_lanes; + mp_int_t lane_bit_rate_mbps; + // Bayer element order of the sensor's RAW output, only used when the ISP + // converts from a RAW format. + mp_int_t bayer_order; + size_t buffer_len; // Length handed to the driver, includes cache-alignment padding. +}; diff --git a/ports/espressif/common-hal/mipicsi/__init__.c b/ports/espressif/common-hal/mipicsi/__init__.c new file mode 100644 index 00000000000..816fea93ae9 --- /dev/null +++ b/ports/espressif/common-hal/mipicsi/__init__.c @@ -0,0 +1,7 @@ +// This file is part of the CircuitPython project: https://circuitpython.org +// +// SPDX-FileCopyrightText: Copyright (c) 2025 Jeff Epler for Adafruit Industries +// +// SPDX-License-Identifier: MIT + +#include "shared-bindings/mipicsi/__init__.h" diff --git a/ports/espressif/common-hal/mipidsi/Bus.c b/ports/espressif/common-hal/mipidsi/Bus.c index dbb559ea30f..dab781f8cc9 100644 --- a/ports/espressif/common-hal/mipidsi/Bus.c +++ b/ports/espressif/common-hal/mipidsi/Bus.c @@ -9,6 +9,39 @@ #include "py/runtime.h" #include +// Shared MIPI PHY LDO rail. DSI buses and CSI receivers both need it; it is +// acquired once and released only when the last user goes away. +static esp_ldo_channel_handle_t mipidsi_phy_ldo = NULL; +static uint8_t mipidsi_phy_ldo_users = 0; + +esp_ldo_channel_handle_t mipidsi_phy_ldo_acquire(void) { + if (mipidsi_phy_ldo_users == 0) { + esp_ldo_channel_handle_t ldo = NULL; + esp_ldo_channel_config_t ldo_config = { + .chan_id = 3, + .voltage_mv = 2500, + // Fixed-voltage (non-adjustable) acquire so both users share the + // rail. An adjustable acquire would block the other user. + .flags.adjustable = false, + }; + ESP_ERROR_CHECK(esp_ldo_acquire_channel(&ldo_config, &ldo)); + mipidsi_phy_ldo = ldo; + } + mipidsi_phy_ldo_users++; + return mipidsi_phy_ldo; +} + +void mipidsi_phy_ldo_release(void) { + if (mipidsi_phy_ldo_users == 0) { + return; + } + mipidsi_phy_ldo_users--; + if (mipidsi_phy_ldo_users == 0 && mipidsi_phy_ldo != NULL) { + esp_ldo_release_channel(mipidsi_phy_ldo); + mipidsi_phy_ldo = NULL; + } +} + void common_hal_mipidsi_bus_construct(mipidsi_bus_obj_t *self, mp_uint_t frequency, uint8_t num_lanes) { self->frequency = frequency; self->num_data_lanes = num_lanes; @@ -18,12 +51,7 @@ void common_hal_mipidsi_bus_construct(mipidsi_bus_obj_t *self, mp_uint_t frequen mp_raise_ValueError_varg(MP_ERROR_TEXT("%q in use"), MP_QSTR_mipidsi); } - esp_ldo_channel_handle_t ldo_mipi_phy = NULL; - esp_ldo_channel_config_t ldo_mipi_phy_config = { - .chan_id = 3, - .voltage_mv = 2500, - }; - ESP_ERROR_CHECK(esp_ldo_acquire_channel(&ldo_mipi_phy_config, &ldo_mipi_phy)); + self->phy_ldo = mipidsi_phy_ldo_acquire(); // Create the DSI bus esp_lcd_dsi_bus_config_t bus_config = { @@ -50,6 +78,11 @@ void common_hal_mipidsi_bus_deinit(mipidsi_bus_obj_t *self) { self->bus_handle = NULL; } + // Release our hold on the shared MIPI PHY LDO; it stays powered as long + // as a CSI receiver (or other user) still holds it. + mipidsi_phy_ldo_release(); + self->phy_ldo = NULL; + self->frequency = 0; self->num_data_lanes = 0; } diff --git a/ports/espressif/common-hal/mipidsi/Bus.h b/ports/espressif/common-hal/mipidsi/Bus.h index b6257fd7223..1920275d430 100644 --- a/ports/espressif/common-hal/mipidsi/Bus.h +++ b/ports/espressif/common-hal/mipidsi/Bus.h @@ -8,14 +8,23 @@ #include "py/obj.h" #include +#include typedef struct { mp_obj_base_t base; mp_uint_t frequency; esp_lcd_dsi_bus_handle_t bus_handle; + esp_ldo_channel_handle_t phy_ldo; uint8_t num_data_lanes; uint8_t use_count; // Up to 4 displays } mipidsi_bus_obj_t; void mipidsi_bus_increment_use_count(mipidsi_bus_obj_t *self); void mipidsi_bus_decrement_use_count(mipidsi_bus_obj_t *self); + +// Acquire the shared MIPI PHY LDO rail (LDO channel 3, 2.5 V). DSI and CSI +// users share the rail; a shared reference count keeps it powered until the +// last user releases it, so the CSI receiver can stay active while a DSI bus +// comes and goes (and vice versa). +esp_ldo_channel_handle_t mipidsi_phy_ldo_acquire(void); +void mipidsi_phy_ldo_release(void); diff --git a/ports/espressif/common-hal/mipidsi/Display.c b/ports/espressif/common-hal/mipidsi/Display.c index ea204924fe6..9ebaec1bd35 100644 --- a/ports/espressif/common-hal/mipidsi/Display.c +++ b/ports/espressif/common-hal/mipidsi/Display.c @@ -13,6 +13,7 @@ #include "bindings/espidf/__init__.h" #include #include +#include #include "py/runtime.h" // Cache write-back function (should be from rom/cache.h but it's not always available) @@ -49,6 +50,7 @@ void common_hal_mipidsi_display_construct(mipidsi_display_obj_t *self, self->framebuffer = NULL; self->dbi_io_handle = NULL; self->dpi_panel_handle = NULL; + self->backlight_inout.base.type = &mp_type_NoneType; // Create the DBI interface for sending commands esp_lcd_dbi_io_config_t dbi_config = { @@ -111,6 +113,10 @@ void common_hal_mipidsi_display_construct(mipidsi_display_obj_t *self, self->framebuffer = (uint8_t *)fb; self->framebuffer_size = width * height * (color_depth / 8); + // The panel's reset line is not managed here, matching the other displayio + // display drivers: callers pulse/hold reset themselves (e.g. with + // digitalio) when a board needs it. + // Send initialization sequence (format matches busdisplay) #define DELAY 0x80 uint32_t i = 0; @@ -141,8 +147,12 @@ void common_hal_mipidsi_display_construct(mipidsi_display_obj_t *self, CHECK_ESP_RESULT(ret); } + // Send display-ON (0x29) after the DPI/video stream is running, matching + // the reference BSP flow order. + esp_lcd_panel_io_tx_param(self->dbi_io_handle, 0x29, NULL, 0); + common_hal_time_delay_ms(20); + // Setup backlight PWM if pin is provided - self->backlight_inout.base.type = &mp_type_NoneType; if (backlight_pin != NULL && common_hal_mcu_pin_is_free(backlight_pin)) { #if (CIRCUITPY_PWMIO) pwmout_result_t result = common_hal_pwmio_pwmout_construct(&self->backlight_pwm, backlight_pin, 0, 50000, false); @@ -192,6 +202,7 @@ void common_hal_mipidsi_display_deinit(mipidsi_display_obj_t *self) { common_hal_digitalio_digitalinout_deinit(&self->backlight_inout); } + // Delete the DPI panel if (self->dpi_panel_handle != NULL) { esp_lcd_panel_del(self->dpi_panel_handle); diff --git a/ports/espressif/mpconfigport.mk b/ports/espressif/mpconfigport.mk index 24760821290..6373fd82e35 100644 --- a/ports/espressif/mpconfigport.mk +++ b/ports/espressif/mpconfigport.mk @@ -402,6 +402,13 @@ CIRCUITPY_ESPCAMERA = 0 # P4 has MIPI-DSI CIRCUITPY_MIPIDSI = 1 +# P4 has MIPI-CSI +CIRCUITPY_MIPICSI = 1 + +# P4 can play MIPI-CSI video into MIPI-DSI framebuffers +CIRCUITPY_VIDEOCORE = 1 + + #### esp32s31 ######################################################### else ifeq ($(IDF_TARGET),esp32s31) diff --git a/ports/espressif/supervisor/port.c b/ports/espressif/supervisor/port.c index 91fe8cf5a2e..81eb4714db6 100644 --- a/ports/espressif/supervisor/port.c +++ b/ports/espressif/supervisor/port.c @@ -343,9 +343,11 @@ void *port_malloc(size_t size, bool dma_capable) { } void *ptr = NULL; - // Try SPIRAM first if available. + // Try SPIRAM first if available. Aligned to the GC block size so heap + // areas (and with 64-byte GC blocks, every GC allocation in them) are + // cache-line aligned, which video DMA destinations require. #ifdef CONFIG_SPIRAM - ptr = heap_caps_malloc(size, MALLOC_CAP_8BIT | MALLOC_CAP_SPIRAM); + ptr = heap_caps_aligned_alloc(MICROPY_BYTES_PER_GC_BLOCK, size, MALLOC_CAP_8BIT | MALLOC_CAP_SPIRAM); #endif if (ptr == NULL) { ptr = heap_caps_malloc(size, MALLOC_CAP_8BIT); diff --git a/py/circuitpy_defns.mk b/py/circuitpy_defns.mk index ec84d3ec381..8a4b9c59a18 100755 --- a/py/circuitpy_defns.mk +++ b/py/circuitpy_defns.mk @@ -324,6 +324,9 @@ endif ifeq ($(CIRCUITPY_MICROCONTROLLER),1) SRC_PATTERNS += microcontroller/% endif +ifeq ($(CIRCUITPY_MIPICSI),1) +SRC_PATTERNS += mipicsi/% +endif ifeq ($(CIRCUITPY_MIPIDSI),1) SRC_PATTERNS += mipidsi/% endif @@ -492,6 +495,9 @@ endif ifeq ($(CIRCUITPY_VECTORIO),1) SRC_PATTERNS += vectorio/% endif +ifeq ($(CIRCUITPY_VIDEOCORE),1) +SRC_PATTERNS += videocore/% +endif ifeq ($(CIRCUITPY_WARNINGS),1) SRC_PATTERNS += warnings/% endif @@ -586,6 +592,8 @@ SRC_COMMON_HAL_ALL = \ mdns/__init__.c \ mdns/Server.c \ mdns/RemoteService.c \ + mipicsi/__init__.c \ + mipicsi/Camera.c \ mipidsi/Bus.c \ mipidsi/Display.c \ mipidsi/__init__.c \ @@ -690,6 +698,8 @@ $(filter $(SRC_PATTERNS), \ imagecapture/ParallelImageCapture.c \ locale/__init__.c \ math/__init__.c \ + mipicsi/BayerOrder.c \ + mipicsi/PixelFormat.c \ microcontroller/ResetReason.c \ microcontroller/RunMode.c \ msgpack/__init__.c \ @@ -895,6 +905,8 @@ SRC_SHARED_MODULE_ALL = \ vectorio/Rectangle.c \ vectorio/VectorShape.c \ vectorio/__init__.c \ + videocore/VideoBuffer.c \ + videocore/__init__.c \ warnings/__init__.c \ watchdog/__init__.c \ zlib/__init__.c \ diff --git a/py/circuitpy_mpconfig.mk b/py/circuitpy_mpconfig.mk index 0fc82d03704..1d8bedebbc5 100755 --- a/py/circuitpy_mpconfig.mk +++ b/py/circuitpy_mpconfig.mk @@ -469,6 +469,9 @@ CFLAGS += -DCIRCUITPY_MEMORYMONITOR=$(CIRCUITPY_MEMORYMONITOR) CIRCUITPY_MICROCONTROLLER ?= 1 CFLAGS += -DCIRCUITPY_MICROCONTROLLER=$(CIRCUITPY_MICROCONTROLLER) +CIRCUITPY_MIPICSI ?= 0 +CFLAGS += -DCIRCUITPY_MIPICSI=$(CIRCUITPY_MIPICSI) + CIRCUITPY_MIPIDSI ?= 0 CFLAGS += -DCIRCUITPY_MIPIDSI=$(CIRCUITPY_MIPIDSI) @@ -828,6 +831,12 @@ CFLAGS += -DCIRCUITPY_ULAB=$(CIRCUITPY_ULAB) # element-wise array arithmetic roughly 1.5x slower. CIRCUITPY_ULAB_OPTIMIZE_SIZE ?= 0 +# Video pipeline: playing video sources into framebuffers. Requires a port +# with a video source such as mipicsi and a framebuffer output such as +# mipidsi. +CIRCUITPY_VIDEOCORE ?= 0 +CFLAGS += -DCIRCUITPY_VIDEOCORE=$(CIRCUITPY_VIDEOCORE) + CIRCUITPY_WARNINGS ?= $(CIRCUITPY_FULL_BUILD) CFLAGS += -DCIRCUITPY_WARNINGS=$(CIRCUITPY_WARNINGS) diff --git a/shared-bindings/adafruit_pixelbuf/PixelBuf.c b/shared-bindings/adafruit_pixelbuf/PixelBuf.c index 4144f086b98..dd5e4d7b4ea 100644 --- a/shared-bindings/adafruit_pixelbuf/PixelBuf.c +++ b/shared-bindings/adafruit_pixelbuf/PixelBuf.c @@ -31,7 +31,13 @@ static MP_NORETURN void invalid_byteorder(void) { static void parse_byteorder(mp_obj_t byteorder_obj, pixelbuf_byteorder_details_t *parsed); //| class PixelBuf: -//| """A fast RGB[W] pixel buffer for LED and similar devices.""" +//| """A fast RGB[W] pixel buffer for LED and similar devices. +//| +//| Subclasses must define ``_transmit(self, buffer)``, which `show` calls +//| with a `bytearray` of the bytes to send: the header, the pixel +//| values, and the trailer. ``_transmit`` is internal, for PixelBuf's +//| use only; user code shows pixels through `show` and `auto_write`. +//| """ //| //| def __init__( //| self, diff --git a/shared-bindings/busdisplay/BusDisplay.c b/shared-bindings/busdisplay/BusDisplay.c index 0bf171a5d08..57fc1afa71c 100644 --- a/shared-bindings/busdisplay/BusDisplay.c +++ b/shared-bindings/busdisplay/BusDisplay.c @@ -16,6 +16,9 @@ #include "shared-bindings/displayio/Group.h" #include "shared-bindings/microcontroller/Pin.h" #include "shared-bindings/util.h" +#if CIRCUITPY_VIDEOCORE +#include "shared-module/videocore/__init__.h" +#endif #include "shared-module/displayio/__init__.h" //| import displayio @@ -408,6 +411,11 @@ static mp_obj_t busdisplay_busdisplay_obj_set_root_group(mp_obj_t self_in, mp_ob displayio_group_t *group = NULL; if (group_in != mp_const_none) { group = MP_OBJ_TO_PTR(native_group(group_in)); + #if CIRCUITPY_VIDEOCORE + // Validate that draw protocol layers (e.g. a VideoBuffer) fit the + // display they will render on. + videocore_validate_root_group(group_in, self_in); + #endif } common_hal_busdisplay_busdisplay_set_root_group(self, group); diff --git a/shared-bindings/framebufferio/FramebufferDisplay.c b/shared-bindings/framebufferio/FramebufferDisplay.c index 71ce18a1b4b..da7f18e4caf 100644 --- a/shared-bindings/framebufferio/FramebufferDisplay.c +++ b/shared-bindings/framebufferio/FramebufferDisplay.c @@ -17,6 +17,9 @@ #include "shared-bindings/displayio/Group.h" #include "shared-bindings/microcontroller/Pin.h" #include "shared-bindings/util.h" +#if CIRCUITPY_VIDEOCORE +#include "shared-module/videocore/__init__.h" +#endif #include "shared-module/displayio/__init__.h" //| class FramebufferDisplay: @@ -325,6 +328,11 @@ static mp_obj_t framebufferio_framebufferdisplay_obj_set_root_group(mp_obj_t sel displayio_group_t *group = NULL; if (group_in != mp_const_none) { group = MP_OBJ_TO_PTR(native_group(group_in)); + #if CIRCUITPY_VIDEOCORE + // Validate that draw protocol layers (e.g. a VideoBuffer) fit the + // display they will render on. + videocore_validate_root_group(group_in, self_in); + #endif } bool ok = common_hal_framebufferio_framebufferdisplay_set_root_group(self, group); diff --git a/shared-bindings/mipicsi/BayerOrder.c b/shared-bindings/mipicsi/BayerOrder.c new file mode 100644 index 00000000000..9f1058dd3e9 --- /dev/null +++ b/shared-bindings/mipicsi/BayerOrder.c @@ -0,0 +1,50 @@ +// This file is part of the CircuitPython project: https://circuitpython.org +// +// SPDX-FileCopyrightText: Copyright (c) 2026 Scott Shawcroft for Adafruit Industries +// +// SPDX-License-Identifier: MIT + +#include + +#include "py/enum.h" +#include "py/obj.h" +#include "py/runtime.h" + +#include "shared-bindings/mipicsi/__init__.h" + +//| class BayerOrder: +//| """The Bayer element order of a sensor's RAW output, in the sequence a 2x2 +//| pixel block transmits (top-left first, reading order)""" +//| +//| BGGR: BayerOrder +//| """The first element is Blue, then Green, then the next row starts with Green, then Red""" +//| +//| GBRG: BayerOrder +//| """The first element is Green, then Blue, then the next row starts with Red, then Green""" +//| +//| GRBG: BayerOrder +//| """The first element is Green, then Red, then the next row starts with Blue, then Green""" +//| +//| RGGB: BayerOrder +//| """The first element is Red, then Green, then the next row starts with Green, then Blue""" +//| + +MAKE_ENUM_VALUE(mipicsi_bayer_order_type, bayer_order, BGGR, MIPICSI_BAYER_ORDER_BGGR); +MAKE_ENUM_VALUE(mipicsi_bayer_order_type, bayer_order, GBRG, MIPICSI_BAYER_ORDER_GBRG); +MAKE_ENUM_VALUE(mipicsi_bayer_order_type, bayer_order, GRBG, MIPICSI_BAYER_ORDER_GRBG); +MAKE_ENUM_VALUE(mipicsi_bayer_order_type, bayer_order, RGGB, MIPICSI_BAYER_ORDER_RGGB); + +MAKE_ENUM_MAP(mipicsi_bayer_order) { + MAKE_ENUM_MAP_ENTRY(bayer_order, BGGR), + MAKE_ENUM_MAP_ENTRY(bayer_order, GBRG), + MAKE_ENUM_MAP_ENTRY(bayer_order, GRBG), + MAKE_ENUM_MAP_ENTRY(bayer_order, RGGB), +}; + +static MP_DEFINE_CONST_DICT(mipicsi_bayer_order_locals_dict, mipicsi_bayer_order_locals_table); +MAKE_PRINTER(mipicsi, mipicsi_bayer_order); +MAKE_ENUM_TYPE(mipicsi, BayerOrder, mipicsi_bayer_order); + +mipicsi_bayer_order_t validate_bayer_order(mp_obj_t obj, qstr arg_name) { + return cp_enum_value(&mipicsi_bayer_order_type, obj, arg_name); +} diff --git a/shared-bindings/mipicsi/Camera.c b/shared-bindings/mipicsi/Camera.c new file mode 100644 index 00000000000..c899dda2d71 --- /dev/null +++ b/shared-bindings/mipicsi/Camera.c @@ -0,0 +1,231 @@ +// This file is part of the CircuitPython project: https://circuitpython.org +// +// SPDX-FileCopyrightText: Copyright (c) 2025 Jeff Epler for Adafruit Industries +// +// SPDX-License-Identifier: MIT + +#include + +#include "py/mphal.h" +#include "py/obj.h" +#include "py/objproperty.h" +#include "py/runtime.h" + +#include "shared-bindings/displayio/Bitmap.h" +#include "shared-bindings/mipicsi/__init__.h" +#include "shared-bindings/mipicsi/Camera.h" +#include "shared-bindings/util.h" +#include "shared/runtime/context_manager_helpers.h" + +#if CIRCUITPY_VIDEOCORE +#include "shared-module/videocore/__init__.h" +#endif + +//| class Camera: +//| """Receive frames from a camera sensor on a MIPI CSI-2 interface +//| +//| The camera sensor is configured separately (typically over I2C) to +//| transmit frames. The MIPI CSI-2 receiver hardware has dedicated pins, +//| so no pins are configured here. +//| +//| Pass the configured sensor to the constructor; the receiver matches +//| itself to the sensor's current mode and starts it streaming. The +//| sensor object must provide the properties and methods of +//| ``circuitpython_typing.CameraSensor``: ``width``, ``height``, +//| ``pixel_format``, ``lane_bit_rate_mbps``, ``data_lanes`` and +//| ``bayer_order`` (a `BayerOrder`) properties plus ``_start()`` and +//| ``_stop()`` methods. Those two are internal: the camera calls them +//| itself as the stream starts and stops, so user code should not. +//| All are validated here (and again at play, so the sensor may change +//| its configuration freely while it isn't being played). +//| +//| The sensor decides the output image size, and the receiver is built +//| around it: a sensor cannot change its mode while frames are captured. +//| Sensors are started (``_start()``) when a `videocore.VideoBuffer` +//| starts playing the camera; they are never stopped by the camera, so +//| implementations should refuse to reconfigure (raise) while streaming +//| rather than change the output size or format mid-stream, stop through +//| their own finaliser, and have a new Camera made when the mode +//| changes. +//| +//| Camera objects implement the internal videosample protocol, so a +//| `videocore.VideoBuffer` can stream their frames into a framebuffer. +//| """ +//| +//| def __init__( +//| self, +//| sensor: circuitpython_typing.CameraSensor, +//| *, +//| output_format: Optional[PixelFormat] = None, +//| ) -> None: +//| """ +//| Configure and initialize the MIPI CSI-2 camera receiver +//| +//| .. code-block:: python +//| +//| sensor = sc2336.SC2336(i2c) +//| camera = mipicsi.Camera(sensor) +//| +//| :param sensor: A camera sensor to receive frames from. The receiver +//| is configured to match the sensor's current mode. The sensor +//| starts streaming when a `videocore.VideoBuffer` plays the +//| camera. +//| :param output_format: The format of the captured frames. When given and different from ``pixel_format``, the ISP converts each frame as it is captured (e.g. the sensor's RAW10 to `PixelFormat.RGB565`). Only RAW formats convert. Default: no conversion. +//| """ +//| +// Read the sensor's parameters off of the sensor object. A missing or +// invalid property fails here rather than later; mp_load_attr raises +// AttributeError for any property the sensor object doesn't provide. When +// pixel_format_override is not None it replaces the sensor's reported +// transmission format. +static mipicsi_pixel_format_t mipicsi_camera_load_sensor_params( + mp_obj_t sensor, + mipicsi_sensor_params_t *params) { + params->width = mp_arg_validate_int_min(mp_obj_get_int(mp_load_attr(sensor, MP_QSTR_width)), 1, MP_QSTR_width); + params->height = mp_arg_validate_int_min(mp_obj_get_int(mp_load_attr(sensor, MP_QSTR_height)), 1, MP_QSTR_height); + params->lane_bit_rate_mbps = mp_arg_validate_int_min(mp_obj_get_int(mp_load_attr(sensor, MP_QSTR_lane_bit_rate_mbps)), 1, MP_QSTR_lane_bit_rate_mbps); + params->data_lanes = mp_arg_validate_int_range(mp_obj_get_int(mp_load_attr(sensor, MP_QSTR_data_lanes)), 1, 2, MP_QSTR_data_lanes); + params->bayer_order = validate_bayer_order(mp_load_attr(sensor, MP_QSTR_bayer_order), MP_QSTR_bayer_order); + return validate_pixel_format(mp_load_attr(sensor, MP_QSTR_pixel_format), MP_QSTR_pixel_format); +} + +static mp_obj_t mipicsi_camera_make_new(const mp_obj_type_t *type, size_t n_args, size_t n_kw, const mp_obj_t *all_args) { + mp_arg_check_num(n_args, n_kw, 1, 1, true); + mp_obj_t sensor = all_args[0]; + + enum { ARG_output_format, NUM_ARGS }; + static const mp_arg_t allowed_args[] = { + { MP_QSTR_output_format, MP_ARG_OBJ | MP_ARG_KW_ONLY, { .u_obj = MP_ROM_NONE } }, + }; + mp_arg_val_t args[MP_ARRAY_SIZE(allowed_args)]; + MP_STATIC_ASSERT(MP_ARRAY_SIZE(allowed_args) == NUM_ARGS); + mp_arg_parse_all_kw_array(0, n_kw, all_args + 1, MP_ARRAY_SIZE(allowed_args), allowed_args, args); + + // All of the sensor's parameters (including the transmission format) are + // read off of the sensor object now, so a missing or invalid property + // fails here rather than at play time. They are read again at play, + // since the sensor is free to reconfigure between streams. + mipicsi_sensor_params_t params; + mipicsi_pixel_format_t pixel_format = mipicsi_camera_load_sensor_params(sensor, ¶ms); + mipicsi_pixel_format_t output_format = pixel_format; + if (args[ARG_output_format].u_obj != mp_const_none) { + output_format = validate_pixel_format(args[ARG_output_format].u_obj, MP_QSTR_output_format); + } + + // Validate the sensor contract methods (_start/_stop) exist now too, + // since the camera drives them from C as the stream starts and stops. + mp_obj_t validated[2]; + mp_load_method(sensor, MP_QSTR__start, validated); + mp_load_method(sensor, MP_QSTR__stop, validated); + + mipicsi_camera_obj_t *self = mp_obj_malloc_with_finaliser(mipicsi_camera_obj_t, &mipicsi_camera_type); + common_hal_mipicsi_camera_construct(self, output_format); + // Keep the sensor alive while receiving from it. It is only started + // streaming when a videocore.VideoBuffer starts playing (see the + // videosample play step): until then nothing captures or transmits. + common_hal_mipicsi_camera_set_sensor(self, sensor); + return MP_OBJ_FROM_PTR(self); +} + +//| def deinit(self) -> None: +//| """Deinitialises the camera receiver and releases all memory resources for reuse.""" +//| ... +//| +static mp_obj_t mipicsi_camera_deinit(mp_obj_t self_in) { + mipicsi_camera_obj_t *self = MP_OBJ_TO_PTR(self_in); + common_hal_mipicsi_camera_deinit(self); + return mp_const_none; +} +static MP_DEFINE_CONST_FUN_OBJ_1(mipicsi_camera_deinit_obj, mipicsi_camera_deinit); + +//| def __enter__(self) -> Camera: +//| """No-op used by Context Managers.""" +//| ... +//| +//| def __exit__(self) -> None: +//| """Automatically deinitializes the hardware when exiting a context. See +//| :ref:`lifetime-and-contextmanagers` for more info.""" +//| ... +//| +//| + +static const mp_rom_map_elem_t mipicsi_camera_locals_dict_table[] = { + { MP_ROM_QSTR(MP_QSTR___del__), MP_ROM_PTR(&mipicsi_camera_deinit_obj) }, + { MP_ROM_QSTR(MP_QSTR___enter__), MP_ROM_PTR(&default___enter___obj) }, + { MP_ROM_QSTR(MP_QSTR___exit__), MP_ROM_PTR(&default___exit___obj) }, + { MP_ROM_QSTR(MP_QSTR_deinit), MP_ROM_PTR(&mipicsi_camera_deinit_obj) }, +}; + +static MP_DEFINE_CONST_DICT(mipicsi_camera_locals_dict, mipicsi_camera_locals_dict_table); + +#if CIRCUITPY_VIDEOCORE +// Implementation of the videocore videosample protocol so a VideoBuffer can +// tell the Camera where to put frames. +static void mipicsi_camera_play(mp_obj_t self_in) { + mipicsi_camera_obj_t *self = MP_OBJ_TO_PTR(mp_obj_cast_to_native_base(self_in, &mipicsi_camera_type)); + // The camera always has a sensor from make_new on. deinit() clears it + // (the sensor stops through its own finaliser), so a missing sensor only + // means the camera was deinitialised. + if (self->sensor == mp_const_none) { + raise_deinited_error(); + } + // Re-read the sensor's mode now: it is free to reconfigure between + // streams, and only a mode change while streaming is prevented (the + // sensor refuses it itself). + mipicsi_sensor_params_t params; + self->pixel_format = mipicsi_camera_load_sensor_params(self->sensor, ¶ms); + common_hal_mipicsi_camera_videosample_play(self, ¶ms); +} + +static void mipicsi_camera_stop(mp_obj_t self_in) { + mipicsi_camera_obj_t *self = MP_OBJ_TO_PTR(mp_obj_cast_to_native_base(self_in, &mipicsi_camera_type)); + common_hal_mipicsi_camera_videosample_stop(self); +} + +static bool mipicsi_camera_set_destination(mp_obj_t self_in, const videocore_destination_t *dest, + videocore_on_frame_fun on_frame, void *ctx) { + mipicsi_camera_obj_t *self = MP_OBJ_TO_PTR(mp_obj_cast_to_native_base(self_in, &mipicsi_camera_type)); + // The protocol has no stop value on the destination: stopping is stop()'s + // job. + return common_hal_mipicsi_camera_set_destination(self, dest->base, dest->stride, on_frame, ctx); +} + +static mp_int_t mipicsi_camera_get_width(mp_obj_t self_in) { + return common_hal_mipicsi_camera_get_width(mp_obj_cast_to_native_base(self_in, &mipicsi_camera_type)); +} + +static mp_int_t mipicsi_camera_get_height(mp_obj_t self_in) { + return common_hal_mipicsi_camera_get_height(mp_obj_cast_to_native_base(self_in, &mipicsi_camera_type)); +} + +static mp_int_t mipicsi_camera_get_bits_per_pixel(mp_obj_t self_in) { + return common_hal_mipicsi_camera_get_bits_per_pixel(mp_obj_cast_to_native_base(self_in, &mipicsi_camera_type)); +} + +static const videosample_p_t mipicsi_camera_videosample_protocol = { + MP_PROTO_IMPLEMENT(MP_QSTR_protocol_videosample) + .play = mipicsi_camera_play, + .stop = mipicsi_camera_stop, + .set_destination = mipicsi_camera_set_destination, + .get_width = mipicsi_camera_get_width, + .get_height = mipicsi_camera_get_height, + .get_bits_per_pixel = mipicsi_camera_get_bits_per_pixel, +}; + +MP_DEFINE_CONST_OBJ_TYPE( + mipicsi_camera_type, + MP_QSTR_Camera, + MP_TYPE_FLAG_HAS_SPECIAL_ACCESSORS, + make_new, mipicsi_camera_make_new, + locals_dict, &mipicsi_camera_locals_dict, + protocol, &mipicsi_camera_videosample_protocol + ); +#else +MP_DEFINE_CONST_OBJ_TYPE( + mipicsi_camera_type, + MP_QSTR_Camera, + MP_TYPE_FLAG_HAS_SPECIAL_ACCESSORS, + make_new, mipicsi_camera_make_new, + locals_dict, &mipicsi_camera_locals_dict + ); +#endif diff --git a/shared-bindings/mipicsi/Camera.h b/shared-bindings/mipicsi/Camera.h new file mode 100644 index 00000000000..d39e748936d --- /dev/null +++ b/shared-bindings/mipicsi/Camera.h @@ -0,0 +1,60 @@ +// This file is part of the CircuitPython project: https://circuitpython.org +// +// SPDX-FileCopyrightText: Copyright (c) 2025 Jeff Epler for Adafruit Industries +// +// SPDX-License-Identifier: MIT + +#pragma once + +#include "py/obj.h" + +#include "common-hal/mipicsi/Camera.h" +#include "shared-bindings/mipicsi/__init__.h" + +extern const mp_obj_type_t mipicsi_camera_type; + +// Record the output format (possibly the sensor's). No hardware is created +// here: that happens in play(). +void common_hal_mipicsi_camera_construct( + mipicsi_camera_obj_t *self, + mipicsi_pixel_format_t output_format); + +// Remember the sensor configured to transmit to this receiver, if any. The +// receiver reads the sensor's mode from it at play time; the sensor stops +// itself (its finaliser does), never the receiver. +void common_hal_mipicsi_camera_set_sensor(mipicsi_camera_obj_t *self, mp_obj_t sensor); + +// Videosample protocol play step: create the receiver hardware for the +// sensor's current mode and start the sensor streaming. Not a user facing +// method: it is driven by the videosample protocol (from VideoBuffer.play), +// so it may raise. +void common_hal_mipicsi_camera_videosample_play( + mipicsi_camera_obj_t *self, + const mipicsi_sensor_params_t *params); + +// Videosample protocol stop step: stop the stream and the sensor. Not a user +// facing method: it is driven by the videosample protocol (from +// VideoBuffer.stop), so it may raise. +void common_hal_mipicsi_camera_videosample_stop(mipicsi_camera_obj_t *self); + +void common_hal_mipicsi_camera_deinit(mipicsi_camera_obj_t *self); +bool common_hal_mipicsi_camera_deinited(mipicsi_camera_obj_t *self); + +// Stream frames directly into base row after row. base must lead to a +// cache-line-aligned, contiguous DMA region: when base itself isn't aligned, +// the destination is not accepted. The destination must hold the whole +// (padded) frame back to back: the caller validates the fit, since the DMA +// has no stride. Returns whether those conditions hold (false instead of +// raising; the caller may ignore the result). +bool common_hal_mipicsi_camera_set_destination( + mipicsi_camera_obj_t *self, + uint8_t *base, + size_t stride, + void (*on_frame)(void *ctx), + void *ctx); + +mipicsi_pixel_format_t common_hal_mipicsi_camera_get_pixel_format(mipicsi_camera_obj_t *self); +mipicsi_pixel_format_t common_hal_mipicsi_camera_get_output_format(mipicsi_camera_obj_t *self); +mp_int_t common_hal_mipicsi_camera_get_width(mipicsi_camera_obj_t *self); +mp_int_t common_hal_mipicsi_camera_get_height(mipicsi_camera_obj_t *self); +mp_int_t common_hal_mipicsi_camera_get_bits_per_pixel(mipicsi_camera_obj_t *self); diff --git a/shared-bindings/mipicsi/PixelFormat.c b/shared-bindings/mipicsi/PixelFormat.c new file mode 100644 index 00000000000..84340019745 --- /dev/null +++ b/shared-bindings/mipicsi/PixelFormat.c @@ -0,0 +1,66 @@ +// This file is part of the CircuitPython project: https://circuitpython.org +// +// SPDX-FileCopyrightText: Copyright (c) 2025 Jeff Epler for Adafruit Industries +// +// SPDX-License-Identifier: MIT + +#include + +#include "py/enum.h" +#include "py/obj.h" +#include "py/runtime.h" + +#include "shared-bindings/mipicsi/__init__.h" + +//| class PixelFormat: +//| """Format of the data in the captured frames. The camera sensor must be +//| configured (typically over I2C) to transmit the corresponding MIPI CSI-2 +//| data type.""" +//| +//| RGB565: PixelFormat +//| """A 16-bit format with 5 bits of Red, 6 bits of Green, and 5 bits of Blue (MIPI data type ``RGB565``)""" +//| +//| RGB888: PixelFormat +//| """A 24-bit format with 8 bits of Red, Green and Blue (MIPI data type ``RGB888``)""" +//| +//| GRAYSCALE: PixelFormat +//| """An 8-bit format with 8 bits of luminance (MIPI data types for 8-bit grayscale)""" +//| +//| YUV422: PixelFormat +//| """A 16-bit YUV 4:2:2 format, in the byte order the sensor transmits (MIPI data type ``YUV422 8-bit``)""" +//| +//| RAW8: PixelFormat +//| """8 bits per pixel of raw Bayer data (MIPI data type ``RAW8``)""" +//| +//| RAW10: PixelFormat +//| """10 bits per pixel of raw Bayer data (MIPI data type ``RAW10``)""" +//| +//| RAW12: PixelFormat +//| """12 bits per pixel of raw Bayer data (MIPI data type ``RAW12``)""" +//| + +MAKE_ENUM_VALUE(mipicsi_pixel_format_type, pixel_format, RGB565, MIPICSI_FORMAT_RGB565); +MAKE_ENUM_VALUE(mipicsi_pixel_format_type, pixel_format, RGB888, MIPICSI_FORMAT_RGB888); +MAKE_ENUM_VALUE(mipicsi_pixel_format_type, pixel_format, GRAYSCALE, MIPICSI_FORMAT_GRAYSCALE); +MAKE_ENUM_VALUE(mipicsi_pixel_format_type, pixel_format, YUV422, MIPICSI_FORMAT_YUV422); +MAKE_ENUM_VALUE(mipicsi_pixel_format_type, pixel_format, RAW8, MIPICSI_FORMAT_RAW8); +MAKE_ENUM_VALUE(mipicsi_pixel_format_type, pixel_format, RAW10, MIPICSI_FORMAT_RAW10); +MAKE_ENUM_VALUE(mipicsi_pixel_format_type, pixel_format, RAW12, MIPICSI_FORMAT_RAW12); + +MAKE_ENUM_MAP(mipicsi_pixel_format) { + MAKE_ENUM_MAP_ENTRY(pixel_format, RGB565), + MAKE_ENUM_MAP_ENTRY(pixel_format, RGB888), + MAKE_ENUM_MAP_ENTRY(pixel_format, GRAYSCALE), + MAKE_ENUM_MAP_ENTRY(pixel_format, YUV422), + MAKE_ENUM_MAP_ENTRY(pixel_format, RAW8), + MAKE_ENUM_MAP_ENTRY(pixel_format, RAW10), + MAKE_ENUM_MAP_ENTRY(pixel_format, RAW12), +}; + +static MP_DEFINE_CONST_DICT(mipicsi_pixel_format_locals_dict, mipicsi_pixel_format_locals_table); +MAKE_PRINTER(mipicsi, mipicsi_pixel_format); +MAKE_ENUM_TYPE(mipicsi, PixelFormat, mipicsi_pixel_format); + +mipicsi_pixel_format_t validate_pixel_format(mp_obj_t obj, qstr arg_name) { + return cp_enum_value(&mipicsi_pixel_format_type, obj, arg_name); +} diff --git a/shared-bindings/mipicsi/__init__.c b/shared-bindings/mipicsi/__init__.c new file mode 100644 index 00000000000..d46e0eedf1d --- /dev/null +++ b/shared-bindings/mipicsi/__init__.c @@ -0,0 +1,46 @@ +// This file is part of the CircuitPython project: https://circuitpython.org +// +// SPDX-FileCopyrightText: Copyright (c) 2025 Jeff Epler for Adafruit Industries +// +// SPDX-License-Identifier: MIT + +#include + +#include "py/enum.h" +#include "py/obj.h" +#include "py/runtime.h" + +#include "shared-bindings/mipicsi/__init__.h" +#include "shared-bindings/mipicsi/Camera.h" + +//| """Support for MIPI CSI-2 camera interfaces +//| +//| Captures image frames from camera sensors attached to a MIPI CSI-2 +//| receiver, such as the one in the ESP32-P4. The camera sensor itself is +//| configured separately (typically over I2C); this module only receives +//| the image data the sensor transmits. +//| +//| The image data arrives in the format the sensor transmits, described by +//| `PixelFormat`. RAW formats carry Bayer data, whose element order is +//| described by `BayerOrder`. For instance, a sensor transmitting RGB565 +//| data yields frames of RGB565 pixels. Optionally, `mipicsi.Camera` can +//| convert frames with the ISP processor (set ``output_format`` and +//| ``pixel_format``), for example from the sensor's RAW output to RGB565 for +//| display. +//| """ + +static const mp_rom_map_elem_t mipicsi_module_globals_table[] = { + { MP_ROM_QSTR(MP_QSTR___name__), MP_ROM_QSTR(MP_QSTR_mipicsi) }, + { MP_ROM_QSTR(MP_QSTR_PixelFormat), MP_ROM_PTR(&mipicsi_pixel_format_type) }, + { MP_ROM_QSTR(MP_QSTR_BayerOrder), MP_ROM_PTR(&mipicsi_bayer_order_type) }, + { MP_ROM_QSTR(MP_QSTR_Camera), MP_ROM_PTR(&mipicsi_camera_type) }, +}; + +static MP_DEFINE_CONST_DICT(mipicsi_module_globals, mipicsi_module_globals_table); + +const mp_obj_module_t mipicsi_module = { + .base = { &mp_type_module }, + .globals = (mp_obj_dict_t *)&mipicsi_module_globals, +}; + +MP_REGISTER_MODULE(MP_QSTR_mipicsi, mipicsi_module); diff --git a/shared-bindings/mipicsi/__init__.h b/shared-bindings/mipicsi/__init__.h new file mode 100644 index 00000000000..5892379ce88 --- /dev/null +++ b/shared-bindings/mipicsi/__init__.h @@ -0,0 +1,33 @@ +// This file is part of the CircuitPython project: https://circuitpython.org +// +// SPDX-FileCopyrightText: Copyright (c) 2025 Jeff Epler for Adafruit Industries +// +// SPDX-License-Identifier: MIT + +#pragma once + +#include "py/enum.h" + +extern const mp_obj_type_t mipicsi_pixel_format_type; +extern const mp_obj_type_t mipicsi_bayer_order_type; +extern const cp_enum_obj_t pixel_format_RGB565_obj; + +typedef enum { + MIPICSI_FORMAT_RAW8, + MIPICSI_FORMAT_RAW10, + MIPICSI_FORMAT_RAW12, + MIPICSI_FORMAT_RGB565, + MIPICSI_FORMAT_RGB888, + MIPICSI_FORMAT_YUV422, + MIPICSI_FORMAT_GRAYSCALE, +} mipicsi_pixel_format_t; + +typedef enum { + MIPICSI_BAYER_ORDER_BGGR, + MIPICSI_BAYER_ORDER_GBRG, + MIPICSI_BAYER_ORDER_GRBG, + MIPICSI_BAYER_ORDER_RGGB, +} mipicsi_bayer_order_t; + +mipicsi_pixel_format_t validate_pixel_format(mp_obj_t obj, qstr arg_name); +mipicsi_bayer_order_t validate_bayer_order(mp_obj_t obj, qstr arg_name); diff --git a/shared-bindings/vectorio/__init__.h b/shared-bindings/vectorio/__init__.h index c5dcc026567..cc9b523ecb1 100644 --- a/shared-bindings/vectorio/__init__.h +++ b/shared-bindings/vectorio/__init__.h @@ -23,6 +23,9 @@ typedef bool (*draw_get_dirty_area_fun)(mp_obj_t draw_protocol_self, displayio_a typedef void (*draw_update_transform_fun)(mp_obj_t draw_protocol_self, displayio_buffer_transform_t *group_transform); typedef void (*draw_finish_refresh_fun)(mp_obj_t draw_protocol_self); typedef void (*draw_set_dirty_fun)(mp_obj_t draw_protocol_self); +// Optional: propagate a group's hidden state to the layer. May be NULL; the +// group then falls back to draw_set_dirty. +typedef void (*draw_set_hidden_fun)(mp_obj_t draw_protocol_self, bool hidden); typedef bool (*draw_set_in_group_fun)(mp_obj_t draw_protocol_self, bool in_group); typedef displayio_area_t *(*draw_get_refresh_areas_fun)(mp_obj_t draw_protocol_self, displayio_area_t *tail); @@ -33,6 +36,8 @@ typedef struct _vectorio_draw_protocol_impl_t { draw_finish_refresh_fun draw_finish_refresh; draw_get_refresh_areas_fun draw_get_refresh_areas; draw_set_dirty_fun draw_set_dirty; + // May be NULL. + draw_set_hidden_fun draw_set_hidden; draw_set_in_group_fun draw_set_in_group; } vectorio_draw_protocol_impl_t; diff --git a/shared-bindings/videocore/VideoBuffer.c b/shared-bindings/videocore/VideoBuffer.c new file mode 100644 index 00000000000..eb1a415f9bd --- /dev/null +++ b/shared-bindings/videocore/VideoBuffer.c @@ -0,0 +1,241 @@ +// This file is part of the CircuitPython project: https://circuitpython.org +// +// SPDX-FileCopyrightText: Copyright (c) 2026 Scott Shawcroft for Adafruit Industries +// +// SPDX-License-Identifier: MIT + +#include "py/obj.h" +#include "py/objproperty.h" +#include "py/runtime.h" + +#include "shared-bindings/videocore/VideoBuffer.h" +#include "shared-bindings/util.h" +#include "shared/runtime/context_manager_helpers.h" + +//| class VideoBuffer: +//| """Stream a video source into a framebuffer or buffer. +//| +//| A VideoBuffer can be added to displayio.Groups to show on a display +//| but may have size and position limitations. +//| +//| A VideoBuffer connects a video source (an object implementing the +//| videosample protocol, such as `mipicsi.Camera`) to a target: either a +//| framebuffer (an object implementing the framebuffer protocol, such as +//| `mipidsi.Display`) or a writable buffer such as a ``bytearray`` or +//| ``memoryview``. It is the video counterpart of `audioio.AudioOut`. +//| +//| The VideoBuffer tells the source where to put frames and moves the +//| destination as needed; the source streams continuously into it. While +//| playing, the target always holds the newest frame. +//| +//| When the target is a framebuffer, position the video with ``x`` and +//| ``y``: they may be changed at any time, including while playing, which +//| moves where the source puts its next frame. Some implementations may +//| restrict size and position. Setting position directly will raise an +//| error but indirectly through Group may result in weird output. For +//| example, on ESP32-P4 a VideoBuffer written into the framebuffer +//| directly must be full width because the ESP can't handle a stride +//| without an intermediate buffer. +//| +//| When the target is a plain buffer, frames are written compactly (row +//| stride equals ``width * bits_per_pixel / 8``) and can be read from +//| Python at any time — for example to process or save camera pixels. +//| +//| Two complete examples are in the `videocore` module overview: one +//| streaming a camera directly into a display, and one capturing a +//| 320x240 window into a ``bytearray`` that displayio places in a Group. +//| """ +//| +//| def __init__( +//| self, +//| framebuffer: Union[ +//| circuitpython_typing.FrameBuffer, circuitpython_typing.WriteableBuffer +//| ], +//| ) -> None: +//| """Create a VideoBuffer that plays video into the given framebuffer +//| +//| :param framebuffer: The framebuffer to copy video frames into, or a +//| `framebufferio.FramebufferDisplay` wrapping one, or a writable +//| buffer such as a ``bytearray`` or ``memoryview`` to read the raw +//| pixels from +//| """ +//| +static mp_obj_t videocore_videobuffer_make_new(const mp_obj_type_t *type, size_t n_args, size_t n_kw, const mp_obj_t *all_args) { + enum { ARG_framebuffer, NUM_ARGS }; + static const mp_arg_t allowed_args[] = { + { MP_QSTR_framebuffer, MP_ARG_REQUIRED | MP_ARG_OBJ }, + }; + + mp_arg_val_t args[MP_ARRAY_SIZE(allowed_args)]; + MP_STATIC_ASSERT(MP_ARRAY_SIZE(allowed_args) == NUM_ARGS); + mp_arg_parse_all_kw_array(n_args, n_kw, all_args, MP_ARRAY_SIZE(allowed_args), allowed_args, args); + + videocore_videobuffer_obj_t *self = mp_obj_malloc_with_finaliser(videocore_videobuffer_obj_t, &videocore_videobuffer_type); + common_hal_videocore_videobuffer_construct(self, args[ARG_framebuffer].u_obj); + return MP_OBJ_FROM_PTR(self); +} + +//| def play(self, source: circuitpython_typing.VideoSample) -> None: +//| """Begin streaming the given video source into the target. +//| +//| The source streams continuously, writing each new frame where the +//| VideoBuffer tells it to, replacing the previous one. The source's +//| frame format must be byte aligned. When the target is a +//| framebuffer, its bits per pixel must match the framebuffer color +//| depth and the whole frame must fit within the framebuffer at the +//| current ``x``/``y`` position (rows can't wrap past its end). When +//| the target is a plain buffer, it must be large enough for a whole +//| frame. Buffer size is validated here; the +//| source's size and format are then fixed for the duration of the +//| play. Sources that change them require ``stop`` and a new +//| ``play``. +//| +//| :param ~circuitpython_typing.VideoSample source: The video source to +//| play, such as `mipicsi.Camera` +//| """ +//| +static mp_obj_t videocore_videobuffer_play(mp_obj_t self_in, mp_obj_t source) { + videocore_videobuffer_obj_t *self = MP_OBJ_TO_PTR(self_in); + common_hal_videocore_videobuffer_play(self, source); + return mp_const_none; +} +static MP_DEFINE_CONST_FUN_OBJ_2(videocore_videobuffer_play_obj, videocore_videobuffer_play); + +//| def stop(self) -> None: +//| """Stop playing the video source.""" +//| ... +//| +static mp_obj_t videocore_videobuffer_stop(mp_obj_t self_in) { + videocore_videobuffer_obj_t *self = MP_OBJ_TO_PTR(self_in); + common_hal_videocore_videobuffer_stop(self); + return mp_const_none; +} +static MP_DEFINE_CONST_FUN_OBJ_1(videocore_videobuffer_stop_obj, videocore_videobuffer_stop); + +//| playing: bool +//| """True when a video source is being played, False otherwise. (read-only)""" +//| +static mp_obj_t videocore_videobuffer_get_playing(mp_obj_t self_in) { + videocore_videobuffer_obj_t *self = MP_OBJ_TO_PTR(self_in); + return mp_obj_new_bool(common_hal_videocore_videobuffer_get_playing(self)); +} +static MP_DEFINE_CONST_FUN_OBJ_1(videocore_videobuffer_get_playing_obj, videocore_videobuffer_get_playing); +MP_PROPERTY_GETTER(videocore_videobuffer_playing_obj, + (mp_obj_t)&videocore_videobuffer_get_playing_obj); + +//| source: Optional[circuitpython_typing.VideoSample] +//| """The video source being played, or None when not playing. (read-only)""" +//| +static mp_obj_t videocore_videobuffer_get_source(mp_obj_t self_in) { + videocore_videobuffer_obj_t *self = MP_OBJ_TO_PTR(self_in); + return common_hal_videocore_videobuffer_get_source(self); +} +static MP_DEFINE_CONST_FUN_OBJ_1(videocore_videobuffer_get_source_obj, videocore_videobuffer_get_source); +MP_PROPERTY_GETTER(videocore_videobuffer_source_obj, + (mp_obj_t)&videocore_videobuffer_get_source_obj); + +//| x: int +//| """The x position of the video within the display. Framebuffer targets +//| must be at x == 0 (video sources DMA whole rows): ``ValueError`` +//| otherwise. Plain buffer targets only use it to place the video in a +//| Group: it doesn't change where frames land within the buffer.""" +//| +//| y: int +//| """The y position of the video within the display.""" +//| +//| +static void videocore_videobuffer_get_position(mp_obj_t self_in, mp_int_t *x, mp_int_t *y) { + videocore_videobuffer_obj_t *self = MP_OBJ_TO_PTR(self_in); + *x = self->x; + *y = self->y; +} + +static mp_obj_t videocore_videobuffer_get_x(mp_obj_t self_in) { + mp_int_t x, y; + videocore_videobuffer_get_position(self_in, &x, &y); + return MP_OBJ_NEW_SMALL_INT(x); +} +static MP_DEFINE_CONST_FUN_OBJ_1(videocore_videobuffer_get_x_obj, videocore_videobuffer_get_x); + +static mp_obj_t videocore_videobuffer_set_x(mp_obj_t self_in, mp_obj_t value) { + videocore_videobuffer_obj_t *self = MP_OBJ_TO_PTR(self_in); + mp_int_t x = mp_obj_get_int(value); + common_hal_videocore_videobuffer_set_position(self, x, self->y); + return mp_const_none; +} +static MP_DEFINE_CONST_FUN_OBJ_2(videocore_videobuffer_set_x_obj, videocore_videobuffer_set_x); +MP_PROPERTY_GETSET(videocore_videobuffer_x_obj, + (mp_obj_t)&videocore_videobuffer_get_x_obj, + (mp_obj_t)&videocore_videobuffer_set_x_obj); + +static mp_obj_t videocore_videobuffer_get_y(mp_obj_t self_in) { + mp_int_t x, y; + videocore_videobuffer_get_position(self_in, &x, &y); + return MP_OBJ_NEW_SMALL_INT(y); +} +static MP_DEFINE_CONST_FUN_OBJ_1(videocore_videobuffer_get_y_obj, videocore_videobuffer_get_y); + +static mp_obj_t videocore_videobuffer_set_y(mp_obj_t self_in, mp_obj_t value) { + videocore_videobuffer_obj_t *self = MP_OBJ_TO_PTR(self_in); + mp_int_t y = mp_obj_get_int(value); + common_hal_videocore_videobuffer_set_position(self, self->x, y); + return mp_const_none; +} +static MP_DEFINE_CONST_FUN_OBJ_2(videocore_videobuffer_set_y_obj, videocore_videobuffer_set_y); +MP_PROPERTY_GETSET(videocore_videobuffer_y_obj, + (mp_obj_t)&videocore_videobuffer_get_y_obj, + (mp_obj_t)&videocore_videobuffer_set_y_obj); + +//| def deinit(self) -> None: +//| """Stop playing. The video source stops itself.""" +//| ... +//| +//| def __enter__(self) -> VideoBuffer: +//| """No-op used by Context Managers.""" +//| ... +//| +//| def __exit__(self) -> None: +//| """Automatically deinitializes the hardware when exiting a context. See +//| :ref:`lifetime-and-contextmanagers` for more info.""" +//| ... +//| +static mp_obj_t videocore_videobuffer_deinit(mp_obj_t self_in) { + videocore_videobuffer_obj_t *self = MP_OBJ_TO_PTR(self_in); + common_hal_videocore_videobuffer_stop(self); + return mp_const_none; +} +static MP_DEFINE_CONST_FUN_OBJ_1(videocore_videobuffer_deinit_obj, videocore_videobuffer_deinit); + +// The finaliser must not raise (it runs during garbage collection), so stop +// without one instead of deinit(): a failed stop stays best-effort, matching +// the source's own finaliser. +static mp_obj_t videocore_videobuffer___del__(mp_obj_t self_in) { + videocore_videobuffer_obj_t *self = MP_OBJ_TO_PTR(self_in); + videocore_videobuffer_stop_quiet(self); + return mp_const_none; +} +static MP_DEFINE_CONST_FUN_OBJ_1(videocore_videobuffer___del___obj, videocore_videobuffer___del__); + +static const mp_rom_map_elem_t videocore_videobuffer_locals_dict_table[] = { + { MP_ROM_QSTR(MP_QSTR___del__), MP_ROM_PTR(&videocore_videobuffer___del___obj) }, + { MP_ROM_QSTR(MP_QSTR___enter__), MP_ROM_PTR(&default___enter___obj) }, + { MP_ROM_QSTR(MP_QSTR___exit__), MP_ROM_PTR(&default___exit___obj) }, + { MP_ROM_QSTR(MP_QSTR_deinit), MP_ROM_PTR(&videocore_videobuffer_deinit_obj) }, + { MP_ROM_QSTR(MP_QSTR_play), MP_ROM_PTR(&videocore_videobuffer_play_obj) }, + { MP_ROM_QSTR(MP_QSTR_stop), MP_ROM_PTR(&videocore_videobuffer_stop_obj) }, + { MP_ROM_QSTR(MP_QSTR_playing), MP_ROM_PTR(&videocore_videobuffer_playing_obj) }, + { MP_ROM_QSTR(MP_QSTR_source), MP_ROM_PTR(&videocore_videobuffer_source_obj) }, + { MP_ROM_QSTR(MP_QSTR_x), MP_ROM_PTR(&videocore_videobuffer_x_obj) }, + { MP_ROM_QSTR(MP_QSTR_y), MP_ROM_PTR(&videocore_videobuffer_y_obj) }, +}; + +static MP_DEFINE_CONST_DICT(videocore_videobuffer_locals_dict, videocore_videobuffer_locals_dict_table); + +MP_DEFINE_CONST_OBJ_TYPE( + videocore_videobuffer_type, + MP_QSTR_VideoBuffer, + MP_TYPE_FLAG_HAS_SPECIAL_ACCESSORS, + make_new, videocore_videobuffer_make_new, + locals_dict, &videocore_videobuffer_locals_dict, + protocol, &videocore_videobuffer_draw_protocol + ); diff --git a/shared-bindings/videocore/VideoBuffer.h b/shared-bindings/videocore/VideoBuffer.h new file mode 100644 index 00000000000..927169bc4b3 --- /dev/null +++ b/shared-bindings/videocore/VideoBuffer.h @@ -0,0 +1,11 @@ +// This file is part of the CircuitPython project: https://circuitpython.org +// +// SPDX-FileCopyrightText: Copyright (c) 2026 Scott Shawcroft for Adafruit Industries +// +// SPDX-License-Identifier: MIT + +#pragma once + +#include "shared-module/videocore/VideoBuffer.h" + +extern const mp_obj_type_t videocore_videobuffer_type; diff --git a/shared-bindings/videocore/__init__.c b/shared-bindings/videocore/__init__.c new file mode 100644 index 00000000000..cc55331b6df --- /dev/null +++ b/shared-bindings/videocore/__init__.c @@ -0,0 +1,86 @@ +// This file is part of the CircuitPython project: https://circuitpython.org +// +// SPDX-FileCopyrightText: Copyright (c) 2026 Scott Shawcroft for Adafruit Industries +// +// SPDX-License-Identifier: MIT + +#include "py/obj.h" +#include "py/runtime.h" + +#include "shared-bindings/videocore/VideoBuffer.h" +#include "shared-bindings/videocore/__init__.h" + +//| """Support for playing video sources +//| +//| This module provides the video pipeline shared between video sources and +//| video outputs. It is the video equivalent of the `audiocore` module. +//| +//| Video sources, such as `mipicsi.Camera`, implement the internal videosample +//| protocol. A `VideoBuffer` tells a video source where to put its frames in a +//| framebuffer, such as `mipidsi.Display`, and can move it as needed. +//| +//| Stream a MIPI CSI camera live into the board's display (a MIPI DSI panel +//| on ESP32-P4-Function-EV, for example): +//| +//| .. code-block:: python +//| +//| import board +//| import mipicsi +//| import sc2336 +//| import videocore +//| +//| display = board.DISPLAY() +//| sensor = sc2336.SC2336(board.I2C()) +//| sensor.mode = sc2336.Mode.HD_1280x720_RAW10_30FPS +//| # Trim the sensor output to the display size. +//| sensor.width = display.width +//| sensor.height = display.height +//| camera = mipicsi.Camera(sensor, output_format=mipicsi.PixelFormat.RGB565) +//| +//| video = videocore.VideoBuffer(display) +//| video.play(camera) # camera frames stream live into the panel +//| +//| Capture a smaller part of the sensor's image into a `bytearray` and let +//| displayio place it in a Group, together with other layers. displayio bulk +//| copies new frames into the group's area of the display (no color +//| conversion): +//| +//| .. code-block:: python +//| +//| import board +//| import displayio +//| import mipicsi +//| import sc2336 +//| import videocore +//| +//| sensor = sc2336.SC2336(board.I2C()) +//| sensor.mode = sc2336.Mode.VGA_640x480_RAW10_50FPS +//| # Trim the sensor output to a 320x240 window. +//| sensor.width = 320 +//| sensor.height = 240 +//| camera = mipicsi.Camera(sensor, output_format=mipicsi.PixelFormat.RGB565) +//| +//| capture = videocore.VideoBuffer(bytearray(320 * 240 * 2)) +//| capture.play(camera) +//| +//| root = displayio.Group() +//| root.append(capture) +//| capture.x = 350 # position of the live video within the group +//| capture.y = 180 +//| board.DISPLAY().root_group = root # live video joins the displayio scene +//| """ +//| + +static const mp_rom_map_elem_t videocore_module_globals_table[] = { + { MP_ROM_QSTR(MP_QSTR___name__), MP_ROM_QSTR(MP_QSTR_videocore) }, + { MP_ROM_QSTR(MP_QSTR_VideoBuffer), MP_ROM_PTR(&videocore_videobuffer_type) }, +}; + +static MP_DEFINE_CONST_DICT(videocore_module_globals, videocore_module_globals_table); + +const mp_obj_module_t videocore_module = { + .base = { &mp_type_module }, + .globals = (mp_obj_dict_t *)&videocore_module_globals, +}; + +MP_REGISTER_MODULE(MP_QSTR_videocore, videocore_module); diff --git a/shared-bindings/videocore/__init__.h b/shared-bindings/videocore/__init__.h new file mode 100644 index 00000000000..d309fcc838b --- /dev/null +++ b/shared-bindings/videocore/__init__.h @@ -0,0 +1,9 @@ +// This file is part of the CircuitPython project: https://circuitpython.org +// +// SPDX-FileCopyrightText: Copyright (c) 2026 Scott Shawcroft for Adafruit Industries +// +// SPDX-License-Identifier: MIT + +#pragma once + +#include "shared-module/videocore/__init__.h" diff --git a/shared-module/displayio/Group.c b/shared-module/displayio/Group.c index 84ebffdab6d..1e84da278f6 100644 --- a/shared-module/displayio/Group.c +++ b/shared-module/displayio/Group.c @@ -10,7 +10,7 @@ #include "py/objlist.h" #include "shared-bindings/displayio/TileGrid.h" -#if CIRCUITPY_VECTORIO +#if CIRCUITPY_VECTORIO || CIRCUITPY_VIDEOCORE #include "shared-bindings/vectorio/VectorShape.h" #endif @@ -52,11 +52,15 @@ void common_hal_displayio_group_set_hidden(displayio_group_t *self, bool hidden) displayio_group_set_hidden_by_parent(layer, hidden); continue; } - #if CIRCUITPY_VECTORIO + #if CIRCUITPY_VECTORIO || CIRCUITPY_VIDEOCORE const vectorio_draw_protocol_t *draw_protocol = mp_proto_get(MP_QSTR_protocol_draw, self->members->items[i]); if (draw_protocol != NULL) { layer = draw_protocol->draw_get_protocol_self(self->members->items[i]); - draw_protocol->draw_protocol_impl->draw_set_dirty(layer); + if (draw_protocol->draw_protocol_impl->draw_set_hidden != NULL) { + draw_protocol->draw_protocol_impl->draw_set_hidden(layer, hidden); + } else { + draw_protocol->draw_protocol_impl->draw_set_dirty(layer); + } continue; } #endif @@ -87,11 +91,15 @@ void displayio_group_set_hidden_by_parent(displayio_group_t *self, bool hidden) displayio_group_set_hidden_by_parent(layer, hidden); continue; } - #if CIRCUITPY_VECTORIO + #if CIRCUITPY_VECTORIO || CIRCUITPY_VIDEOCORE const vectorio_draw_protocol_t *draw_protocol = mp_proto_get(MP_QSTR_protocol_draw, self->members->items[i]); if (draw_protocol != NULL) { layer = draw_protocol->draw_get_protocol_self(self->members->items[i]); - draw_protocol->draw_protocol_impl->draw_set_dirty(layer); + if (draw_protocol->draw_protocol_impl->draw_set_hidden != NULL) { + draw_protocol->draw_protocol_impl->draw_set_hidden(layer, hidden); + } else { + draw_protocol->draw_protocol_impl->draw_set_dirty(layer); + } continue; } #endif @@ -146,7 +154,7 @@ static void _update_child_transforms(displayio_group_t *self) { } for (size_t i = 0; i < self->members->len; i++) { mp_obj_t layer; - #if CIRCUITPY_VECTORIO + #if CIRCUITPY_VECTORIO || CIRCUITPY_VIDEOCORE const vectorio_draw_protocol_t *draw_protocol = mp_proto_get(MP_QSTR_protocol_draw, self->members->items[i]); if (draw_protocol != NULL) { layer = draw_protocol->draw_get_protocol_self(self->members->items[i]); @@ -248,7 +256,7 @@ void common_hal_displayio_group_set_y(displayio_group_t *self, mp_int_t y) { static void _add_layer(displayio_group_t *self, mp_obj_t layer) { check_readonly(self); - #if CIRCUITPY_VECTORIO + #if CIRCUITPY_VECTORIO || CIRCUITPY_VIDEOCORE const vectorio_draw_protocol_t *draw_protocol = mp_proto_get(MP_QSTR_protocol_draw, layer); if (draw_protocol != NULL) { mp_obj_t protocol_self = draw_protocol->draw_get_protocol_self(layer); @@ -293,7 +301,7 @@ static void _remove_layer(displayio_group_t *self, size_t index) { mp_obj_t layer; displayio_area_t layer_area; bool rendered_last_frame = false; - #if CIRCUITPY_VECTORIO + #if CIRCUITPY_VECTORIO || CIRCUITPY_VIDEOCORE const vectorio_draw_protocol_t *draw_protocol = mp_proto_get(MP_QSTR_protocol_draw, self->members->items[index]); if (draw_protocol != NULL) { layer = draw_protocol->draw_get_protocol_self(self->members->items[index]); @@ -377,7 +385,7 @@ bool displayio_group_fill_area(displayio_group_t *self, const _displayio_colorsp if (self->hidden == false) { for (int32_t i = self->members->len - 1; i >= 0; i--) { mp_obj_t layer; - #if CIRCUITPY_VECTORIO + #if CIRCUITPY_VECTORIO || CIRCUITPY_VIDEOCORE const vectorio_draw_protocol_t *draw_protocol = mp_proto_get(MP_QSTR_protocol_draw, self->members->items[i]); if (draw_protocol != NULL) { layer = draw_protocol->draw_get_protocol_self(self->members->items[i]); @@ -412,7 +420,7 @@ void displayio_group_finish_refresh(displayio_group_t *self) { self->item_removed = false; for (int32_t i = self->members->len - 1; i >= 0; i--) { mp_obj_t layer; - #if CIRCUITPY_VECTORIO + #if CIRCUITPY_VECTORIO || CIRCUITPY_VIDEOCORE const vectorio_draw_protocol_t *draw_protocol = mp_proto_get(MP_QSTR_protocol_draw, self->members->items[i]); if (draw_protocol != NULL) { layer = draw_protocol->draw_get_protocol_self(self->members->items[i]); @@ -443,7 +451,7 @@ static displayio_area_t *group_get_refresh_areas_impl(displayio_group_t *self, d for (int32_t i = self->members->len - 1; i >= 0; i--) { mp_obj_t layer; - #if CIRCUITPY_VECTORIO + #if CIRCUITPY_VECTORIO || CIRCUITPY_VIDEOCORE const vectorio_draw_protocol_t *draw_protocol = mp_proto_get(MP_QSTR_protocol_draw, self->members->items[i]); if (draw_protocol != NULL) { layer = draw_protocol->draw_get_protocol_self(self->members->items[i]); diff --git a/shared-module/videocore/VideoBuffer.c b/shared-module/videocore/VideoBuffer.c new file mode 100644 index 00000000000..c9aca74c197 --- /dev/null +++ b/shared-module/videocore/VideoBuffer.c @@ -0,0 +1,547 @@ +// This file is part of the CircuitPython project: https://circuitpython.org +// +// SPDX-FileCopyrightText: Copyright (c) 2026 Scott Shawcroft for Adafruit Industries +// +// SPDX-License-Identifier: MIT + +#include + +#include "shared-module/videocore/VideoBuffer.h" + +#include "py/runtime.h" + +#include "shared-bindings/displayio/Group.h" +#include "shared-bindings/videocore/VideoBuffer.h" +#include "shared-bindings/videocore/__init__.h" +#include "shared-module/displayio/Group.h" +#include "shared-module/videocore/__init__.h" + +#if CIRCUITPY_FRAMEBUFFERIO +#include "shared-bindings/framebufferio/FramebufferDisplay.h" +#endif + +#define FB_GETTER(method, default_value) \ + (self->framebuffer_protocol->method \ + ? self->framebuffer_protocol->method(self->framebuffer) \ + : (default_value)) + +// Called from interrupt context when the source lands a frame. The frame +// landed in stream_base, the buffer the accepted destination gave (set when +// the last set_destination returned true), so keep it tiny. +static void videocore_videobuffer_on_frame(void *ctx) { + videocore_videobuffer_obj_t *self = ctx; + self->frame_ready = true; +} + +// Tear down playback state: clear our fields. +static void videocore_videobuffer_clear(videocore_videobuffer_obj_t *self); + +// Recompute the rendered rect (screen coordinates) of the video from the +// x/y position and the absolute transform. Video only supports scale 1 with +// no mirroring (validated in update_transform), so the rect is the frame +// size translated into place. +static void videocore_videobuffer_update_area(videocore_videobuffer_obj_t *self) { + const displayio_buffer_transform_t *t = &self->absolute_transform; + self->current_area.x1 = self->x + t->x; + self->current_area.y1 = self->y + t->y; + self->current_area.x2 = self->current_area.x1 + self->source_width; + self->current_area.y2 = self->current_area.y1 + self->source_height; + self->current_area.next = NULL; +} + +// Compute the destination for the source in the current target, honoring the +// current x/y position and the absolute transform's translation (a video in +// a group). Raises unless the whole frame fits in the target: the camera +// DMA's whole, contiguous frames with no stride padding, so rows can't wrap +// past the buffer and there is no cropping yet. The caller validates the +// fit (this runs from Python contexts); framebuffers in groups also move +// their destination from draw contexts with clamping (see _move_dest). +static void videocore_videobuffer_compute_destination( + videocore_videobuffer_obj_t *self, videocore_destination_t *dest) { + const size_t bytes_per_pixel = self->source_bits_per_pixel / 8; + const size_t row_bytes = (size_t)self->source_width * bytes_per_pixel; + + if (self->framebuffer_protocol == NULL) { + // Raw buffer mode: frames are written compactly from the start. + dest->base = self->bufinfo.buf; + dest->stride = row_bytes; + dest->row_bytes = row_bytes; + dest->height = self->source_height; + if (self->bufinfo.len < self->source_frame_bytes) { + mp_raise_ValueError(MP_ERROR_TEXT("Buffer too small")); + } + return; + } + + mp_buffer_info_t bufinfo; + self->framebuffer_protocol->get_bufinfo(self->framebuffer, &bufinfo); + if (bufinfo.buf == NULL) { + mp_raise_ValueError(MP_ERROR_TEXT("framebuffer went away")); + } + + const mp_int_t fb_width = FB_GETTER(get_width, 0); + const mp_int_t row_stride = FB_GETTER(get_row_stride, fb_width * self->source_bits_per_pixel / 8); + const mp_int_t first_pixel_offset = FB_GETTER(get_first_pixel_offset, 0); + + // The absolute position includes a containing group's translation. The + // shift may land the frame's rows mid-row: the image won't look right, + // but the contiguous DMA stays contiguous. Only the frame's start and + // end must remain within the buffer. + const displayio_buffer_transform_t *t = &self->absolute_transform; + const mp_int_t shift_x = self->x + t->x; + const mp_int_t shift_y = self->y + t->y; + if (self->x < 0 || self->y < 0 || shift_x < 0 || shift_y < 0 || + first_pixel_offset + shift_y * row_stride + shift_x * bytes_per_pixel + + (mp_int_t)self->source_frame_bytes > + bufinfo.len) { + mp_raise_ValueError(MP_ERROR_TEXT("video doesn't fit in framebuffer")); + } + + dest->base = (uint8_t *)bufinfo.buf + first_pixel_offset + + shift_y * row_stride + shift_x * bytes_per_pixel; + dest->stride = row_stride; + dest->row_bytes = row_bytes; + dest->height = self->source_height; +} + +// Move the framebuffer destination for a new absolute position, capping the +// shift so the frame stays completely within the framebuffer. Best effort: +// never raises, because this runs from displayio's refresh. +static void videocore_videobuffer_move_dest( + videocore_videobuffer_obj_t *self, mp_int_t shift_x, mp_int_t shift_y) { + if (!self->playing) { + return; + } + mp_buffer_info_t bufinfo; + self->framebuffer_protocol->get_bufinfo(self->framebuffer, &bufinfo); + if (bufinfo.buf == NULL) { + return; + } + const size_t bytes_per_pixel = self->source_bits_per_pixel / 8; + const size_t row_bytes = (size_t)self->source_width * bytes_per_pixel; + const size_t frame_bytes = (size_t)self->source_height * row_bytes; + const mp_int_t fb_width = FB_GETTER(get_width, 0); + const mp_int_t row_stride = FB_GETTER(get_row_stride, fb_width * self->source_bits_per_pixel / 8); + const mp_int_t first_pixel_offset = FB_GETTER(get_first_pixel_offset, 0); + + if (shift_x < 0) { + shift_x = 0; + } + if (shift_y < 0) { + shift_y = 0; + } + // Cap the start and the end. Cap the rows first: enough rows must fit + // above the frame for it to fit at all. + const mp_int_t max_end = (mp_int_t)bufinfo.len - first_pixel_offset - (mp_int_t)frame_bytes; + if (shift_y > max_end / row_stride) { + shift_y = max_end / row_stride; + } + if (shift_y < 0) { + shift_y = 0; + } + const mp_int_t max_x = (max_end - shift_y * row_stride) / bytes_per_pixel; + if (shift_x > max_x) { + shift_x = max_x; + } + if (shift_x < 0) { + shift_x = 0; + } + + videocore_destination_t dest = { + .base = bufinfo.buf + first_pixel_offset + shift_y * row_stride + shift_x * bytes_per_pixel, + .stride = row_stride, + .row_bytes = row_bytes, + .height = self->source_height, + }; + // The result is ignored here: the source rejects a destination by + // returning false (e.g. a mid-row shift isn't DMA-aligned) and the + // stream continues where it was. on_frame points at the old destination + // until then, which keeps the last-known-good frame. + if (videosample_set_destination(self->source, &dest, videocore_videobuffer_on_frame, self)) { + self->stream_base = dest.base; + } +} + +void common_hal_videocore_videobuffer_construct(videocore_videobuffer_obj_t *self, mp_obj_t framebuffer_or_display) { + mp_obj_t framebuffer = framebuffer_or_display; + #if CIRCUITPY_FRAMEBUFFERIO + // Allow passing the display wrapping the framebuffer too. + mp_obj_t native_display = mp_obj_cast_to_native_base(framebuffer_or_display, &framebufferio_framebufferdisplay_type); + if (native_display != MP_OBJ_NULL) { + framebufferio_framebufferdisplay_obj_t *display = MP_OBJ_TO_PTR(native_display); + framebuffer = display->framebuffer; + } + #endif + self->framebuffer = framebuffer; + self->framebuffer_protocol = mp_proto_get(MP_QSTR_protocol_framebuffer, framebuffer); + if (self->framebuffer_protocol != NULL) { + if (self->framebuffer_protocol->get_bufinfo == NULL || + self->framebuffer_protocol->get_width == NULL || + self->framebuffer_protocol->get_height == NULL) { + mp_raise_ValueError(MP_ERROR_TEXT("framebuffer missing required functions")); + } + } else if (!mp_get_buffer(framebuffer, &self->bufinfo, MP_BUFFER_WRITE)) { + // Not a framebuffer and not a writable buffer such as a bytearray or + // memoryview. + mp_raise_TypeError_varg(MP_ERROR_TEXT("expected FrameBuffer or %q"), MP_QSTR_bytearray); + } + + self->source = mp_const_none; + self->stream_base = NULL; + self->x = 0; + self->y = 0; + self->frame_ready = false; + self->in_group = false; + self->hidden = false; + self->stale_size = false; + displayio_area_t empty = {}; + self->current_area = empty; + self->ephemeral_dirty_area = empty; + self->absolute_transform = null_transform; + self->playing = false; +} + +void common_hal_videocore_videobuffer_play(videocore_videobuffer_obj_t *self, mp_obj_t source) { + if (self->playing) { + mp_raise_RuntimeError(MP_ERROR_TEXT("already playing")); + } + const videosample_p_t *protocol = videosample_check(source); + + // Prepare and start the source first: it may reconfigure itself (e.g. a + // sensor picks its mode), which fixes what get_* report from here on. + videosample_play(source); + + mp_int_t bits_per_pixel = protocol->get_bits_per_pixel(source); + if (bits_per_pixel % 8 != 0) { + mp_raise_ValueError(MP_ERROR_TEXT("video format must be byte aligned")); + } + mp_int_t width = protocol->get_width(source); + mp_int_t height = protocol->get_height(source); + size_t frame_bytes = (size_t)((width * height * bits_per_pixel) / 8); + + if (self->framebuffer_protocol != NULL) { + mp_int_t color_depth = FB_GETTER(get_color_depth, 16); + if (color_depth % 8 != 0 || bits_per_pixel != color_depth) { + mp_raise_ValueError(MP_ERROR_TEXT("video format doesn't match framebuffer")); + } + } + + self->source = source; + self->source_width = width; + self->source_height = height; + self->source_bits_per_pixel = bits_per_pixel; + self->source_frame_bytes = frame_bytes; + + if (self->framebuffer_protocol == NULL && (mp_int_t)self->bufinfo.len < (mp_int_t)frame_bytes) { + mp_raise_ValueError(MP_ERROR_TEXT("Buffer too small")); + } + + videocore_destination_t dest; + videocore_videobuffer_compute_destination(self, &dest); + + // Tell the source where to put its pixels. The source streams from now + // on; on_frame tells us when a new frame has landed. It is the + // displayio group tree (and this object's Python references) that keep + // the VideoBuffer alive; a GC'd buffer's finaliser stops the stream. + if (!videosample_set_destination(source, &dest, videocore_videobuffer_on_frame, self)) { + mp_raise_ValueError(MP_ERROR_TEXT("video source rejected the destination")); + } + self->stream_base = dest.base; + self->playing = true; +} + +void common_hal_videocore_videobuffer_set_position(videocore_videobuffer_obj_t *self, mp_int_t x, mp_int_t y) { + if (self->framebuffer_protocol != NULL && x != 0) { + // The source DMAs whole, contiguous frames with no stride, so every + // frame row must land on a framebuffer row: the video's x must be 0. + mp_raise_ValueError(MP_ERROR_TEXT("video in a framebuffer must be at x == 0")); + } + self->ephemeral_dirty_area = self->current_area; + self->x = x; + self->y = y; + videocore_videobuffer_update_area(self); + if (!self->playing || self->framebuffer_protocol == NULL) { + // Raw buffer targets are rendered by displayio; their capture + // destination doesn't move with the position. + return; + } + // The source's frame size was fixed when play() was called. Reject + // sources that changed it (e.g. a sensor mode change) rather than + // blitting at stale geometry. + const videosample_p_t *protocol = videosample_check(self->source); + mp_int_t width = protocol->get_width(self->source); + mp_int_t height = protocol->get_height(self->source); + mp_int_t bits_per_pixel = protocol->get_bits_per_pixel(self->source); + if (width != self->source_width || height != self->source_height || + bits_per_pixel != self->source_bits_per_pixel) { + mp_raise_ValueError(MP_ERROR_TEXT("video size changed since play; stop and play again")); + } + // Move the destination between frames; the source picks up the change on + // its next frame. The result may be ignored here (the source rejects a + // destination by returning false; the stream continues where it was). + videocore_destination_t dest; + videocore_videobuffer_compute_destination(self, &dest); + (void)videosample_set_destination(self->source, &dest, videocore_videobuffer_on_frame, self); + self->stream_base = dest.base; +} + +static void videocore_videobuffer_clear(videocore_videobuffer_obj_t *self) { + self->playing = false; + self->frame_ready = false; + self->stream_base = NULL; + self->source = mp_const_none; +} + +void common_hal_videocore_videobuffer_stop(videocore_videobuffer_obj_t *self) { + // Stop the source; the camera also stops the sensor so its settings can + // be changed. That may raise: surface it, after cleaning up, since the + // stream did stop. + if (self->source != mp_const_none) { + nlr_buf_t nlr; + if (nlr_push(&nlr) == 0) { + videosample_stop(self->source); + nlr_pop(); + } else { + videocore_videobuffer_clear(self); + nlr_raise(nlr.ret_val); + } + } + videocore_videobuffer_clear(self); +} + +// Stop without raising: runs from background contexts (reload). A failed +// stop is best-effort; the source's own finaliser stops it as well. +void videocore_videobuffer_stop_quiet(videocore_videobuffer_obj_t *self) { + nlr_buf_t nlr; + if (nlr_push(&nlr) == 0) { + if (self->source != mp_const_none) { + videosample_stop(self->source); + } + nlr_pop(); + } + videocore_videobuffer_clear(self); +} + +bool common_hal_videocore_videobuffer_get_playing(videocore_videobuffer_obj_t *self) { + return self->playing; +} + +mp_obj_t common_hal_videocore_videobuffer_get_source(videocore_videobuffer_obj_t *self) { + return self->source; +} + +// Draw protocol implementation: a VideoBuffer holding a plain buffer gets +// bulk copied into the display by displayio's rendering; a framebuffer +// target's pixels are streamed into its framebuffer directly, so displayio's +// refresh only claims the video's area and pushes the dirty rows to the +// framebuffer (the fill is a no-op there). + +static mp_obj_t videocore_videobuffer_draw_protocol_self(mp_obj_t protocol_container) { + return protocol_container; +} + +static bool videocore_videobuffer_draw_fill_area(mp_obj_t self_in, const _displayio_colorspace_t *colorspace, const displayio_area_t *area, uint32_t *mask, uint32_t *buffer) { + videocore_videobuffer_obj_t *self = MP_OBJ_TO_PTR(self_in); + if (!self->playing || self->stream_base == NULL || self->hidden) { + return false; + } + + displayio_area_t overlap; + if (!displayio_area_compute_overlap(area, &self->current_area, &overlap)) { + return false; + } + const bool full_coverage = displayio_area_equal(area, &overlap); + + // Opaque: claim the pixels. + const uint16_t span_px = displayio_area_width(area); + for (int16_t y = overlap.y1; y < overlap.y2; y++) { + const size_t px = (size_t)(y - area->y1) * span_px + (overlap.x1 - area->x1); + for (size_t i = 0; i < displayio_area_width(&overlap); i++) { + mask[(px + i) / 32] |= 1u << ((px + i) % 32); + } + } + + if (self->framebuffer_protocol != NULL) { + // Framebuffer target: the source streams the pixels straight into + // its framebuffer, so the fill itself is a no-op. Claiming the area + // is what adds its rows to the display's refresh. + return full_coverage; + } + + // Raw buffer target: bulk copy, no color conversion: the display depth + // must match the video's pixel format byte for byte. + if (colorspace->depth != self->source_bits_per_pixel || colorspace->depth != 16) { + return false; + } + + const size_t bytes_per_pixel = self->source_bits_per_pixel / 8; + const size_t row_bytes = (size_t)self->source_width * bytes_per_pixel; + const uint8_t *frame = (const uint8_t *)self->stream_base; + + for (int16_t y = overlap.y1; y < overlap.y2; y++) { + const size_t px = (size_t)(y - area->y1) * span_px + (overlap.x1 - area->x1); + const int16_t fy = y - self->current_area.y1; + memcpy( + (uint16_t *)buffer + px, + frame + (size_t)fy * row_bytes + (size_t)(overlap.x1 - self->current_area.x1) * bytes_per_pixel, + (size_t)displayio_area_width(&overlap) * bytes_per_pixel); + } + return full_coverage; +} + +static bool videocore_videobuffer_draw_get_dirty_area(mp_obj_t self_in, displayio_area_t *out_area) { + videocore_videobuffer_obj_t *self = MP_OBJ_TO_PTR(self_in); + out_area->x1 = out_area->x2; + displayio_area_union(&self->ephemeral_dirty_area, &self->current_area, out_area); + return true; +} + +static void videocore_videobuffer_draw_update_transform(mp_obj_t self_in, displayio_buffer_transform_t *group_transform) { + videocore_videobuffer_obj_t *self = MP_OBJ_TO_PTR(self_in); + if (group_transform == NULL) { + self->absolute_transform = null_transform; + } else { + // Validate the transform: video frames are bulk copied (raw targets) + // or DMA'd contiguously (framebuffer targets), so only translation + // moves. + if (group_transform->scale != 0 && + (group_transform->scale != 1 || + group_transform->mirror_x || group_transform->mirror_y || + group_transform->transpose_xy || + group_transform->dx != 1 || group_transform->dy != 1)) { + mp_raise_ValueError(MP_ERROR_TEXT("VideoBuffer does not support transforms")); + } + self->absolute_transform = *group_transform; + } + videocore_videobuffer_update_area(self); + + // Framebuffer targets stream straight into the buffer: move the + // destination along with the shift. Best effort, since this can't raise: + // the shift may land the start mid-row, which won't look right but is + // obvious; the frame is capped to stay completely within the buffer. + if (self->framebuffer_protocol != NULL) { + videocore_videobuffer_move_dest( + self, + self->x + self->absolute_transform.x, + self->y + self->absolute_transform.y); + } +} + +static void videocore_videobuffer_draw_finish_refresh(mp_obj_t self_in) { + videocore_videobuffer_obj_t *self = MP_OBJ_TO_PTR(self_in); + // The frame (or move) rendered; stop flagging it. + self->frame_ready = false; + displayio_area_t empty = {}; + self->ephemeral_dirty_area = empty; +} + +static displayio_area_t *videocore_videobuffer_draw_get_refresh_areas(mp_obj_t self_in, displayio_area_t *tail) { + videocore_videobuffer_obj_t *self = MP_OBJ_TO_PTR(self_in); + if (!self->playing || self->hidden || self->stale_size) { + return tail; + } + if (!displayio_area_empty(&self->ephemeral_dirty_area)) { + // The video moved: repaint the vacated area with the lower layers. + self->ephemeral_dirty_area.next = tail; + tail = &self->ephemeral_dirty_area; + } + if (self->frame_ready) { + // New frame: bulk copy it in. + self->current_area.next = tail; + tail = &self->current_area; + } + return tail; +} + +static void videocore_videobuffer_draw_set_dirty(mp_obj_t self_in) { + (void)self_in; +} + +static void videocore_videobuffer_draw_set_hidden(mp_obj_t self_in, bool hidden) { + videocore_videobuffer_obj_t *self = MP_OBJ_TO_PTR(self_in); + if (self->hidden == hidden) { + return; + } + self->hidden = hidden; + // Repaint what the video showed or vacated. + self->ephemeral_dirty_area = self->current_area; + if (!hidden && self->playing) { + // The source is free to reconfigure while hidden. Validate its + // sizing again but quietly: if it no longer matches, don't raise; + // simply stop showing the stream until stop and play again. + const videosample_p_t *protocol = mp_proto_get(MP_QSTR_protocol_videosample, self->source); + if (protocol != NULL && + (protocol->get_width(self->source) != self->source_width || + protocol->get_height(self->source) != self->source_height || + protocol->get_bits_per_pixel(self->source) != self->source_bits_per_pixel)) { + self->stale_size = true; + } + } +} + +static bool videocore_videobuffer_draw_set_in_group(mp_obj_t self_in, bool in_group) { + videocore_videobuffer_obj_t *self = MP_OBJ_TO_PTR(self_in); + bool was_in_group = self->in_group; + self->in_group = in_group; + return was_in_group; +} + +static vectorio_draw_protocol_impl_t videocore_videobuffer_draw_protocol_impl = { + .draw_fill_area = videocore_videobuffer_draw_fill_area, + .draw_get_dirty_area = videocore_videobuffer_draw_get_dirty_area, + .draw_update_transform = videocore_videobuffer_draw_update_transform, + .draw_finish_refresh = videocore_videobuffer_draw_finish_refresh, + .draw_get_refresh_areas = videocore_videobuffer_draw_get_refresh_areas, + .draw_set_dirty = videocore_videobuffer_draw_set_dirty, + .draw_set_hidden = videocore_videobuffer_draw_set_hidden, + .draw_set_in_group = videocore_videobuffer_draw_set_in_group, +}; + +const vectorio_draw_protocol_t videocore_videobuffer_draw_protocol = { + MP_PROTO_IMPLEMENT(MP_QSTR_protocol_draw) + .draw_get_protocol_self = videocore_videobuffer_draw_protocol_self, + .draw_protocol_impl = &videocore_videobuffer_draw_protocol_impl, +}; + +// Root group validation: a VideoBuffer that streams straight into a +// framebuffer bypasses displayio rendering, so it may only be shown on a +// display that renders that same framebuffer. Plain buffer targets are drawn +// by displayio and are fine anywhere. Walks the group tree and raises when a +// VideoBuffer doesn't match the display it would render on. +static void videocore_validate_member(mp_obj_t member, mp_obj_t display) { + displayio_group_t *group = mp_obj_cast_to_native_base(member, &displayio_group_type); + if (group != MP_OBJ_NULL) { + for (size_t i = 0; i < group->members->len; i++) { + videocore_validate_member(group->members->items[i], display); + } + return; + } + const vectorio_draw_protocol_t *draw_protocol = mp_proto_get(MP_QSTR_protocol_draw, member); + if (draw_protocol == NULL) { + return; + } + mp_obj_t layer = draw_protocol->draw_get_protocol_self(member); + if (mp_obj_get_type(layer) != &videocore_videobuffer_type) { + return; + } + videocore_videobuffer_obj_t *self = MP_OBJ_TO_PTR(layer); + if (self->framebuffer_protocol == NULL || display == self->framebuffer) { + return; + } + #if CIRCUITPY_FRAMEBUFFERIO + // The display may wrap the framebuffer the video plays to (e.g. a + // framebufferio.FramebufferDisplay around the same object). + mp_obj_t native_display = mp_obj_cast_to_native_base(display, &framebufferio_framebufferdisplay_type); + if (native_display != MP_OBJ_NULL) { + framebufferio_framebufferdisplay_obj_t *fd = MP_OBJ_TO_PTR(native_display); + if (fd->framebuffer == self->framebuffer) { + return; + } + } + #endif + mp_raise_ValueError(MP_ERROR_TEXT("VideoBuffer is playing into a different framebuffer")); +} + +void videocore_validate_root_group(mp_obj_t root_group, mp_obj_t display) { + videocore_validate_member(root_group, display); +} diff --git a/shared-module/videocore/VideoBuffer.h b/shared-module/videocore/VideoBuffer.h new file mode 100644 index 00000000000..0837b34fdfb --- /dev/null +++ b/shared-module/videocore/VideoBuffer.h @@ -0,0 +1,73 @@ +// This file is part of the CircuitPython project: https://circuitpython.org +// +// SPDX-FileCopyrightText: Copyright (c) 2026 Scott Shawcroft for Adafruit Industries +// +// SPDX-License-Identifier: MIT + +#pragma once + +#include "py/obj.h" + +#include "shared-bindings/vectorio/__init__.h" // draw protocol +#include "shared-module/displayio/area.h" +#include "shared-module/framebufferio/FramebufferDisplay.h" // framebuffer protocol +#include "shared-module/videocore/__init__.h" + +typedef struct { + mp_obj_base_t base; + // Object frames are streamed into: an object implementing the framebuffer + // protocol, such as mipidsi.Display, or a writable buffer such as a + // bytearray or memoryview. + mp_obj_t framebuffer; + const framebuffer_p_t *framebuffer_protocol; + // Target buffer info in raw buffer mode (framebuffer_protocol == NULL). + mp_buffer_info_t bufinfo; + // The video source being played, or None. + mp_obj_t source; + // Destination position of the video within the target. + mp_int_t x; + mp_int_t y; + mp_int_t source_width; + mp_int_t source_height; + mp_int_t source_bits_per_pixel; + size_t source_frame_bytes; + // Where the source streams frames into: the requested destination, or a + // later, hardware-aligned position inside it. "Later in the buffer" only + // happens for plain-buffer targets; set by play() and the x/y + // properties. + const void *stream_base; + // Set from interrupt context when the source lands a frame in the + // destination; cleared by the refresh pass once the new frame has been + // rendered into the display. + volatile bool frame_ready; + // displayio draw protocol state: the rect currently rendered (in screen + // coordinates) and the rect that changed since the last refresh. + displayio_area_t current_area; + displayio_area_t ephemeral_dirty_area; + displayio_buffer_transform_t absolute_transform; + bool in_group; + // Set by the group when hidden; while hidden the video doesn't show and + // the source is free to reconfigure (it is revalidated on unhide). + bool hidden; + // True when the source's sizing no longer matches what play() captured: + // the video won't show until stop and play again. + bool stale_size; + bool playing; +} videocore_videobuffer_obj_t; + +void common_hal_videocore_videobuffer_construct(videocore_videobuffer_obj_t *self, mp_obj_t framebuffer_or_display); +void common_hal_videocore_videobuffer_play(videocore_videobuffer_obj_t *self, mp_obj_t source); +void common_hal_videocore_videobuffer_set_position(videocore_videobuffer_obj_t *self, mp_int_t x, mp_int_t y); +void common_hal_videocore_videobuffer_stop(videocore_videobuffer_obj_t *self); +// Stop without raising (from the finaliser). A failed stop is best-effort; +// the source's own finaliser stops it as well. +void videocore_videobuffer_stop_quiet(videocore_videobuffer_obj_t *self); +bool common_hal_videocore_videobuffer_get_playing(videocore_videobuffer_obj_t *self); +mp_obj_t common_hal_videocore_videobuffer_get_source(videocore_videobuffer_obj_t *self); + +// The draw protocol implementation that lets a VideoBuffer be added to a +// displayio Group: plain-buffer targets get bulk copied into the framebuffer +// by displayio's rendering; framebuffer targets claim their area (a no-op +// fill: the source streams their pixels straight into the framebuffer) so +// displayio's refresh pushes the video's rows to the framebuffer. +extern const vectorio_draw_protocol_t videocore_videobuffer_draw_protocol; diff --git a/shared-module/videocore/__init__.c b/shared-module/videocore/__init__.c new file mode 100644 index 00000000000..ac3b7f25d3a --- /dev/null +++ b/shared-module/videocore/__init__.c @@ -0,0 +1,48 @@ +// This file is part of the CircuitPython project: https://circuitpython.org +// +// SPDX-FileCopyrightText: Copyright (c) 2026 Scott Shawcroft for Adafruit Industries +// +// SPDX-License-Identifier: MIT + +#include "shared-module/videocore/__init__.h" +#include "shared-module/videocore/VideoBuffer.h" + +// Playback, presentation and garbage collection are tied to the displayio +// group the VideoBuffer is in: displayio's refresh drives frame presentation +// through the draw protocol, and the group (or a Python reference) keeps the +// object alive; its finaliser stops the stream. + +const videosample_p_t *videosample_check(mp_obj_t obj) { + return (const videosample_p_t *)mp_proto_get_or_throw(MP_QSTR_protocol_videosample, obj); +} + +void videosample_play(mp_obj_t source) { + const videosample_p_t *protocol = videosample_check(source); + if (protocol->play != NULL) { + protocol->play(source); + } +} + +void videosample_stop(mp_obj_t source) { + const videosample_p_t *protocol = videosample_check(source); + if (protocol->stop != NULL) { + protocol->stop(source); + } +} + +bool videosample_set_destination(mp_obj_t source, const videocore_destination_t *dest, + videocore_on_frame_fun on_frame, void *ctx) { + return videosample_check(source)->set_destination(source, dest, on_frame, ctx); +} + +mp_int_t videosample_get_width(mp_obj_t source) { + return videosample_check(source)->get_width(source); +} + +mp_int_t videosample_get_height(mp_obj_t source) { + return videosample_check(source)->get_height(source); +} + +mp_int_t videosample_get_bits_per_pixel(mp_obj_t source) { + return videosample_check(source)->get_bits_per_pixel(source); +} diff --git a/shared-module/videocore/__init__.h b/shared-module/videocore/__init__.h new file mode 100644 index 00000000000..df679a91822 --- /dev/null +++ b/shared-module/videocore/__init__.h @@ -0,0 +1,96 @@ +// This file is part of the CircuitPython project: https://circuitpython.org +// +// SPDX-FileCopyrightText: Copyright (c) 2026 Scott Shawcroft for Adafruit Industries +// +// SPDX-License-Identifier: MIT + +#pragma once + +#include +#include +#include + +#include "py/obj.h" +#include "py/proto.h" + +// Destination for a video stream: where to put pixels, as a pointer to the +// first pixel and the destination row stride. row_bytes/height describe the +// source frame; the source assumes the destination holds the whole frame +// (the ESP CSI DMA is contiguous-only), so the rect is validated before +// streaming starts and when the position changes. +typedef struct { + // First pixel of the destination. + uint8_t *base; + // Destination row stride, in bytes. + size_t stride; + // Source frame row size, in bytes. + size_t row_bytes; + // Source frame height, in rows. + size_t height; +} videocore_destination_t; + +// Called (in interrupt context) when a frame has landed in the buffer +// set_destination returned. +typedef void (*videocore_on_frame_fun)(void *ctx); + +// Protocol implemented by video streamers such as mipicsi.Camera. The streams +// continuously; the target sets (and can change) where the pixels go by +// calling set_destination() at any time, including between frames. +// +// This is the video equivalent of the audiosample protocol in the audiocore +// module, inverted: instead of the target pulling frames, the source writes +// where it is told to. +typedef struct _videosample_p_t { + MP_PROTOCOL_HEAD // MP_QSTR_protocol_videosample + + // Create whatever streaming hardware the source needs and start it (e.g. + // start a sensor streaming). Called once before the first + // set_destination of a stream, from Python (VideoBuffer.play), so it may + // raise. Raises when the source is already streaming for someone else. + // May be NULL for sources that need no preparation. + void (*play)(mp_obj_t self); + + // Direct future frames into dest. The destination may be changed + // between frames; there is no NULL/stop value: stopping is stop()'s + // job. Never raises: returns whether the stream accepted the + // destination (the caller may ignore the result). on_frame(ctx) is + // called from interrupt context whenever a frame has landed in + // dest.base (the source never relocates frames elsewhere). + bool (*set_destination)(mp_obj_t self, const videocore_destination_t *dest, + videocore_on_frame_fun on_frame, void *ctx); + + // Stop streaming and release the streaming hardware (e.g. stop the + // sensor so its settings can be changed). Called once per stream, from + // Python (VideoBuffer.stop), so it may raise; exceptions there are + // clearer than implicit stops. Only called when play() succeeded. + void (*stop)(mp_obj_t self); + + // Width of the frames the source streams, in pixels. + mp_int_t (*get_width)(mp_obj_t self); + // Height of the frames the source streams, in lines. + mp_int_t (*get_height)(mp_obj_t self); + // Bits per pixel of the frames the source streams. + mp_int_t (*get_bits_per_pixel)(mp_obj_t self); +} videosample_p_t; + +// Protocol function helpers. videosample_play/stop are no-ops for sources +// whose protocol doesn't implement them. +void videosample_play(mp_obj_t source); +void videosample_stop(mp_obj_t source); + +// Return the videosample protocol of obj or raise TypeError. +const videosample_p_t *videosample_check(mp_obj_t obj); + +// Dispatch helpers mirroring the audiosample_* functions. videosample_play +// is a no-op for sources whose protocol has no play. +bool videosample_set_destination(mp_obj_t source, const videocore_destination_t *dest, + videocore_on_frame_fun on_frame, void *ctx); +mp_int_t videosample_get_width(mp_obj_t source); +mp_int_t videosample_get_height(mp_obj_t source); +mp_int_t videosample_get_bits_per_pixel(mp_obj_t source); + +// Raise when the group tree contains a VideoBuffer that streams into a +// framebuffer other than the one this display renders. Called when a group +// is made a display's root group. Implemented in shared-module/videocore/ +// VideoBuffer.c. +void videocore_validate_root_group(mp_obj_t root_group, mp_obj_t display); From 77ef329131430236081f3d53ddde73b8503f1573 Mon Sep 17 00:00:00 2001 From: Scott Shawcroft Date: Fri, 16 Oct 2026 13:41:49 -0700 Subject: [PATCH 2/2] Rename VideoSample to VideoSource --- ports/espressif/common-hal/mipicsi/Camera.c | 16 ++++++------ shared-bindings/mipicsi/Camera.c | 16 ++++++------ shared-bindings/mipicsi/Camera.h | 12 ++++----- shared-bindings/videocore/VideoBuffer.c | 8 +++--- shared-bindings/videocore/__init__.c | 2 +- shared-module/videocore/VideoBuffer.c | 18 ++++++------- shared-module/videocore/__init__.c | 28 ++++++++++----------- shared-module/videocore/__init__.h | 26 +++++++++---------- 8 files changed, 63 insertions(+), 63 deletions(-) diff --git a/ports/espressif/common-hal/mipicsi/Camera.c b/ports/espressif/common-hal/mipicsi/Camera.c index 368061e53e7..2a68e2d660c 100644 --- a/ports/espressif/common-hal/mipicsi/Camera.c +++ b/ports/espressif/common-hal/mipicsi/Camera.c @@ -155,16 +155,16 @@ void common_hal_mipicsi_camera_construct( self->phy_ldo = NULL; self->isp_proc = NULL; self->spinlock = (portMUX_TYPE)portMUX_INITIALIZER_UNLOCKED; - // No receiver hardware yet: it is created in the videosample play step, + // No receiver hardware yet: it is created in the videosource play step, // where the sensor's current mode is known. } -// Videosample protocol play step: create the LDO, ISP and CSI controller for +// Videosource protocol play step: create the LDO, ISP and CSI controller for // the given (freshly read) sensor parameters and start the sensor streaming. // Raises when the camera is already streaming for someone else. Not a user -// facing method: it is driven by the videosample protocol (from +// facing method: it is driven by the videosource protocol (from // VideoBuffer.play), so it may raise. -void common_hal_mipicsi_camera_videosample_play( +void common_hal_mipicsi_camera_videosource_play( mipicsi_camera_obj_t *self, const mipicsi_sensor_params_t *params) { @@ -361,11 +361,11 @@ void common_hal_mipicsi_camera_deinit(mipicsi_camera_obj_t *self) { mipicsi_camera_teardown(self); self->sensor = mp_const_none; } -// Videosample protocol stop step: tear down the receiver hardware and stop +// Videosource protocol stop step: tear down the receiver hardware and stop // the sensor so its settings can be changed. Not a user facing method: it is -// driven by the videosample protocol (from VideoBuffer.stop), so it may +// driven by the videosource protocol (from VideoBuffer.stop), so it may // raise; exceptions from the sensor's stop() propagate. -void common_hal_mipicsi_camera_videosample_stop(mipicsi_camera_obj_t *self) { +void common_hal_mipicsi_camera_videosource_stop(mipicsi_camera_obj_t *self) { if (self->handle == NULL) { return; // Not playing. } @@ -396,7 +396,7 @@ bool common_hal_mipicsi_camera_set_destination( const size_t row_bytes = ((size_t)self->width * mipicsi_bits_per_pixel_for(self->output_format) + 7) / 8; const size_t frame_bytes = (size_t)self->height * row_bytes; - // There is no stop value on the destination: stopping is the videosample + // There is no stop value on the destination: stopping is the videosource // stop step's job. The protocol never passes NULL, // so this guard is only about internal misuse. assert(base != NULL); diff --git a/shared-bindings/mipicsi/Camera.c b/shared-bindings/mipicsi/Camera.c index c899dda2d71..7e711ca1ed7 100644 --- a/shared-bindings/mipicsi/Camera.c +++ b/shared-bindings/mipicsi/Camera.c @@ -48,7 +48,7 @@ //| their own finaliser, and have a new Camera made when the mode //| changes. //| -//| Camera objects implement the internal videosample protocol, so a +//| Camera objects implement the internal videosource protocol, so a //| `videocore.VideoBuffer` can stream their frames into a framebuffer. //| """ //| @@ -122,7 +122,7 @@ static mp_obj_t mipicsi_camera_make_new(const mp_obj_type_t *type, size_t n_args common_hal_mipicsi_camera_construct(self, output_format); // Keep the sensor alive while receiving from it. It is only started // streaming when a videocore.VideoBuffer starts playing (see the - // videosample play step): until then nothing captures or transmits. + // videosource play step): until then nothing captures or transmits. common_hal_mipicsi_camera_set_sensor(self, sensor); return MP_OBJ_FROM_PTR(self); } @@ -159,7 +159,7 @@ static const mp_rom_map_elem_t mipicsi_camera_locals_dict_table[] = { static MP_DEFINE_CONST_DICT(mipicsi_camera_locals_dict, mipicsi_camera_locals_dict_table); #if CIRCUITPY_VIDEOCORE -// Implementation of the videocore videosample protocol so a VideoBuffer can +// Implementation of the videocore videosource protocol so a VideoBuffer can // tell the Camera where to put frames. static void mipicsi_camera_play(mp_obj_t self_in) { mipicsi_camera_obj_t *self = MP_OBJ_TO_PTR(mp_obj_cast_to_native_base(self_in, &mipicsi_camera_type)); @@ -174,12 +174,12 @@ static void mipicsi_camera_play(mp_obj_t self_in) { // sensor refuses it itself). mipicsi_sensor_params_t params; self->pixel_format = mipicsi_camera_load_sensor_params(self->sensor, ¶ms); - common_hal_mipicsi_camera_videosample_play(self, ¶ms); + common_hal_mipicsi_camera_videosource_play(self, ¶ms); } static void mipicsi_camera_stop(mp_obj_t self_in) { mipicsi_camera_obj_t *self = MP_OBJ_TO_PTR(mp_obj_cast_to_native_base(self_in, &mipicsi_camera_type)); - common_hal_mipicsi_camera_videosample_stop(self); + common_hal_mipicsi_camera_videosource_stop(self); } static bool mipicsi_camera_set_destination(mp_obj_t self_in, const videocore_destination_t *dest, @@ -202,8 +202,8 @@ static mp_int_t mipicsi_camera_get_bits_per_pixel(mp_obj_t self_in) { return common_hal_mipicsi_camera_get_bits_per_pixel(mp_obj_cast_to_native_base(self_in, &mipicsi_camera_type)); } -static const videosample_p_t mipicsi_camera_videosample_protocol = { - MP_PROTO_IMPLEMENT(MP_QSTR_protocol_videosample) +static const videosource_p_t mipicsi_camera_videosource_protocol = { + MP_PROTO_IMPLEMENT(MP_QSTR_protocol_videosource) .play = mipicsi_camera_play, .stop = mipicsi_camera_stop, .set_destination = mipicsi_camera_set_destination, @@ -218,7 +218,7 @@ MP_DEFINE_CONST_OBJ_TYPE( MP_TYPE_FLAG_HAS_SPECIAL_ACCESSORS, make_new, mipicsi_camera_make_new, locals_dict, &mipicsi_camera_locals_dict, - protocol, &mipicsi_camera_videosample_protocol + protocol, &mipicsi_camera_videosource_protocol ); #else MP_DEFINE_CONST_OBJ_TYPE( diff --git a/shared-bindings/mipicsi/Camera.h b/shared-bindings/mipicsi/Camera.h index d39e748936d..4df6c52b083 100644 --- a/shared-bindings/mipicsi/Camera.h +++ b/shared-bindings/mipicsi/Camera.h @@ -24,18 +24,18 @@ void common_hal_mipicsi_camera_construct( // itself (its finaliser does), never the receiver. void common_hal_mipicsi_camera_set_sensor(mipicsi_camera_obj_t *self, mp_obj_t sensor); -// Videosample protocol play step: create the receiver hardware for the +// Videosource protocol play step: create the receiver hardware for the // sensor's current mode and start the sensor streaming. Not a user facing -// method: it is driven by the videosample protocol (from VideoBuffer.play), +// method: it is driven by the videosource protocol (from VideoBuffer.play), // so it may raise. -void common_hal_mipicsi_camera_videosample_play( +void common_hal_mipicsi_camera_videosource_play( mipicsi_camera_obj_t *self, const mipicsi_sensor_params_t *params); -// Videosample protocol stop step: stop the stream and the sensor. Not a user -// facing method: it is driven by the videosample protocol (from +// Videosource protocol stop step: stop the stream and the sensor. Not a user +// facing method: it is driven by the videosource protocol (from // VideoBuffer.stop), so it may raise. -void common_hal_mipicsi_camera_videosample_stop(mipicsi_camera_obj_t *self); +void common_hal_mipicsi_camera_videosource_stop(mipicsi_camera_obj_t *self); void common_hal_mipicsi_camera_deinit(mipicsi_camera_obj_t *self); bool common_hal_mipicsi_camera_deinited(mipicsi_camera_obj_t *self); diff --git a/shared-bindings/videocore/VideoBuffer.c b/shared-bindings/videocore/VideoBuffer.c index eb1a415f9bd..a6dcbfe87a4 100644 --- a/shared-bindings/videocore/VideoBuffer.c +++ b/shared-bindings/videocore/VideoBuffer.c @@ -19,7 +19,7 @@ //| but may have size and position limitations. //| //| A VideoBuffer connects a video source (an object implementing the -//| videosample protocol, such as `mipicsi.Camera`) to a target: either a +//| videosource protocol, such as `mipicsi.Camera`) to a target: either a //| framebuffer (an object implementing the framebuffer protocol, such as //| `mipidsi.Display`) or a writable buffer such as a ``bytearray`` or //| ``memoryview``. It is the video counterpart of `audioio.AudioOut`. @@ -75,7 +75,7 @@ static mp_obj_t videocore_videobuffer_make_new(const mp_obj_type_t *type, size_t return MP_OBJ_FROM_PTR(self); } -//| def play(self, source: circuitpython_typing.VideoSample) -> None: +//| def play(self, source: circuitpython_typing.VideoSource) -> None: //| """Begin streaming the given video source into the target. //| //| The source streams continuously, writing each new frame where the @@ -90,7 +90,7 @@ static mp_obj_t videocore_videobuffer_make_new(const mp_obj_type_t *type, size_t //| play. Sources that change them require ``stop`` and a new //| ``play``. //| -//| :param ~circuitpython_typing.VideoSample source: The video source to +//| :param ~circuitpython_typing.VideoSource source: The video source to //| play, such as `mipicsi.Camera` //| """ //| @@ -123,7 +123,7 @@ static MP_DEFINE_CONST_FUN_OBJ_1(videocore_videobuffer_get_playing_obj, videocor MP_PROPERTY_GETTER(videocore_videobuffer_playing_obj, (mp_obj_t)&videocore_videobuffer_get_playing_obj); -//| source: Optional[circuitpython_typing.VideoSample] +//| source: Optional[circuitpython_typing.VideoSource] //| """The video source being played, or None when not playing. (read-only)""" //| static mp_obj_t videocore_videobuffer_get_source(mp_obj_t self_in) { diff --git a/shared-bindings/videocore/__init__.c b/shared-bindings/videocore/__init__.c index cc55331b6df..057600deeb4 100644 --- a/shared-bindings/videocore/__init__.c +++ b/shared-bindings/videocore/__init__.c @@ -15,7 +15,7 @@ //| This module provides the video pipeline shared between video sources and //| video outputs. It is the video equivalent of the `audiocore` module. //| -//| Video sources, such as `mipicsi.Camera`, implement the internal videosample +//| Video sources, such as `mipicsi.Camera`, implement the internal videosource //| protocol. A `VideoBuffer` tells a video source where to put its frames in a //| framebuffer, such as `mipidsi.Display`, and can move it as needed. //| diff --git a/shared-module/videocore/VideoBuffer.c b/shared-module/videocore/VideoBuffer.c index c9aca74c197..dafc942d8a1 100644 --- a/shared-module/videocore/VideoBuffer.c +++ b/shared-module/videocore/VideoBuffer.c @@ -157,7 +157,7 @@ static void videocore_videobuffer_move_dest( // returning false (e.g. a mid-row shift isn't DMA-aligned) and the // stream continues where it was. on_frame points at the old destination // until then, which keeps the last-known-good frame. - if (videosample_set_destination(self->source, &dest, videocore_videobuffer_on_frame, self)) { + if (videosource_set_destination(self->source, &dest, videocore_videobuffer_on_frame, self)) { self->stream_base = dest.base; } } @@ -205,11 +205,11 @@ void common_hal_videocore_videobuffer_play(videocore_videobuffer_obj_t *self, mp if (self->playing) { mp_raise_RuntimeError(MP_ERROR_TEXT("already playing")); } - const videosample_p_t *protocol = videosample_check(source); + const videosource_p_t *protocol = videosource_check(source); // Prepare and start the source first: it may reconfigure itself (e.g. a // sensor picks its mode), which fixes what get_* report from here on. - videosample_play(source); + videosource_play(source); mp_int_t bits_per_pixel = protocol->get_bits_per_pixel(source); if (bits_per_pixel % 8 != 0) { @@ -243,7 +243,7 @@ void common_hal_videocore_videobuffer_play(videocore_videobuffer_obj_t *self, mp // on; on_frame tells us when a new frame has landed. It is the // displayio group tree (and this object's Python references) that keep // the VideoBuffer alive; a GC'd buffer's finaliser stops the stream. - if (!videosample_set_destination(source, &dest, videocore_videobuffer_on_frame, self)) { + if (!videosource_set_destination(source, &dest, videocore_videobuffer_on_frame, self)) { mp_raise_ValueError(MP_ERROR_TEXT("video source rejected the destination")); } self->stream_base = dest.base; @@ -268,7 +268,7 @@ void common_hal_videocore_videobuffer_set_position(videocore_videobuffer_obj_t * // The source's frame size was fixed when play() was called. Reject // sources that changed it (e.g. a sensor mode change) rather than // blitting at stale geometry. - const videosample_p_t *protocol = videosample_check(self->source); + const videosource_p_t *protocol = videosource_check(self->source); mp_int_t width = protocol->get_width(self->source); mp_int_t height = protocol->get_height(self->source); mp_int_t bits_per_pixel = protocol->get_bits_per_pixel(self->source); @@ -281,7 +281,7 @@ void common_hal_videocore_videobuffer_set_position(videocore_videobuffer_obj_t * // destination by returning false; the stream continues where it was). videocore_destination_t dest; videocore_videobuffer_compute_destination(self, &dest); - (void)videosample_set_destination(self->source, &dest, videocore_videobuffer_on_frame, self); + (void)videosource_set_destination(self->source, &dest, videocore_videobuffer_on_frame, self); self->stream_base = dest.base; } @@ -299,7 +299,7 @@ void common_hal_videocore_videobuffer_stop(videocore_videobuffer_obj_t *self) { if (self->source != mp_const_none) { nlr_buf_t nlr; if (nlr_push(&nlr) == 0) { - videosample_stop(self->source); + videosource_stop(self->source); nlr_pop(); } else { videocore_videobuffer_clear(self); @@ -315,7 +315,7 @@ void videocore_videobuffer_stop_quiet(videocore_videobuffer_obj_t *self) { nlr_buf_t nlr; if (nlr_push(&nlr) == 0) { if (self->source != mp_const_none) { - videosample_stop(self->source); + videosource_stop(self->source); } nlr_pop(); } @@ -469,7 +469,7 @@ static void videocore_videobuffer_draw_set_hidden(mp_obj_t self_in, bool hidden) // The source is free to reconfigure while hidden. Validate its // sizing again but quietly: if it no longer matches, don't raise; // simply stop showing the stream until stop and play again. - const videosample_p_t *protocol = mp_proto_get(MP_QSTR_protocol_videosample, self->source); + const videosource_p_t *protocol = mp_proto_get(MP_QSTR_protocol_videosource, self->source); if (protocol != NULL && (protocol->get_width(self->source) != self->source_width || protocol->get_height(self->source) != self->source_height || diff --git a/shared-module/videocore/__init__.c b/shared-module/videocore/__init__.c index ac3b7f25d3a..4bf32772cb1 100644 --- a/shared-module/videocore/__init__.c +++ b/shared-module/videocore/__init__.c @@ -12,37 +12,37 @@ // through the draw protocol, and the group (or a Python reference) keeps the // object alive; its finaliser stops the stream. -const videosample_p_t *videosample_check(mp_obj_t obj) { - return (const videosample_p_t *)mp_proto_get_or_throw(MP_QSTR_protocol_videosample, obj); +const videosource_p_t *videosource_check(mp_obj_t obj) { + return (const videosource_p_t *)mp_proto_get_or_throw(MP_QSTR_protocol_videosource, obj); } -void videosample_play(mp_obj_t source) { - const videosample_p_t *protocol = videosample_check(source); +void videosource_play(mp_obj_t source) { + const videosource_p_t *protocol = videosource_check(source); if (protocol->play != NULL) { protocol->play(source); } } -void videosample_stop(mp_obj_t source) { - const videosample_p_t *protocol = videosample_check(source); +void videosource_stop(mp_obj_t source) { + const videosource_p_t *protocol = videosource_check(source); if (protocol->stop != NULL) { protocol->stop(source); } } -bool videosample_set_destination(mp_obj_t source, const videocore_destination_t *dest, +bool videosource_set_destination(mp_obj_t source, const videocore_destination_t *dest, videocore_on_frame_fun on_frame, void *ctx) { - return videosample_check(source)->set_destination(source, dest, on_frame, ctx); + return videosource_check(source)->set_destination(source, dest, on_frame, ctx); } -mp_int_t videosample_get_width(mp_obj_t source) { - return videosample_check(source)->get_width(source); +mp_int_t videosource_get_width(mp_obj_t source) { + return videosource_check(source)->get_width(source); } -mp_int_t videosample_get_height(mp_obj_t source) { - return videosample_check(source)->get_height(source); +mp_int_t videosource_get_height(mp_obj_t source) { + return videosource_check(source)->get_height(source); } -mp_int_t videosample_get_bits_per_pixel(mp_obj_t source) { - return videosample_check(source)->get_bits_per_pixel(source); +mp_int_t videosource_get_bits_per_pixel(mp_obj_t source) { + return videosource_check(source)->get_bits_per_pixel(source); } diff --git a/shared-module/videocore/__init__.h b/shared-module/videocore/__init__.h index df679a91822..3db83d590d1 100644 --- a/shared-module/videocore/__init__.h +++ b/shared-module/videocore/__init__.h @@ -40,8 +40,8 @@ typedef void (*videocore_on_frame_fun)(void *ctx); // This is the video equivalent of the audiosample protocol in the audiocore // module, inverted: instead of the target pulling frames, the source writes // where it is told to. -typedef struct _videosample_p_t { - MP_PROTOCOL_HEAD // MP_QSTR_protocol_videosample +typedef struct _videosource_p_t { + MP_PROTOCOL_HEAD // MP_QSTR_protocol_videosource // Create whatever streaming hardware the source needs and start it (e.g. // start a sensor streaming). Called once before the first @@ -71,23 +71,23 @@ typedef struct _videosample_p_t { mp_int_t (*get_height)(mp_obj_t self); // Bits per pixel of the frames the source streams. mp_int_t (*get_bits_per_pixel)(mp_obj_t self); -} videosample_p_t; +} videosource_p_t; -// Protocol function helpers. videosample_play/stop are no-ops for sources +// Protocol function helpers. videosource_play/stop are no-ops for sources // whose protocol doesn't implement them. -void videosample_play(mp_obj_t source); -void videosample_stop(mp_obj_t source); +void videosource_play(mp_obj_t source); +void videosource_stop(mp_obj_t source); -// Return the videosample protocol of obj or raise TypeError. -const videosample_p_t *videosample_check(mp_obj_t obj); +// Return the videosource protocol of obj or raise TypeError. +const videosource_p_t *videosource_check(mp_obj_t obj); -// Dispatch helpers mirroring the audiosample_* functions. videosample_play +// Dispatch helpers mirroring the audiosample_* functions. videosource_play // is a no-op for sources whose protocol has no play. -bool videosample_set_destination(mp_obj_t source, const videocore_destination_t *dest, +bool videosource_set_destination(mp_obj_t source, const videocore_destination_t *dest, videocore_on_frame_fun on_frame, void *ctx); -mp_int_t videosample_get_width(mp_obj_t source); -mp_int_t videosample_get_height(mp_obj_t source); -mp_int_t videosample_get_bits_per_pixel(mp_obj_t source); +mp_int_t videosource_get_width(mp_obj_t source); +mp_int_t videosource_get_height(mp_obj_t source); +mp_int_t videosource_get_bits_per_pixel(mp_obj_t source); // Raise when the group tree contains a VideoBuffer that streams into a // framebuffer other than the one this display renders. Called when a group