diff --git a/locale/circuitpython.pot b/locale/circuitpython.pot index c03d1bd0ae4..97b798f7f5f 100644 --- a/locale/circuitpython.pot +++ b/locale/circuitpython.pot @@ -974,7 +974,6 @@ msgstr "" #: ports/atmel-samd/common-hal/pulseio/PulseIn.c #: ports/atmel-samd/common-hal/rotaryio/IncrementalEncoder.c #: ports/cxd56/common-hal/pulseio/PulseIn.c -#: ports/espressif/common-hal/mii/RMII.c #: ports/raspberrypi/common-hal/picodvi/Framebuffer_RP2350.c #: ports/zephyr-cp/common-hal/busio/I2C.c #: ports/zephyr-cp/common-hal/busio/SPI.c @@ -1182,6 +1181,7 @@ msgid "Another send is already active" msgstr "" #: ports/atmel-samd/common-hal/sdioio/SDCard.c +#: ports/espressif/common-hal/mii/Ethernet.c #: ports/espressif/common-hal/qspibus/QSPIBus.c shared-bindings/emmcio/EMMC.c #: shared-bindings/socketpool/SocketPool.c msgid "%q failure: %d" @@ -1219,6 +1219,7 @@ msgstr "" #: shared-bindings/busdisplay/BusDisplay.c #: shared-bindings/framebufferio/FramebufferDisplay.c #: shared-bindings/struct/__init__.c shared-module/struct/__init__.c +#: shared-module/videocore/VideoBuffer.c msgid "Buffer too small" msgstr "" @@ -1517,7 +1518,8 @@ msgid "%q must be 8, 16, 24, or 32" msgstr "" #: ports/espressif/common-hal/audiobusio/__init__.c -#: ports/espressif/common-hal/audioi2sin/I2SIn.c shared-bindings/emmcio/EMMC.c +#: ports/espressif/common-hal/audioi2sin/I2SIn.c +#: ports/espressif/common-hal/mii/Ethernet.c shared-bindings/emmcio/EMMC.c msgid "Peripheral in use" msgstr "" @@ -1563,6 +1565,8 @@ msgid "Can't construct AudioOut because continuous channel already open" msgstr "" #: ports/espressif/common-hal/audioio/AudioOut.c +#: ports/espressif/common-hal/mipicsi/Camera.c +#: shared-module/videocore/VideoBuffer.c msgid "already playing" msgstr "" @@ -1751,6 +1755,18 @@ msgstr "" msgid "Address range not allowed" msgstr "" +#: ports/espressif/common-hal/mipicsi/Camera.c +msgid "pixel_format must be RAW8, RAW10 or RAW12 when output_format differs" +msgstr "" + +#: ports/espressif/common-hal/mipicsi/Camera.c +msgid "output_format must be RGB565 or RGB888" +msgstr "" + +#: ports/espressif/common-hal/mipicsi/Camera.c +msgid "ISP conversion not supported on this chip" +msgstr "" + #: ports/espressif/common-hal/nvm/ByteArray.c msgid "NVS Error" msgstr "" @@ -4771,6 +4787,50 @@ msgstr "" msgid "Polygon needs at least 3 points" msgstr "" +#: shared-module/videocore/VideoBuffer.c +msgid "framebuffer went away" +msgstr "" + +#: shared-module/videocore/VideoBuffer.c +msgid "video doesn't fit in framebuffer" +msgstr "" + +#: shared-module/videocore/VideoBuffer.c +msgid "framebuffer missing required functions" +msgstr "" + +#: shared-module/videocore/VideoBuffer.c +msgid "expected FrameBuffer or %q" +msgstr "" + +#: shared-module/videocore/VideoBuffer.c +msgid "video format must be byte aligned" +msgstr "" + +#: shared-module/videocore/VideoBuffer.c +msgid "video format doesn't match framebuffer" +msgstr "" + +#: shared-module/videocore/VideoBuffer.c +msgid "video source rejected the destination" +msgstr "" + +#: shared-module/videocore/VideoBuffer.c +msgid "video in a framebuffer must be at x == 0" +msgstr "" + +#: shared-module/videocore/VideoBuffer.c +msgid "video size changed since play; stop and play again" +msgstr "" + +#: shared-module/videocore/VideoBuffer.c +msgid "VideoBuffer does not support transforms" +msgstr "" + +#: shared-module/videocore/VideoBuffer.c +msgid "VideoBuffer is playing into a different framebuffer" +msgstr "" + #: supervisor/shared/bluetooth/bluetooth.c msgid "Reconnecting" msgstr "" diff --git a/ports/espressif/CMakeLists.txt b/ports/espressif/CMakeLists.txt index 608d4f4c7b3..979afdad548 100644 --- a/ports/espressif/CMakeLists.txt +++ b/ports/espressif/CMakeLists.txt @@ -5,7 +5,22 @@ cmake_minimum_required(VERSION 3.16) set(ENV{IDF_PATH} ${CMAKE_SOURCE_DIR}/esp-idf) # The component list here determines what options we get in menuconfig and what the ninja file can build. -set(COMPONENTS bt driver esp_driver_dac esp_driver_gpio esp_driver_gptimer esp_driver_i2c esp_driver_i2s esp_driver_ledc esp_driver_pcnt esp_driver_rmt esp_driver_sdmmc esp_driver_spi esp_driver_touch_sens esp_driver_tsens esp_driver_uart esp_eth esp-tls esp_adc esp_adc_cal esp_event esp_netif esp_psram esp_security esp_wifi esptool_py freertos log lwip main mbedtls mdns pthread soc ulp wpa_supplicant esp-camera esp_lcd vfs esp_vfs_console esp_stdio sdmmc usb) +# Keep this split into separate set()/list() calls so additions merge easily. +# Core components +set(COMPONENTS bt driver esptool_py freertos log main mbedtls pthread soc) +# Peripheral drivers +list(APPEND COMPONENTS esp_driver_dac esp_driver_gpio esp_driver_gptimer esp_driver_i2c esp_driver_i2s esp_driver_ledc esp_driver_pcnt esp_driver_rmt esp_driver_sdmmc esp_driver_spi esp_driver_touch_sens esp_driver_tsens esp_driver_uart) +# Networking +list(APPEND COMPONENTS esp_eth esp-tls esp_adc esp_adc_cal esp_event esp_netif esp_psram esp_security esp_wifi lwip mdns wpa_supplicant) +# Misc +list(APPEND COMPONENTS ulp vfs esp_vfs_console esp_stdio sdmmc usb) +# Display and camera +list(APPEND COMPONENTS esp-camera esp_lcd) + +if(CIRCUITPY_MIPICSI) + list(APPEND COMPONENTS esp_driver_cam esp_hal_cam) +endif() + set(EXTRA_COMPONENT_DIRS "esp-protocols/components/mdns" "esp-camera") include($ENV{IDF_PATH}/tools/cmake/project.cmake) diff --git a/ports/espressif/Makefile b/ports/espressif/Makefile index 96a71b177b4..e4f93259bce 100644 --- a/ports/espressif/Makefile +++ b/ports/espressif/Makefile @@ -226,6 +226,13 @@ REGISTRATION_FUNCTIONS = \ -u esp_security_init_include_impl \ -u mbedtls_psa_crypto_init_include_impl +# Video DMA destinations come from the Python heap (user bytearrays), and +# the cache line is 64 bytes on the P4, so make GC blocks 64 bytes as well. +# supervisor/port.c aligns the PSRAM heap areas to the same value. +ifeq ($(IDF_TARGET),esp32p4) +CFLAGS += -DMICROPY_BYTES_PER_GC_BLOCK=64 +endif + # RISC-V picolibc ships optimized memcpy/memset/etc. that need forcing # (gated by CONFIG_LIBC_OPTIMIZED_MISALIGNED_ACCESS in IDF's esp_libc CMake). ifeq ($(IDF_TARGET_ARCH),riscv) @@ -504,9 +511,12 @@ endif CHIP_COMPONENTS = \ + esp_driver_cam \ + esp_driver_isp \ esp_driver_touch_sens \ esp_driver_tsens \ esp_driver_usb_serial_jtag \ + esp_hal_cam \ esp_hal_dma \ esp_hal_emac \ esp_hal_i2s \ @@ -735,6 +745,14 @@ CFLAGS += \ -isystem esp-idf/components/esp_lcd/dsi/include endif +ifneq ($(CIRCUITPY_MIPICSI),0) +CFLAGS += \ + -isystem esp-idf/components/esp_driver_cam/include \ + -isystem esp-idf/components/esp_driver_cam/csi/include \ + -isystem esp-idf/components/esp_driver_isp/include \ + -isystem esp-idf/components/esp_hal_cam/include +endif + ifneq ($(CIRCUITPY_ESPCAMERA),0) SRC_CAMERA := \ $(wildcard common-hal/espcamera/*.c) \ @@ -935,7 +953,7 @@ do-sdkconfig: $(BUILD)/esp-idf/config/sdkconfig.h QSTR_GLOBAL_REQUIREMENTS += $(BUILD)/esp-idf/config/sdkconfig.h $(BUILD)/esp-idf/config/sdkconfig.h: boards/$(BOARD)/sdkconfig boards/$(BOARD)/mpconfigboard.mk CMakeLists.txt | $(BUILD)/esp-idf $(STEPECHO) "Create $@" - $(Q)env IDF_PATH=$(IDF_PATH) IDF_COMPONENT_MANAGER=0 cmake -S . -B $(BUILD)/esp-idf -DSDKCONFIG=$(BUILD)/esp-idf/sdkconfig -DSDKCONFIG_DEFAULTS="$(SDKCONFIGS)" -DCMAKE_TOOLCHAIN_FILE=$(IDF_PATH)/tools/cmake/toolchain-$(IDF_TARGET).cmake -DIDF_TARGET=$(IDF_TARGET) -GNinja + $(Q)env IDF_PATH=$(IDF_PATH) IDF_COMPONENT_MANAGER=0 cmake -S . -B $(BUILD)/esp-idf -DSDKCONFIG=$(BUILD)/esp-idf/sdkconfig -DSDKCONFIG_DEFAULTS="$(SDKCONFIGS)" -DCMAKE_TOOLCHAIN_FILE=$(IDF_PATH)/tools/cmake/toolchain-$(IDF_TARGET).cmake -DIDF_TARGET=$(IDF_TARGET) -DCIRCUITPY_MIPICSI=$(CIRCUITPY_MIPICSI) -GNinja $(Q)$(PYTHON) tools/check-sdkconfig.py \ CIRCUITPY_DUALBANK=$(CIRCUITPY_DUALBANK) \ CIRCUITPY_STORAGE_EXTEND=$(CIRCUITPY_STORAGE_EXTEND) \ @@ -1060,6 +1078,9 @@ endif ifneq ($(CIRCUITPY_MIPIDSI),0) ESP_IDF_COMPONENTS_LINK += esp_lcd endif +ifneq ($(CIRCUITPY_MIPICSI),0) + ESP_IDF_COMPONENTS_LINK += esp_driver_cam esp_driver_isp esp_hal_cam +endif ifneq ($(CIRCUITPY_PARALLELDISPLAYBUS),0) ESP_IDF_COMPONENTS_LINK += esp_lcd endif diff --git a/ports/espressif/boards/espressif_esp32p4_function_ev/sdkconfig b/ports/espressif/boards/espressif_esp32p4_function_ev/sdkconfig index fb8986613ff..8ad98a7b081 100644 --- a/ports/espressif/boards/espressif_esp32p4_function_ev/sdkconfig +++ b/ports/espressif/boards/espressif_esp32p4_function_ev/sdkconfig @@ -1,3 +1,27 @@ +# +# Espressif IoT Development Framework Configuration +# +# Bootloader +# +CONFIG_BOOTLOADER_LOG_LEVEL_WARN=y +CONFIG_BOOTLOADER_LOG_LEVEL=2 +# end of Bootloader + +# This board ships with pre-v3 (eco2) silicon: link the 2nd stage bootloader +# into the lower memory layout used by revisions 0.x and 1.x. CONFIG_ESP32P4_SELECTS_REV_LESS_V3=y +# This board's silicon is rev v1.3; match the working BSP-demo build (REV_MIN +# v1.0) so IDF compiles the same errata/clock handling as the reference stack. CONFIG_ESP32P4_REV_MIN_100=y CONFIG_ETH_USE_ESP32_EMAC=y + +# ESP-STDIO: send the ESP-IDF console to the USB-Serial-JTAG port so bootloader +# logs are visible on the board's serial connector. +# +CONFIG_ESP_CONSOLE_USB_SERIAL_JTAG=y +CONFIG_ESP_CONSOLE_USB_SERIAL_JTAG_ENABLED=y +CONFIG_ESP_CONSOLE_UART_NUM=-1 +CONFIG_ESP_CONSOLE_ROM_SERIAL_PORT_NUM=6 +# end of ESP-STDIO + +# CONFIG_ESP_SYSTEM_MEMPROT is not set diff --git a/ports/espressif/common-hal/mipicsi/Camera.c b/ports/espressif/common-hal/mipicsi/Camera.c new file mode 100644 index 00000000000..2a68e2d660c --- /dev/null +++ b/ports/espressif/common-hal/mipicsi/Camera.c @@ -0,0 +1,468 @@ +// This file is part of the CircuitPython project: https://circuitpython.org +// +// SPDX-FileCopyrightText: Copyright (c) 2025 Jeff Epler for Adafruit Industries +// +// SPDX-License-Identifier: MIT + +#include "py/mperrno.h" +#include "py/mphal.h" +#include "py/nlr.h" +#include "py/runtime.h" + +#include "common-hal/mipicsi/Camera.h" +#include "shared-bindings/mipicsi/Camera.h" +#include "shared-bindings/mipicsi/__init__.h" +#include "bindings/espidf/__init__.h" + +#include "common-hal/mipidsi/Bus.h" + +#include +#include "esp_attr.h" +#include "esp_cache.h" +#include "esp_cam_ctlr.h" +#include "esp_private/esp_cache_private.h" +#include "esp_cam_ctlr_csi.h" +#include "hal/color_hal.h" + +// MIPI CSI receives data from a fixed, dedicated set of pins, so no pin +// configuration or claiming is done here. + +#define ISP_CLK_HZ (80 * 1000 * 1000) + +static cam_ctlr_color_t mipicsi_color_for(mipicsi_pixel_format_t format) { + switch (format) { + case MIPICSI_FORMAT_RAW8: + return CAM_CTLR_COLOR_RAW8; + case MIPICSI_FORMAT_RAW10: + return CAM_CTLR_COLOR_RAW10; + case MIPICSI_FORMAT_RAW12: + return CAM_CTLR_COLOR_RAW12; + case MIPICSI_FORMAT_RGB888: + return CAM_CTLR_COLOR_RGB888; + case MIPICSI_FORMAT_YUV422: + return CAM_CTLR_COLOR_YUV422_YUYV; + case MIPICSI_FORMAT_GRAYSCALE: + return CAM_CTLR_COLOR_GRAY8; + case MIPICSI_FORMAT_RGB565: + default: + return CAM_CTLR_COLOR_RGB565; + } +} + +static mp_int_t mipicsi_bits_per_pixel_for(mipicsi_pixel_format_t format) { + switch (format) { + case MIPICSI_FORMAT_RAW10: + return 10; + case MIPICSI_FORMAT_RAW12: + return 12; + case MIPICSI_FORMAT_RGB888: + return 24; + case MIPICSI_FORMAT_RGB565: + case MIPICSI_FORMAT_YUV422: + return 16; + case MIPICSI_FORMAT_RAW8: + case MIPICSI_FORMAT_GRAYSCALE: + default: + return 8; + } +} + +#if SOC_ISP_SUPPORTED +// The ISP color types share their FOURCC values with the CAM_CTLR color +// types, except for GRAYSCALE (no ISP grey mode) and YUV422 (the ISP input is +// UYVY, the CAM_CTLR YUYV). Only sensor RAW formats convert here. +static isp_color_t mipicsi_isp_color_for(mipicsi_pixel_format_t format) { + switch (format) { + case MIPICSI_FORMAT_RAW8: + return ISP_COLOR_RAW8; + case MIPICSI_FORMAT_RAW10: + return ISP_COLOR_RAW10; + case MIPICSI_FORMAT_RAW12: + return ISP_COLOR_RAW12; + case MIPICSI_FORMAT_RGB565: + return ISP_COLOR_RGB565; + case MIPICSI_FORMAT_RGB888: + return ISP_COLOR_RGB888; + case MIPICSI_FORMAT_YUV422: + case MIPICSI_FORMAT_GRAYSCALE: + default: + return 0; + } +} +#endif + +// Called by the driver (in ISR context) when the CSI host detects an error +// (PHY, packet, frame or CRC problems). Only installed for diagnostics: the +// counts are printed to the ROM console so they show up on the serial console. +static IRAM_ATTR bool mipicsi_on_error( + esp_cam_ctlr_handle_t handle, const esp_cam_ctlr_error_event_data_t *edata, void *user_data) { + mipicsi_camera_obj_t *self = user_data; + self->error_count++; + self->error_events |= edata->csi_host_err_evts; + if (self->error_count <= 3) { + esp_rom_printf("mipicsi: csi error #%lu events 0x%lx\n", + (unsigned long)self->error_count, (unsigned long)edata->csi_host_err_evts); + } + return false; +} + +// Called by the driver (in ISR context) to get the next buffer to DMA into. +static IRAM_ATTR bool mipicsi_on_get_new_trans( + esp_cam_ctlr_handle_t handle, esp_cam_ctlr_trans_t *trans, void *user_data) { + mipicsi_camera_obj_t *self = user_data; + + // Direct streaming: DMA the whole frame into the current destination. + // The videobuffer can change the destination between frames; the change + // is picked up here on the next frame. + // + // The controller only runs while a destination is set, so there is + // always somewhere to DMA into here; set_destination(NULL) stops the + // controller before clearing the destination. With the driver's backup + // buffer disabled, no buffer to hand over is unrecoverable (the driver + // asserts, or DMAs to address 0 in release builds), so there is no + // "give NULL" path. + trans->buffer = (void *)self->dest_base; + trans->buflen = self->buffer_len; + return false; +} + +// Called by the driver (in ISR context) when a frame has been written. +static IRAM_ATTR bool mipicsi_on_trans_finished( + esp_cam_ctlr_handle_t handle, esp_cam_ctlr_trans_t *trans, void *user_data) { + mipicsi_camera_obj_t *self = user_data; + + if (self->dest_base != NULL && self->dest_on_frame != NULL) { + self->dest_on_frame(self->dest_ctx); + } + return false; +} + +void common_hal_mipicsi_camera_construct( + mipicsi_camera_obj_t *self, + mipicsi_pixel_format_t output_format) { + + self->width = 1; + self->height = 1; + self->pixel_format = output_format; + self->output_format = output_format; + self->dest_base = NULL; + self->dest_on_frame = NULL; + self->dest_ctx = NULL; + self->ctlr_started = false; + self->error_count = 0; + self->error_events = 0; + self->sensor = mp_const_none; + self->phy_ldo = NULL; + self->isp_proc = NULL; + self->spinlock = (portMUX_TYPE)portMUX_INITIALIZER_UNLOCKED; + // No receiver hardware yet: it is created in the videosource play step, + // where the sensor's current mode is known. +} + +// Videosource protocol play step: create the LDO, ISP and CSI controller for +// the given (freshly read) sensor parameters and start the sensor streaming. +// Raises when the camera is already streaming for someone else. Not a user +// facing method: it is driven by the videosource protocol (from +// VideoBuffer.play), so it may raise. +void common_hal_mipicsi_camera_videosource_play( + mipicsi_camera_obj_t *self, + const mipicsi_sensor_params_t *params) { + + if (self->handle != NULL) { + mp_raise_RuntimeError(MP_ERROR_TEXT("already playing")); + } + + // Adopt what we need from the sensor's current mode; it may have + // changed since the Camera was made (sensors are free to reconfigure + // while they aren't being played). + self->width = params->width; + self->height = params->height; + self->bayer_order = params->bayer_order; + self->data_lanes = params->data_lanes; + self->lane_bit_rate_mbps = params->lane_bit_rate_mbps; + + const cam_ctlr_color_t color = mipicsi_color_for(self->pixel_format); + + #if SOC_ISP_SUPPORTED + if (self->output_format != self->pixel_format) { + // The sensor transmits in a different format than the one requested. + // On gen-1 P4 silicon the CSI receiver cannot convert, so route it + // through the ISP processor instead, mirroring the IDF + // esp_driver_cam csi test. + const isp_color_t isp_in = mipicsi_isp_color_for(self->pixel_format); + const isp_color_t isp_out = mipicsi_isp_color_for(self->output_format); + if (isp_in == 0 || (isp_in != ISP_COLOR_RAW8 && isp_in != ISP_COLOR_RAW10 && isp_in != ISP_COLOR_RAW12)) { + mp_raise_ValueError(MP_ERROR_TEXT("pixel_format must be RAW8, RAW10 or RAW12 when output_format differs")); + } + if (isp_out != ISP_COLOR_RGB565 && isp_out != ISP_COLOR_RGB888) { + mp_raise_ValueError(MP_ERROR_TEXT("output_format must be RGB565 or RGB888")); + } + esp_isp_processor_cfg_t isp_config = { + .clk_hz = ISP_CLK_HZ, + .input_data_source = ISP_INPUT_DATA_SOURCE_CSI, + .input_data_color_type = isp_in, + .output_data_color_type = isp_out, + .has_line_start_packet = false, + .has_line_end_packet = false, + .h_res = self->width, + .v_res = self->height, + .bayer_order = (color_raw_element_order_t)self->bayer_order, + }; + CHECK_ESP_RESULT(esp_isp_new_processor(&isp_config, &self->isp_proc)); + } + #else + if (self->output_format != self->pixel_format) { + mp_raise_NotImplementedError(MP_ERROR_TEXT("ISP conversion not supported on this chip")); + } + #endif + + // Power the MIPI D-PHY. The CSI and DSI PHYs share LDO channel 3; the + // shared reference count (see mipidsi_phy_ldo_acquire) keeps the rail up + // while either the camera or a mipidsi.Bus is alive. The sensor's I2C + // control interface is on a different rail and works without this. + self->phy_ldo = mipidsi_phy_ldo_acquire(); + + // The received images are not converted, so the input and output color + // types are the same and color conversion is bypassed. + esp_cam_ctlr_csi_config_t csi_config = { + .ctlr_id = 0, + .h_res = self->width, + .v_res = self->height, + .data_lane_num = self->data_lanes, + .lane_bit_rate_mbps = self->lane_bit_rate_mbps, + .input_data_color_type = color, + .output_data_color_type = color, + // One transaction queue item; frames go straight to the current + // destination. The controller is only started while a destination + // is set, so the driver's backup buffer is never needed and its + // frame-sized allocation is skipped. + .queue_items = 1, + .bk_buffer_dis = true, + }; + + esp_err_t result = esp_cam_new_csi_ctlr(&csi_config, &self->handle); + if (result != ESP_OK) { + mipidsi_phy_ldo_release(); + self->phy_ldo = NULL; + raise_esp_error(result); + } + + // Exact frame size. Destinations are written directly (no cache-line + // padding is ever DMA'd past the last row), so they only need to be wide + // enough to hold the whole frame; the DSI framebuffer always is. When + // the ISP is in the pipeline, the CSI receiver hands its converted + // output to us. + self->buffer_len = ((size_t)self->width * self->height * mipicsi_bits_per_pixel_for(self->output_format) + 7) / 8; + + esp_cam_ctlr_evt_cbs_t cbs = { + .on_get_new_trans = mipicsi_on_get_new_trans, + .on_trans_finished = mipicsi_on_trans_finished, + .on_error = mipicsi_on_error, + }; + result = esp_cam_ctlr_register_event_callbacks(self->handle, &cbs, self); + if (result != ESP_OK) { + common_hal_mipicsi_camera_deinit(self); + self->sensor = mp_const_none; + raise_esp_error(result); + } + + result = esp_cam_ctlr_enable(self->handle); + if (result != ESP_OK) { + common_hal_mipicsi_camera_deinit(self); + self->sensor = mp_const_none; + raise_esp_error(result); + } + #if SOC_ISP_SUPPORTED + if (self->isp_proc) { + result = esp_isp_enable(self->isp_proc); + if (result != ESP_OK) { + common_hal_mipicsi_camera_deinit(self); + self->sensor = mp_const_none; + raise_esp_error(result); + } + } + #endif + + // Start the sensor streaming so frames exist to capture once a + // destination is set. Sensors' _start() is expected to be idempotent. + // The camera never calls the sensor's stop(): when a stream ends (or on + // reload) only the CSI receiver and ISP are torn down; the sensor stops + // through its own finaliser (deinit/__del__). + if (self->sensor != mp_const_none) { + nlr_buf_t nlr; + if (nlr_push(&nlr) == 0) { + mp_obj_t start_method[2]; + mp_load_method(self->sensor, MP_QSTR__start, start_method); + mp_call_method_n_kw(0, 0, start_method); + nlr_pop(); + } else { + common_hal_mipicsi_camera_deinit(self); + self->sensor = mp_const_none; + nlr_raise(nlr.ret_val); + } + } + + // Don't start the controller yet: it only runs while a destination is + // set (see common_hal_mipicsi_camera_set_destination), so no frame is + // ever captured without a place to put it. +} + +// Tear down the receiver hardware quietly: this runs from Python (a +// VideoBuffer stopped) and from background contexts (destination dropped on +// reload), so it must not raise. The sensor is left alone: its own finaliser +// stops it. Only the CSI receiver and ISP turn off. +static void mipicsi_camera_teardown(mipicsi_camera_obj_t *self) { + if (self->ctlr_started) { + esp_cam_ctlr_stop(self->handle); + self->ctlr_started = false; + } + portENTER_CRITICAL(&self->spinlock); + self->dest_base = NULL; + self->dest_on_frame = NULL; + self->dest_ctx = NULL; + portEXIT_CRITICAL(&self->spinlock); + + nlr_buf_t nlr; + if (nlr_push(&nlr) == 0) { + if (self->handle != NULL) { + esp_cam_ctlr_disable(self->handle); + esp_cam_ctlr_del(self->handle); + self->handle = NULL; + } + #if SOC_ISP_SUPPORTED + if (self->isp_proc) { + esp_isp_disable(self->isp_proc); + esp_isp_del_processor(self->isp_proc); + self->isp_proc = NULL; + } + #endif + mipidsi_phy_ldo_release(); + self->phy_ldo = NULL; + nlr_pop(); + } else { + // Nothing to do about a failed teardown, not even a partially + // released one; swallow the exception and continue. + } +} + +void common_hal_mipicsi_camera_set_sensor(mipicsi_camera_obj_t *self, mp_obj_t sensor) { + self->sensor = sensor; +} + +void common_hal_mipicsi_camera_deinit(mipicsi_camera_obj_t *self) { + if (self->handle == NULL) { + // Not currently playing: nothing to tear down. + return; + } + + // Only turn off the CSI receiver and ISP here (including from background + // contexts such as reload): the sensor stops itself through its own + // finaliser (deinit/__del__). + mipicsi_camera_teardown(self); + self->sensor = mp_const_none; +} +// Videosource protocol stop step: tear down the receiver hardware and stop +// the sensor so its settings can be changed. Not a user facing method: it is +// driven by the videosource protocol (from VideoBuffer.stop), so it may +// raise; exceptions from the sensor's stop() propagate. +void common_hal_mipicsi_camera_videosource_stop(mipicsi_camera_obj_t *self) { + if (self->handle == NULL) { + return; // Not playing. + } + + // The receiver first: after the CSI controller stops, no more frames are + // captured, whatever the sensor does. + mipicsi_camera_teardown(self); + + // Stop the sensor (e.g. the sc2336 stops streaming and unlocks its + // mode). Sensors' _stop() is expected to be idempotent. + if (self->sensor != mp_const_none) { + mp_obj_t stop_method[2]; + mp_load_method(self->sensor, MP_QSTR__stop, stop_method); + mp_call_method_n_kw(0, 0, stop_method); + } +} + +bool common_hal_mipicsi_camera_deinited(mipicsi_camera_obj_t *self) { + return self->handle == NULL; +} + +bool common_hal_mipicsi_camera_set_destination( + mipicsi_camera_obj_t *self, + uint8_t *base, + size_t stride, + void (*on_frame)(void *ctx), + void *ctx) { + const size_t row_bytes = ((size_t)self->width * mipicsi_bits_per_pixel_for(self->output_format) + 7) / 8; + const size_t frame_bytes = (size_t)self->height * row_bytes; + + // There is no stop value on the destination: stopping is the videosource + // stop step's job. The protocol never passes NULL, + // so this guard is only about internal misuse. + assert(base != NULL); + + // DMA destinations must be cache-line aligned: the driver syncs the + // buffer on every frame with esp_cache_msync, which rejects anything + // else. PSRAM heap areas (and with 64-byte GC blocks, every GC + // allocation in them, e.g. bytearrays) are allocated 64-byte aligned on + // this chip, so this is a safety net. + size_t cache_line = 0; + esp_cache_get_alignment(MALLOC_CAP_SPIRAM, &cache_line); + if (cache_line == 0) { + cache_line = 64; + } + if ((uintptr_t)base % cache_line != 0) { + return false; + } + + // The destination is assumed to hold the frame rows with no stride + // padding: the CSI link-list DMA only does contiguous transfers in this + // IDF. The videobuffer validates the fit before getting here; the whole + // (padded) frame must lie within the destination: rows can't wrap past + // its end, so there is no cropping or overhang. + if (stride != row_bytes) { + return false; + } + + // Size handed to the driver: the frame, cache-line padded at the end so + // the driver's per-frame sync is happy. + const size_t buffer_len = (frame_bytes + cache_line - 1) / cache_line * cache_line; + + portENTER_CRITICAL(&self->spinlock); + self->dest_base = base; + self->dest_on_frame = on_frame; + self->dest_ctx = ctx; + self->buffer_len = buffer_len; + portEXIT_CRITICAL(&self->spinlock); + + // The sensor streaming and the receiver hardware exist from play(); + // here only the controller runs or doesn't. + if (!self->ctlr_started) { + esp_err_t result = esp_cam_ctlr_start(self->handle); + if (result != ESP_OK) { + return false; + } + self->ctlr_started = true; + } + return true; +} + +mipicsi_pixel_format_t common_hal_mipicsi_camera_get_pixel_format(mipicsi_camera_obj_t *self) { + return self->pixel_format; +} + +mipicsi_pixel_format_t common_hal_mipicsi_camera_get_output_format(mipicsi_camera_obj_t *self) { + return self->output_format; +} + +mp_int_t common_hal_mipicsi_camera_get_width(mipicsi_camera_obj_t *self) { + return self->width; +} + +mp_int_t common_hal_mipicsi_camera_get_height(mipicsi_camera_obj_t *self) { + return self->height; +} + +mp_int_t common_hal_mipicsi_camera_get_bits_per_pixel(mipicsi_camera_obj_t *self) { + return mipicsi_bits_per_pixel_for(self->output_format); +} diff --git a/ports/espressif/common-hal/mipicsi/Camera.h b/ports/espressif/common-hal/mipicsi/Camera.h new file mode 100644 index 00000000000..b3e0d269e5d --- /dev/null +++ b/ports/espressif/common-hal/mipicsi/Camera.h @@ -0,0 +1,89 @@ +// This file is part of the CircuitPython project: https://circuitpython.org +// +// SPDX-FileCopyrightText: Copyright (c) 2025 Jeff Epler for Adafruit Industries +// +// SPDX-License-Identifier: MIT + +#pragma once + +#include + +#include "py/obj.h" + +#include "freertos/FreeRTOS.h" +#include "freertos/portmacro.h" + +#include "esp_cam_ctlr.h" +#include "esp_ldo_regulator.h" + +#if SOC_ISP_SUPPORTED +#include "driver/isp.h" +#endif + +#include "shared-bindings/mipicsi/__init__.h" + +typedef struct mipicsi_camera_obj mipicsi_camera_obj_t; + +// Sensor parameters used to build the receiver. Read off the sensor object +// when a stream starts (the sensor is free to reconfigure between streams). +typedef struct { + mp_int_t width; + mp_int_t height; + mp_int_t bayer_order; + mp_int_t data_lanes; + mp_int_t lane_bit_rate_mbps; +} mipicsi_sensor_params_t; + +struct mipicsi_camera_obj { + mp_obj_base_t base; + esp_cam_ctlr_handle_t handle; + + // Power for the MIPI D-PHY, shared with the DSI transmitter. Acquired + // when a stream starts and released when it (or the camera) ends. + esp_ldo_channel_handle_t phy_ldo; + + // Camera sensor configured to transmit to this receiver, or None. + mp_obj_t sensor; + + // ISP processor used to convert sensor frames to output_format, only + // present when the CSI receiver itself cannot convert (e.g. gen-1 P4). + mipicsi_pixel_format_t output_format; + #if SOC_ISP_SUPPORTED + isp_proc_handle_t isp_proc; + #endif + + // The receiver hardware (LDO, ISP, CSI controller) is created when a + // stream starts (see play) and torn down when it ends. The sensor's + // parameters (width, height, data_lanes, lane_bit_rate_mbps, bayer_order) + // come from the sensor at play time as well; pixel_format only when it + // wasn't overridden at construction. + // Streaming destination, set by videocore.VideoBuffer. While base is + // NULL the CSI controller is stopped: no capture buffers are used at + // all. The destination is assumed to be a whole frame: DMA'd straight + // into base, stride equal to the row size. on_frame(ctx) is called from + // ISR context each time a frame lands in base. + // The DMA destination: set_destination requires it to be cache-line + // aligned (the driver's per-frame cache sync needs that) and reports a + // rejected destination with a false return, rather than shifting frames + // elsewhere. The frame must be contiguous. + volatile void *dest_base; + void (*dest_on_frame)(void *ctx); + void *dest_ctx; + bool ctlr_started; + + // Diagnostics from the CSI host error interrupt. + volatile uint32_t error_count; + volatile uint32_t error_events; + + portMUX_TYPE spinlock; + + mp_int_t width; + mp_int_t height; + mipicsi_pixel_format_t pixel_format; + mp_int_t data_lanes; + mp_int_t lane_bit_rate_mbps; + // Bayer element order of the sensor's RAW output, only used when the ISP + // converts from a RAW format. + mp_int_t bayer_order; + size_t buffer_len; // Length handed to the driver, includes cache-alignment padding. +}; diff --git a/ports/espressif/common-hal/mipicsi/__init__.c b/ports/espressif/common-hal/mipicsi/__init__.c new file mode 100644 index 00000000000..816fea93ae9 --- /dev/null +++ b/ports/espressif/common-hal/mipicsi/__init__.c @@ -0,0 +1,7 @@ +// This file is part of the CircuitPython project: https://circuitpython.org +// +// SPDX-FileCopyrightText: Copyright (c) 2025 Jeff Epler for Adafruit Industries +// +// SPDX-License-Identifier: MIT + +#include "shared-bindings/mipicsi/__init__.h" diff --git a/ports/espressif/common-hal/mipidsi/Bus.c b/ports/espressif/common-hal/mipidsi/Bus.c index dbb559ea30f..dab781f8cc9 100644 --- a/ports/espressif/common-hal/mipidsi/Bus.c +++ b/ports/espressif/common-hal/mipidsi/Bus.c @@ -9,6 +9,39 @@ #include "py/runtime.h" #include +// Shared MIPI PHY LDO rail. DSI buses and CSI receivers both need it; it is +// acquired once and released only when the last user goes away. +static esp_ldo_channel_handle_t mipidsi_phy_ldo = NULL; +static uint8_t mipidsi_phy_ldo_users = 0; + +esp_ldo_channel_handle_t mipidsi_phy_ldo_acquire(void) { + if (mipidsi_phy_ldo_users == 0) { + esp_ldo_channel_handle_t ldo = NULL; + esp_ldo_channel_config_t ldo_config = { + .chan_id = 3, + .voltage_mv = 2500, + // Fixed-voltage (non-adjustable) acquire so both users share the + // rail. An adjustable acquire would block the other user. + .flags.adjustable = false, + }; + ESP_ERROR_CHECK(esp_ldo_acquire_channel(&ldo_config, &ldo)); + mipidsi_phy_ldo = ldo; + } + mipidsi_phy_ldo_users++; + return mipidsi_phy_ldo; +} + +void mipidsi_phy_ldo_release(void) { + if (mipidsi_phy_ldo_users == 0) { + return; + } + mipidsi_phy_ldo_users--; + if (mipidsi_phy_ldo_users == 0 && mipidsi_phy_ldo != NULL) { + esp_ldo_release_channel(mipidsi_phy_ldo); + mipidsi_phy_ldo = NULL; + } +} + void common_hal_mipidsi_bus_construct(mipidsi_bus_obj_t *self, mp_uint_t frequency, uint8_t num_lanes) { self->frequency = frequency; self->num_data_lanes = num_lanes; @@ -18,12 +51,7 @@ void common_hal_mipidsi_bus_construct(mipidsi_bus_obj_t *self, mp_uint_t frequen mp_raise_ValueError_varg(MP_ERROR_TEXT("%q in use"), MP_QSTR_mipidsi); } - esp_ldo_channel_handle_t ldo_mipi_phy = NULL; - esp_ldo_channel_config_t ldo_mipi_phy_config = { - .chan_id = 3, - .voltage_mv = 2500, - }; - ESP_ERROR_CHECK(esp_ldo_acquire_channel(&ldo_mipi_phy_config, &ldo_mipi_phy)); + self->phy_ldo = mipidsi_phy_ldo_acquire(); // Create the DSI bus esp_lcd_dsi_bus_config_t bus_config = { @@ -50,6 +78,11 @@ void common_hal_mipidsi_bus_deinit(mipidsi_bus_obj_t *self) { self->bus_handle = NULL; } + // Release our hold on the shared MIPI PHY LDO; it stays powered as long + // as a CSI receiver (or other user) still holds it. + mipidsi_phy_ldo_release(); + self->phy_ldo = NULL; + self->frequency = 0; self->num_data_lanes = 0; } diff --git a/ports/espressif/common-hal/mipidsi/Bus.h b/ports/espressif/common-hal/mipidsi/Bus.h index b6257fd7223..1920275d430 100644 --- a/ports/espressif/common-hal/mipidsi/Bus.h +++ b/ports/espressif/common-hal/mipidsi/Bus.h @@ -8,14 +8,23 @@ #include "py/obj.h" #include +#include typedef struct { mp_obj_base_t base; mp_uint_t frequency; esp_lcd_dsi_bus_handle_t bus_handle; + esp_ldo_channel_handle_t phy_ldo; uint8_t num_data_lanes; uint8_t use_count; // Up to 4 displays } mipidsi_bus_obj_t; void mipidsi_bus_increment_use_count(mipidsi_bus_obj_t *self); void mipidsi_bus_decrement_use_count(mipidsi_bus_obj_t *self); + +// Acquire the shared MIPI PHY LDO rail (LDO channel 3, 2.5 V). DSI and CSI +// users share the rail; a shared reference count keeps it powered until the +// last user releases it, so the CSI receiver can stay active while a DSI bus +// comes and goes (and vice versa). +esp_ldo_channel_handle_t mipidsi_phy_ldo_acquire(void); +void mipidsi_phy_ldo_release(void); diff --git a/ports/espressif/common-hal/mipidsi/Display.c b/ports/espressif/common-hal/mipidsi/Display.c index ea204924fe6..9ebaec1bd35 100644 --- a/ports/espressif/common-hal/mipidsi/Display.c +++ b/ports/espressif/common-hal/mipidsi/Display.c @@ -13,6 +13,7 @@ #include "bindings/espidf/__init__.h" #include #include +#include #include "py/runtime.h" // Cache write-back function (should be from rom/cache.h but it's not always available) @@ -49,6 +50,7 @@ void common_hal_mipidsi_display_construct(mipidsi_display_obj_t *self, self->framebuffer = NULL; self->dbi_io_handle = NULL; self->dpi_panel_handle = NULL; + self->backlight_inout.base.type = &mp_type_NoneType; // Create the DBI interface for sending commands esp_lcd_dbi_io_config_t dbi_config = { @@ -111,6 +113,10 @@ void common_hal_mipidsi_display_construct(mipidsi_display_obj_t *self, self->framebuffer = (uint8_t *)fb; self->framebuffer_size = width * height * (color_depth / 8); + // The panel's reset line is not managed here, matching the other displayio + // display drivers: callers pulse/hold reset themselves (e.g. with + // digitalio) when a board needs it. + // Send initialization sequence (format matches busdisplay) #define DELAY 0x80 uint32_t i = 0; @@ -141,8 +147,12 @@ void common_hal_mipidsi_display_construct(mipidsi_display_obj_t *self, CHECK_ESP_RESULT(ret); } + // Send display-ON (0x29) after the DPI/video stream is running, matching + // the reference BSP flow order. + esp_lcd_panel_io_tx_param(self->dbi_io_handle, 0x29, NULL, 0); + common_hal_time_delay_ms(20); + // Setup backlight PWM if pin is provided - self->backlight_inout.base.type = &mp_type_NoneType; if (backlight_pin != NULL && common_hal_mcu_pin_is_free(backlight_pin)) { #if (CIRCUITPY_PWMIO) pwmout_result_t result = common_hal_pwmio_pwmout_construct(&self->backlight_pwm, backlight_pin, 0, 50000, false); @@ -192,6 +202,7 @@ void common_hal_mipidsi_display_deinit(mipidsi_display_obj_t *self) { common_hal_digitalio_digitalinout_deinit(&self->backlight_inout); } + // Delete the DPI panel if (self->dpi_panel_handle != NULL) { esp_lcd_panel_del(self->dpi_panel_handle); diff --git a/ports/espressif/mpconfigport.mk b/ports/espressif/mpconfigport.mk index 24760821290..6373fd82e35 100644 --- a/ports/espressif/mpconfigport.mk +++ b/ports/espressif/mpconfigport.mk @@ -402,6 +402,13 @@ CIRCUITPY_ESPCAMERA = 0 # P4 has MIPI-DSI CIRCUITPY_MIPIDSI = 1 +# P4 has MIPI-CSI +CIRCUITPY_MIPICSI = 1 + +# P4 can play MIPI-CSI video into MIPI-DSI framebuffers +CIRCUITPY_VIDEOCORE = 1 + + #### esp32s31 ######################################################### else ifeq ($(IDF_TARGET),esp32s31) diff --git a/ports/espressif/supervisor/port.c b/ports/espressif/supervisor/port.c index 91fe8cf5a2e..81eb4714db6 100644 --- a/ports/espressif/supervisor/port.c +++ b/ports/espressif/supervisor/port.c @@ -343,9 +343,11 @@ void *port_malloc(size_t size, bool dma_capable) { } void *ptr = NULL; - // Try SPIRAM first if available. + // Try SPIRAM first if available. Aligned to the GC block size so heap + // areas (and with 64-byte GC blocks, every GC allocation in them) are + // cache-line aligned, which video DMA destinations require. #ifdef CONFIG_SPIRAM - ptr = heap_caps_malloc(size, MALLOC_CAP_8BIT | MALLOC_CAP_SPIRAM); + ptr = heap_caps_aligned_alloc(MICROPY_BYTES_PER_GC_BLOCK, size, MALLOC_CAP_8BIT | MALLOC_CAP_SPIRAM); #endif if (ptr == NULL) { ptr = heap_caps_malloc(size, MALLOC_CAP_8BIT); diff --git a/py/circuitpy_defns.mk b/py/circuitpy_defns.mk index ec84d3ec381..8a4b9c59a18 100755 --- a/py/circuitpy_defns.mk +++ b/py/circuitpy_defns.mk @@ -324,6 +324,9 @@ endif ifeq ($(CIRCUITPY_MICROCONTROLLER),1) SRC_PATTERNS += microcontroller/% endif +ifeq ($(CIRCUITPY_MIPICSI),1) +SRC_PATTERNS += mipicsi/% +endif ifeq ($(CIRCUITPY_MIPIDSI),1) SRC_PATTERNS += mipidsi/% endif @@ -492,6 +495,9 @@ endif ifeq ($(CIRCUITPY_VECTORIO),1) SRC_PATTERNS += vectorio/% endif +ifeq ($(CIRCUITPY_VIDEOCORE),1) +SRC_PATTERNS += videocore/% +endif ifeq ($(CIRCUITPY_WARNINGS),1) SRC_PATTERNS += warnings/% endif @@ -586,6 +592,8 @@ SRC_COMMON_HAL_ALL = \ mdns/__init__.c \ mdns/Server.c \ mdns/RemoteService.c \ + mipicsi/__init__.c \ + mipicsi/Camera.c \ mipidsi/Bus.c \ mipidsi/Display.c \ mipidsi/__init__.c \ @@ -690,6 +698,8 @@ $(filter $(SRC_PATTERNS), \ imagecapture/ParallelImageCapture.c \ locale/__init__.c \ math/__init__.c \ + mipicsi/BayerOrder.c \ + mipicsi/PixelFormat.c \ microcontroller/ResetReason.c \ microcontroller/RunMode.c \ msgpack/__init__.c \ @@ -895,6 +905,8 @@ SRC_SHARED_MODULE_ALL = \ vectorio/Rectangle.c \ vectorio/VectorShape.c \ vectorio/__init__.c \ + videocore/VideoBuffer.c \ + videocore/__init__.c \ warnings/__init__.c \ watchdog/__init__.c \ zlib/__init__.c \ diff --git a/py/circuitpy_mpconfig.mk b/py/circuitpy_mpconfig.mk index 0fc82d03704..1d8bedebbc5 100755 --- a/py/circuitpy_mpconfig.mk +++ b/py/circuitpy_mpconfig.mk @@ -469,6 +469,9 @@ CFLAGS += -DCIRCUITPY_MEMORYMONITOR=$(CIRCUITPY_MEMORYMONITOR) CIRCUITPY_MICROCONTROLLER ?= 1 CFLAGS += -DCIRCUITPY_MICROCONTROLLER=$(CIRCUITPY_MICROCONTROLLER) +CIRCUITPY_MIPICSI ?= 0 +CFLAGS += -DCIRCUITPY_MIPICSI=$(CIRCUITPY_MIPICSI) + CIRCUITPY_MIPIDSI ?= 0 CFLAGS += -DCIRCUITPY_MIPIDSI=$(CIRCUITPY_MIPIDSI) @@ -828,6 +831,12 @@ CFLAGS += -DCIRCUITPY_ULAB=$(CIRCUITPY_ULAB) # element-wise array arithmetic roughly 1.5x slower. CIRCUITPY_ULAB_OPTIMIZE_SIZE ?= 0 +# Video pipeline: playing video sources into framebuffers. Requires a port +# with a video source such as mipicsi and a framebuffer output such as +# mipidsi. +CIRCUITPY_VIDEOCORE ?= 0 +CFLAGS += -DCIRCUITPY_VIDEOCORE=$(CIRCUITPY_VIDEOCORE) + CIRCUITPY_WARNINGS ?= $(CIRCUITPY_FULL_BUILD) CFLAGS += -DCIRCUITPY_WARNINGS=$(CIRCUITPY_WARNINGS) diff --git a/shared-bindings/adafruit_pixelbuf/PixelBuf.c b/shared-bindings/adafruit_pixelbuf/PixelBuf.c index 4144f086b98..dd5e4d7b4ea 100644 --- a/shared-bindings/adafruit_pixelbuf/PixelBuf.c +++ b/shared-bindings/adafruit_pixelbuf/PixelBuf.c @@ -31,7 +31,13 @@ static MP_NORETURN void invalid_byteorder(void) { static void parse_byteorder(mp_obj_t byteorder_obj, pixelbuf_byteorder_details_t *parsed); //| class PixelBuf: -//| """A fast RGB[W] pixel buffer for LED and similar devices.""" +//| """A fast RGB[W] pixel buffer for LED and similar devices. +//| +//| Subclasses must define ``_transmit(self, buffer)``, which `show` calls +//| with a `bytearray` of the bytes to send: the header, the pixel +//| values, and the trailer. ``_transmit`` is internal, for PixelBuf's +//| use only; user code shows pixels through `show` and `auto_write`. +//| """ //| //| def __init__( //| self, diff --git a/shared-bindings/busdisplay/BusDisplay.c b/shared-bindings/busdisplay/BusDisplay.c index 0bf171a5d08..57fc1afa71c 100644 --- a/shared-bindings/busdisplay/BusDisplay.c +++ b/shared-bindings/busdisplay/BusDisplay.c @@ -16,6 +16,9 @@ #include "shared-bindings/displayio/Group.h" #include "shared-bindings/microcontroller/Pin.h" #include "shared-bindings/util.h" +#if CIRCUITPY_VIDEOCORE +#include "shared-module/videocore/__init__.h" +#endif #include "shared-module/displayio/__init__.h" //| import displayio @@ -408,6 +411,11 @@ static mp_obj_t busdisplay_busdisplay_obj_set_root_group(mp_obj_t self_in, mp_ob displayio_group_t *group = NULL; if (group_in != mp_const_none) { group = MP_OBJ_TO_PTR(native_group(group_in)); + #if CIRCUITPY_VIDEOCORE + // Validate that draw protocol layers (e.g. a VideoBuffer) fit the + // display they will render on. + videocore_validate_root_group(group_in, self_in); + #endif } common_hal_busdisplay_busdisplay_set_root_group(self, group); diff --git a/shared-bindings/framebufferio/FramebufferDisplay.c b/shared-bindings/framebufferio/FramebufferDisplay.c index 71ce18a1b4b..da7f18e4caf 100644 --- a/shared-bindings/framebufferio/FramebufferDisplay.c +++ b/shared-bindings/framebufferio/FramebufferDisplay.c @@ -17,6 +17,9 @@ #include "shared-bindings/displayio/Group.h" #include "shared-bindings/microcontroller/Pin.h" #include "shared-bindings/util.h" +#if CIRCUITPY_VIDEOCORE +#include "shared-module/videocore/__init__.h" +#endif #include "shared-module/displayio/__init__.h" //| class FramebufferDisplay: @@ -325,6 +328,11 @@ static mp_obj_t framebufferio_framebufferdisplay_obj_set_root_group(mp_obj_t sel displayio_group_t *group = NULL; if (group_in != mp_const_none) { group = MP_OBJ_TO_PTR(native_group(group_in)); + #if CIRCUITPY_VIDEOCORE + // Validate that draw protocol layers (e.g. a VideoBuffer) fit the + // display they will render on. + videocore_validate_root_group(group_in, self_in); + #endif } bool ok = common_hal_framebufferio_framebufferdisplay_set_root_group(self, group); diff --git a/shared-bindings/mipicsi/BayerOrder.c b/shared-bindings/mipicsi/BayerOrder.c new file mode 100644 index 00000000000..9f1058dd3e9 --- /dev/null +++ b/shared-bindings/mipicsi/BayerOrder.c @@ -0,0 +1,50 @@ +// This file is part of the CircuitPython project: https://circuitpython.org +// +// SPDX-FileCopyrightText: Copyright (c) 2026 Scott Shawcroft for Adafruit Industries +// +// SPDX-License-Identifier: MIT + +#include + +#include "py/enum.h" +#include "py/obj.h" +#include "py/runtime.h" + +#include "shared-bindings/mipicsi/__init__.h" + +//| class BayerOrder: +//| """The Bayer element order of a sensor's RAW output, in the sequence a 2x2 +//| pixel block transmits (top-left first, reading order)""" +//| +//| BGGR: BayerOrder +//| """The first element is Blue, then Green, then the next row starts with Green, then Red""" +//| +//| GBRG: BayerOrder +//| """The first element is Green, then Blue, then the next row starts with Red, then Green""" +//| +//| GRBG: BayerOrder +//| """The first element is Green, then Red, then the next row starts with Blue, then Green""" +//| +//| RGGB: BayerOrder +//| """The first element is Red, then Green, then the next row starts with Green, then Blue""" +//| + +MAKE_ENUM_VALUE(mipicsi_bayer_order_type, bayer_order, BGGR, MIPICSI_BAYER_ORDER_BGGR); +MAKE_ENUM_VALUE(mipicsi_bayer_order_type, bayer_order, GBRG, MIPICSI_BAYER_ORDER_GBRG); +MAKE_ENUM_VALUE(mipicsi_bayer_order_type, bayer_order, GRBG, MIPICSI_BAYER_ORDER_GRBG); +MAKE_ENUM_VALUE(mipicsi_bayer_order_type, bayer_order, RGGB, MIPICSI_BAYER_ORDER_RGGB); + +MAKE_ENUM_MAP(mipicsi_bayer_order) { + MAKE_ENUM_MAP_ENTRY(bayer_order, BGGR), + MAKE_ENUM_MAP_ENTRY(bayer_order, GBRG), + MAKE_ENUM_MAP_ENTRY(bayer_order, GRBG), + MAKE_ENUM_MAP_ENTRY(bayer_order, RGGB), +}; + +static MP_DEFINE_CONST_DICT(mipicsi_bayer_order_locals_dict, mipicsi_bayer_order_locals_table); +MAKE_PRINTER(mipicsi, mipicsi_bayer_order); +MAKE_ENUM_TYPE(mipicsi, BayerOrder, mipicsi_bayer_order); + +mipicsi_bayer_order_t validate_bayer_order(mp_obj_t obj, qstr arg_name) { + return cp_enum_value(&mipicsi_bayer_order_type, obj, arg_name); +} diff --git a/shared-bindings/mipicsi/Camera.c b/shared-bindings/mipicsi/Camera.c new file mode 100644 index 00000000000..7e711ca1ed7 --- /dev/null +++ b/shared-bindings/mipicsi/Camera.c @@ -0,0 +1,231 @@ +// This file is part of the CircuitPython project: https://circuitpython.org +// +// SPDX-FileCopyrightText: Copyright (c) 2025 Jeff Epler for Adafruit Industries +// +// SPDX-License-Identifier: MIT + +#include + +#include "py/mphal.h" +#include "py/obj.h" +#include "py/objproperty.h" +#include "py/runtime.h" + +#include "shared-bindings/displayio/Bitmap.h" +#include "shared-bindings/mipicsi/__init__.h" +#include "shared-bindings/mipicsi/Camera.h" +#include "shared-bindings/util.h" +#include "shared/runtime/context_manager_helpers.h" + +#if CIRCUITPY_VIDEOCORE +#include "shared-module/videocore/__init__.h" +#endif + +//| class Camera: +//| """Receive frames from a camera sensor on a MIPI CSI-2 interface +//| +//| The camera sensor is configured separately (typically over I2C) to +//| transmit frames. The MIPI CSI-2 receiver hardware has dedicated pins, +//| so no pins are configured here. +//| +//| Pass the configured sensor to the constructor; the receiver matches +//| itself to the sensor's current mode and starts it streaming. The +//| sensor object must provide the properties and methods of +//| ``circuitpython_typing.CameraSensor``: ``width``, ``height``, +//| ``pixel_format``, ``lane_bit_rate_mbps``, ``data_lanes`` and +//| ``bayer_order`` (a `BayerOrder`) properties plus ``_start()`` and +//| ``_stop()`` methods. Those two are internal: the camera calls them +//| itself as the stream starts and stops, so user code should not. +//| All are validated here (and again at play, so the sensor may change +//| its configuration freely while it isn't being played). +//| +//| The sensor decides the output image size, and the receiver is built +//| around it: a sensor cannot change its mode while frames are captured. +//| Sensors are started (``_start()``) when a `videocore.VideoBuffer` +//| starts playing the camera; they are never stopped by the camera, so +//| implementations should refuse to reconfigure (raise) while streaming +//| rather than change the output size or format mid-stream, stop through +//| their own finaliser, and have a new Camera made when the mode +//| changes. +//| +//| Camera objects implement the internal videosource protocol, so a +//| `videocore.VideoBuffer` can stream their frames into a framebuffer. +//| """ +//| +//| def __init__( +//| self, +//| sensor: circuitpython_typing.CameraSensor, +//| *, +//| output_format: Optional[PixelFormat] = None, +//| ) -> None: +//| """ +//| Configure and initialize the MIPI CSI-2 camera receiver +//| +//| .. code-block:: python +//| +//| sensor = sc2336.SC2336(i2c) +//| camera = mipicsi.Camera(sensor) +//| +//| :param sensor: A camera sensor to receive frames from. The receiver +//| is configured to match the sensor's current mode. The sensor +//| starts streaming when a `videocore.VideoBuffer` plays the +//| camera. +//| :param output_format: The format of the captured frames. When given and different from ``pixel_format``, the ISP converts each frame as it is captured (e.g. the sensor's RAW10 to `PixelFormat.RGB565`). Only RAW formats convert. Default: no conversion. +//| """ +//| +// Read the sensor's parameters off of the sensor object. A missing or +// invalid property fails here rather than later; mp_load_attr raises +// AttributeError for any property the sensor object doesn't provide. When +// pixel_format_override is not None it replaces the sensor's reported +// transmission format. +static mipicsi_pixel_format_t mipicsi_camera_load_sensor_params( + mp_obj_t sensor, + mipicsi_sensor_params_t *params) { + params->width = mp_arg_validate_int_min(mp_obj_get_int(mp_load_attr(sensor, MP_QSTR_width)), 1, MP_QSTR_width); + params->height = mp_arg_validate_int_min(mp_obj_get_int(mp_load_attr(sensor, MP_QSTR_height)), 1, MP_QSTR_height); + params->lane_bit_rate_mbps = mp_arg_validate_int_min(mp_obj_get_int(mp_load_attr(sensor, MP_QSTR_lane_bit_rate_mbps)), 1, MP_QSTR_lane_bit_rate_mbps); + params->data_lanes = mp_arg_validate_int_range(mp_obj_get_int(mp_load_attr(sensor, MP_QSTR_data_lanes)), 1, 2, MP_QSTR_data_lanes); + params->bayer_order = validate_bayer_order(mp_load_attr(sensor, MP_QSTR_bayer_order), MP_QSTR_bayer_order); + return validate_pixel_format(mp_load_attr(sensor, MP_QSTR_pixel_format), MP_QSTR_pixel_format); +} + +static mp_obj_t mipicsi_camera_make_new(const mp_obj_type_t *type, size_t n_args, size_t n_kw, const mp_obj_t *all_args) { + mp_arg_check_num(n_args, n_kw, 1, 1, true); + mp_obj_t sensor = all_args[0]; + + enum { ARG_output_format, NUM_ARGS }; + static const mp_arg_t allowed_args[] = { + { MP_QSTR_output_format, MP_ARG_OBJ | MP_ARG_KW_ONLY, { .u_obj = MP_ROM_NONE } }, + }; + mp_arg_val_t args[MP_ARRAY_SIZE(allowed_args)]; + MP_STATIC_ASSERT(MP_ARRAY_SIZE(allowed_args) == NUM_ARGS); + mp_arg_parse_all_kw_array(0, n_kw, all_args + 1, MP_ARRAY_SIZE(allowed_args), allowed_args, args); + + // All of the sensor's parameters (including the transmission format) are + // read off of the sensor object now, so a missing or invalid property + // fails here rather than at play time. They are read again at play, + // since the sensor is free to reconfigure between streams. + mipicsi_sensor_params_t params; + mipicsi_pixel_format_t pixel_format = mipicsi_camera_load_sensor_params(sensor, ¶ms); + mipicsi_pixel_format_t output_format = pixel_format; + if (args[ARG_output_format].u_obj != mp_const_none) { + output_format = validate_pixel_format(args[ARG_output_format].u_obj, MP_QSTR_output_format); + } + + // Validate the sensor contract methods (_start/_stop) exist now too, + // since the camera drives them from C as the stream starts and stops. + mp_obj_t validated[2]; + mp_load_method(sensor, MP_QSTR__start, validated); + mp_load_method(sensor, MP_QSTR__stop, validated); + + mipicsi_camera_obj_t *self = mp_obj_malloc_with_finaliser(mipicsi_camera_obj_t, &mipicsi_camera_type); + common_hal_mipicsi_camera_construct(self, output_format); + // Keep the sensor alive while receiving from it. It is only started + // streaming when a videocore.VideoBuffer starts playing (see the + // videosource play step): until then nothing captures or transmits. + common_hal_mipicsi_camera_set_sensor(self, sensor); + return MP_OBJ_FROM_PTR(self); +} + +//| def deinit(self) -> None: +//| """Deinitialises the camera receiver and releases all memory resources for reuse.""" +//| ... +//| +static mp_obj_t mipicsi_camera_deinit(mp_obj_t self_in) { + mipicsi_camera_obj_t *self = MP_OBJ_TO_PTR(self_in); + common_hal_mipicsi_camera_deinit(self); + return mp_const_none; +} +static MP_DEFINE_CONST_FUN_OBJ_1(mipicsi_camera_deinit_obj, mipicsi_camera_deinit); + +//| def __enter__(self) -> Camera: +//| """No-op used by Context Managers.""" +//| ... +//| +//| def __exit__(self) -> None: +//| """Automatically deinitializes the hardware when exiting a context. See +//| :ref:`lifetime-and-contextmanagers` for more info.""" +//| ... +//| +//| + +static const mp_rom_map_elem_t mipicsi_camera_locals_dict_table[] = { + { MP_ROM_QSTR(MP_QSTR___del__), MP_ROM_PTR(&mipicsi_camera_deinit_obj) }, + { MP_ROM_QSTR(MP_QSTR___enter__), MP_ROM_PTR(&default___enter___obj) }, + { MP_ROM_QSTR(MP_QSTR___exit__), MP_ROM_PTR(&default___exit___obj) }, + { MP_ROM_QSTR(MP_QSTR_deinit), MP_ROM_PTR(&mipicsi_camera_deinit_obj) }, +}; + +static MP_DEFINE_CONST_DICT(mipicsi_camera_locals_dict, mipicsi_camera_locals_dict_table); + +#if CIRCUITPY_VIDEOCORE +// Implementation of the videocore videosource protocol so a VideoBuffer can +// tell the Camera where to put frames. +static void mipicsi_camera_play(mp_obj_t self_in) { + mipicsi_camera_obj_t *self = MP_OBJ_TO_PTR(mp_obj_cast_to_native_base(self_in, &mipicsi_camera_type)); + // The camera always has a sensor from make_new on. deinit() clears it + // (the sensor stops through its own finaliser), so a missing sensor only + // means the camera was deinitialised. + if (self->sensor == mp_const_none) { + raise_deinited_error(); + } + // Re-read the sensor's mode now: it is free to reconfigure between + // streams, and only a mode change while streaming is prevented (the + // sensor refuses it itself). + mipicsi_sensor_params_t params; + self->pixel_format = mipicsi_camera_load_sensor_params(self->sensor, ¶ms); + common_hal_mipicsi_camera_videosource_play(self, ¶ms); +} + +static void mipicsi_camera_stop(mp_obj_t self_in) { + mipicsi_camera_obj_t *self = MP_OBJ_TO_PTR(mp_obj_cast_to_native_base(self_in, &mipicsi_camera_type)); + common_hal_mipicsi_camera_videosource_stop(self); +} + +static bool mipicsi_camera_set_destination(mp_obj_t self_in, const videocore_destination_t *dest, + videocore_on_frame_fun on_frame, void *ctx) { + mipicsi_camera_obj_t *self = MP_OBJ_TO_PTR(mp_obj_cast_to_native_base(self_in, &mipicsi_camera_type)); + // The protocol has no stop value on the destination: stopping is stop()'s + // job. + return common_hal_mipicsi_camera_set_destination(self, dest->base, dest->stride, on_frame, ctx); +} + +static mp_int_t mipicsi_camera_get_width(mp_obj_t self_in) { + return common_hal_mipicsi_camera_get_width(mp_obj_cast_to_native_base(self_in, &mipicsi_camera_type)); +} + +static mp_int_t mipicsi_camera_get_height(mp_obj_t self_in) { + return common_hal_mipicsi_camera_get_height(mp_obj_cast_to_native_base(self_in, &mipicsi_camera_type)); +} + +static mp_int_t mipicsi_camera_get_bits_per_pixel(mp_obj_t self_in) { + return common_hal_mipicsi_camera_get_bits_per_pixel(mp_obj_cast_to_native_base(self_in, &mipicsi_camera_type)); +} + +static const videosource_p_t mipicsi_camera_videosource_protocol = { + MP_PROTO_IMPLEMENT(MP_QSTR_protocol_videosource) + .play = mipicsi_camera_play, + .stop = mipicsi_camera_stop, + .set_destination = mipicsi_camera_set_destination, + .get_width = mipicsi_camera_get_width, + .get_height = mipicsi_camera_get_height, + .get_bits_per_pixel = mipicsi_camera_get_bits_per_pixel, +}; + +MP_DEFINE_CONST_OBJ_TYPE( + mipicsi_camera_type, + MP_QSTR_Camera, + MP_TYPE_FLAG_HAS_SPECIAL_ACCESSORS, + make_new, mipicsi_camera_make_new, + locals_dict, &mipicsi_camera_locals_dict, + protocol, &mipicsi_camera_videosource_protocol + ); +#else +MP_DEFINE_CONST_OBJ_TYPE( + mipicsi_camera_type, + MP_QSTR_Camera, + MP_TYPE_FLAG_HAS_SPECIAL_ACCESSORS, + make_new, mipicsi_camera_make_new, + locals_dict, &mipicsi_camera_locals_dict + ); +#endif diff --git a/shared-bindings/mipicsi/Camera.h b/shared-bindings/mipicsi/Camera.h new file mode 100644 index 00000000000..4df6c52b083 --- /dev/null +++ b/shared-bindings/mipicsi/Camera.h @@ -0,0 +1,60 @@ +// This file is part of the CircuitPython project: https://circuitpython.org +// +// SPDX-FileCopyrightText: Copyright (c) 2025 Jeff Epler for Adafruit Industries +// +// SPDX-License-Identifier: MIT + +#pragma once + +#include "py/obj.h" + +#include "common-hal/mipicsi/Camera.h" +#include "shared-bindings/mipicsi/__init__.h" + +extern const mp_obj_type_t mipicsi_camera_type; + +// Record the output format (possibly the sensor's). No hardware is created +// here: that happens in play(). +void common_hal_mipicsi_camera_construct( + mipicsi_camera_obj_t *self, + mipicsi_pixel_format_t output_format); + +// Remember the sensor configured to transmit to this receiver, if any. The +// receiver reads the sensor's mode from it at play time; the sensor stops +// itself (its finaliser does), never the receiver. +void common_hal_mipicsi_camera_set_sensor(mipicsi_camera_obj_t *self, mp_obj_t sensor); + +// Videosource protocol play step: create the receiver hardware for the +// sensor's current mode and start the sensor streaming. Not a user facing +// method: it is driven by the videosource protocol (from VideoBuffer.play), +// so it may raise. +void common_hal_mipicsi_camera_videosource_play( + mipicsi_camera_obj_t *self, + const mipicsi_sensor_params_t *params); + +// Videosource protocol stop step: stop the stream and the sensor. Not a user +// facing method: it is driven by the videosource protocol (from +// VideoBuffer.stop), so it may raise. +void common_hal_mipicsi_camera_videosource_stop(mipicsi_camera_obj_t *self); + +void common_hal_mipicsi_camera_deinit(mipicsi_camera_obj_t *self); +bool common_hal_mipicsi_camera_deinited(mipicsi_camera_obj_t *self); + +// Stream frames directly into base row after row. base must lead to a +// cache-line-aligned, contiguous DMA region: when base itself isn't aligned, +// the destination is not accepted. The destination must hold the whole +// (padded) frame back to back: the caller validates the fit, since the DMA +// has no stride. Returns whether those conditions hold (false instead of +// raising; the caller may ignore the result). +bool common_hal_mipicsi_camera_set_destination( + mipicsi_camera_obj_t *self, + uint8_t *base, + size_t stride, + void (*on_frame)(void *ctx), + void *ctx); + +mipicsi_pixel_format_t common_hal_mipicsi_camera_get_pixel_format(mipicsi_camera_obj_t *self); +mipicsi_pixel_format_t common_hal_mipicsi_camera_get_output_format(mipicsi_camera_obj_t *self); +mp_int_t common_hal_mipicsi_camera_get_width(mipicsi_camera_obj_t *self); +mp_int_t common_hal_mipicsi_camera_get_height(mipicsi_camera_obj_t *self); +mp_int_t common_hal_mipicsi_camera_get_bits_per_pixel(mipicsi_camera_obj_t *self); diff --git a/shared-bindings/mipicsi/PixelFormat.c b/shared-bindings/mipicsi/PixelFormat.c new file mode 100644 index 00000000000..84340019745 --- /dev/null +++ b/shared-bindings/mipicsi/PixelFormat.c @@ -0,0 +1,66 @@ +// This file is part of the CircuitPython project: https://circuitpython.org +// +// SPDX-FileCopyrightText: Copyright (c) 2025 Jeff Epler for Adafruit Industries +// +// SPDX-License-Identifier: MIT + +#include + +#include "py/enum.h" +#include "py/obj.h" +#include "py/runtime.h" + +#include "shared-bindings/mipicsi/__init__.h" + +//| class PixelFormat: +//| """Format of the data in the captured frames. The camera sensor must be +//| configured (typically over I2C) to transmit the corresponding MIPI CSI-2 +//| data type.""" +//| +//| RGB565: PixelFormat +//| """A 16-bit format with 5 bits of Red, 6 bits of Green, and 5 bits of Blue (MIPI data type ``RGB565``)""" +//| +//| RGB888: PixelFormat +//| """A 24-bit format with 8 bits of Red, Green and Blue (MIPI data type ``RGB888``)""" +//| +//| GRAYSCALE: PixelFormat +//| """An 8-bit format with 8 bits of luminance (MIPI data types for 8-bit grayscale)""" +//| +//| YUV422: PixelFormat +//| """A 16-bit YUV 4:2:2 format, in the byte order the sensor transmits (MIPI data type ``YUV422 8-bit``)""" +//| +//| RAW8: PixelFormat +//| """8 bits per pixel of raw Bayer data (MIPI data type ``RAW8``)""" +//| +//| RAW10: PixelFormat +//| """10 bits per pixel of raw Bayer data (MIPI data type ``RAW10``)""" +//| +//| RAW12: PixelFormat +//| """12 bits per pixel of raw Bayer data (MIPI data type ``RAW12``)""" +//| + +MAKE_ENUM_VALUE(mipicsi_pixel_format_type, pixel_format, RGB565, MIPICSI_FORMAT_RGB565); +MAKE_ENUM_VALUE(mipicsi_pixel_format_type, pixel_format, RGB888, MIPICSI_FORMAT_RGB888); +MAKE_ENUM_VALUE(mipicsi_pixel_format_type, pixel_format, GRAYSCALE, MIPICSI_FORMAT_GRAYSCALE); +MAKE_ENUM_VALUE(mipicsi_pixel_format_type, pixel_format, YUV422, MIPICSI_FORMAT_YUV422); +MAKE_ENUM_VALUE(mipicsi_pixel_format_type, pixel_format, RAW8, MIPICSI_FORMAT_RAW8); +MAKE_ENUM_VALUE(mipicsi_pixel_format_type, pixel_format, RAW10, MIPICSI_FORMAT_RAW10); +MAKE_ENUM_VALUE(mipicsi_pixel_format_type, pixel_format, RAW12, MIPICSI_FORMAT_RAW12); + +MAKE_ENUM_MAP(mipicsi_pixel_format) { + MAKE_ENUM_MAP_ENTRY(pixel_format, RGB565), + MAKE_ENUM_MAP_ENTRY(pixel_format, RGB888), + MAKE_ENUM_MAP_ENTRY(pixel_format, GRAYSCALE), + MAKE_ENUM_MAP_ENTRY(pixel_format, YUV422), + MAKE_ENUM_MAP_ENTRY(pixel_format, RAW8), + MAKE_ENUM_MAP_ENTRY(pixel_format, RAW10), + MAKE_ENUM_MAP_ENTRY(pixel_format, RAW12), +}; + +static MP_DEFINE_CONST_DICT(mipicsi_pixel_format_locals_dict, mipicsi_pixel_format_locals_table); +MAKE_PRINTER(mipicsi, mipicsi_pixel_format); +MAKE_ENUM_TYPE(mipicsi, PixelFormat, mipicsi_pixel_format); + +mipicsi_pixel_format_t validate_pixel_format(mp_obj_t obj, qstr arg_name) { + return cp_enum_value(&mipicsi_pixel_format_type, obj, arg_name); +} diff --git a/shared-bindings/mipicsi/__init__.c b/shared-bindings/mipicsi/__init__.c new file mode 100644 index 00000000000..d46e0eedf1d --- /dev/null +++ b/shared-bindings/mipicsi/__init__.c @@ -0,0 +1,46 @@ +// This file is part of the CircuitPython project: https://circuitpython.org +// +// SPDX-FileCopyrightText: Copyright (c) 2025 Jeff Epler for Adafruit Industries +// +// SPDX-License-Identifier: MIT + +#include + +#include "py/enum.h" +#include "py/obj.h" +#include "py/runtime.h" + +#include "shared-bindings/mipicsi/__init__.h" +#include "shared-bindings/mipicsi/Camera.h" + +//| """Support for MIPI CSI-2 camera interfaces +//| +//| Captures image frames from camera sensors attached to a MIPI CSI-2 +//| receiver, such as the one in the ESP32-P4. The camera sensor itself is +//| configured separately (typically over I2C); this module only receives +//| the image data the sensor transmits. +//| +//| The image data arrives in the format the sensor transmits, described by +//| `PixelFormat`. RAW formats carry Bayer data, whose element order is +//| described by `BayerOrder`. For instance, a sensor transmitting RGB565 +//| data yields frames of RGB565 pixels. Optionally, `mipicsi.Camera` can +//| convert frames with the ISP processor (set ``output_format`` and +//| ``pixel_format``), for example from the sensor's RAW output to RGB565 for +//| display. +//| """ + +static const mp_rom_map_elem_t mipicsi_module_globals_table[] = { + { MP_ROM_QSTR(MP_QSTR___name__), MP_ROM_QSTR(MP_QSTR_mipicsi) }, + { MP_ROM_QSTR(MP_QSTR_PixelFormat), MP_ROM_PTR(&mipicsi_pixel_format_type) }, + { MP_ROM_QSTR(MP_QSTR_BayerOrder), MP_ROM_PTR(&mipicsi_bayer_order_type) }, + { MP_ROM_QSTR(MP_QSTR_Camera), MP_ROM_PTR(&mipicsi_camera_type) }, +}; + +static MP_DEFINE_CONST_DICT(mipicsi_module_globals, mipicsi_module_globals_table); + +const mp_obj_module_t mipicsi_module = { + .base = { &mp_type_module }, + .globals = (mp_obj_dict_t *)&mipicsi_module_globals, +}; + +MP_REGISTER_MODULE(MP_QSTR_mipicsi, mipicsi_module); diff --git a/shared-bindings/mipicsi/__init__.h b/shared-bindings/mipicsi/__init__.h new file mode 100644 index 00000000000..5892379ce88 --- /dev/null +++ b/shared-bindings/mipicsi/__init__.h @@ -0,0 +1,33 @@ +// This file is part of the CircuitPython project: https://circuitpython.org +// +// SPDX-FileCopyrightText: Copyright (c) 2025 Jeff Epler for Adafruit Industries +// +// SPDX-License-Identifier: MIT + +#pragma once + +#include "py/enum.h" + +extern const mp_obj_type_t mipicsi_pixel_format_type; +extern const mp_obj_type_t mipicsi_bayer_order_type; +extern const cp_enum_obj_t pixel_format_RGB565_obj; + +typedef enum { + MIPICSI_FORMAT_RAW8, + MIPICSI_FORMAT_RAW10, + MIPICSI_FORMAT_RAW12, + MIPICSI_FORMAT_RGB565, + MIPICSI_FORMAT_RGB888, + MIPICSI_FORMAT_YUV422, + MIPICSI_FORMAT_GRAYSCALE, +} mipicsi_pixel_format_t; + +typedef enum { + MIPICSI_BAYER_ORDER_BGGR, + MIPICSI_BAYER_ORDER_GBRG, + MIPICSI_BAYER_ORDER_GRBG, + MIPICSI_BAYER_ORDER_RGGB, +} mipicsi_bayer_order_t; + +mipicsi_pixel_format_t validate_pixel_format(mp_obj_t obj, qstr arg_name); +mipicsi_bayer_order_t validate_bayer_order(mp_obj_t obj, qstr arg_name); diff --git a/shared-bindings/vectorio/__init__.h b/shared-bindings/vectorio/__init__.h index c5dcc026567..cc9b523ecb1 100644 --- a/shared-bindings/vectorio/__init__.h +++ b/shared-bindings/vectorio/__init__.h @@ -23,6 +23,9 @@ typedef bool (*draw_get_dirty_area_fun)(mp_obj_t draw_protocol_self, displayio_a typedef void (*draw_update_transform_fun)(mp_obj_t draw_protocol_self, displayio_buffer_transform_t *group_transform); typedef void (*draw_finish_refresh_fun)(mp_obj_t draw_protocol_self); typedef void (*draw_set_dirty_fun)(mp_obj_t draw_protocol_self); +// Optional: propagate a group's hidden state to the layer. May be NULL; the +// group then falls back to draw_set_dirty. +typedef void (*draw_set_hidden_fun)(mp_obj_t draw_protocol_self, bool hidden); typedef bool (*draw_set_in_group_fun)(mp_obj_t draw_protocol_self, bool in_group); typedef displayio_area_t *(*draw_get_refresh_areas_fun)(mp_obj_t draw_protocol_self, displayio_area_t *tail); @@ -33,6 +36,8 @@ typedef struct _vectorio_draw_protocol_impl_t { draw_finish_refresh_fun draw_finish_refresh; draw_get_refresh_areas_fun draw_get_refresh_areas; draw_set_dirty_fun draw_set_dirty; + // May be NULL. + draw_set_hidden_fun draw_set_hidden; draw_set_in_group_fun draw_set_in_group; } vectorio_draw_protocol_impl_t; diff --git a/shared-bindings/videocore/VideoBuffer.c b/shared-bindings/videocore/VideoBuffer.c new file mode 100644 index 00000000000..a6dcbfe87a4 --- /dev/null +++ b/shared-bindings/videocore/VideoBuffer.c @@ -0,0 +1,241 @@ +// This file is part of the CircuitPython project: https://circuitpython.org +// +// SPDX-FileCopyrightText: Copyright (c) 2026 Scott Shawcroft for Adafruit Industries +// +// SPDX-License-Identifier: MIT + +#include "py/obj.h" +#include "py/objproperty.h" +#include "py/runtime.h" + +#include "shared-bindings/videocore/VideoBuffer.h" +#include "shared-bindings/util.h" +#include "shared/runtime/context_manager_helpers.h" + +//| class VideoBuffer: +//| """Stream a video source into a framebuffer or buffer. +//| +//| A VideoBuffer can be added to displayio.Groups to show on a display +//| but may have size and position limitations. +//| +//| A VideoBuffer connects a video source (an object implementing the +//| videosource protocol, such as `mipicsi.Camera`) to a target: either a +//| framebuffer (an object implementing the framebuffer protocol, such as +//| `mipidsi.Display`) or a writable buffer such as a ``bytearray`` or +//| ``memoryview``. It is the video counterpart of `audioio.AudioOut`. +//| +//| The VideoBuffer tells the source where to put frames and moves the +//| destination as needed; the source streams continuously into it. While +//| playing, the target always holds the newest frame. +//| +//| When the target is a framebuffer, position the video with ``x`` and +//| ``y``: they may be changed at any time, including while playing, which +//| moves where the source puts its next frame. Some implementations may +//| restrict size and position. Setting position directly will raise an +//| error but indirectly through Group may result in weird output. For +//| example, on ESP32-P4 a VideoBuffer written into the framebuffer +//| directly must be full width because the ESP can't handle a stride +//| without an intermediate buffer. +//| +//| When the target is a plain buffer, frames are written compactly (row +//| stride equals ``width * bits_per_pixel / 8``) and can be read from +//| Python at any time — for example to process or save camera pixels. +//| +//| Two complete examples are in the `videocore` module overview: one +//| streaming a camera directly into a display, and one capturing a +//| 320x240 window into a ``bytearray`` that displayio places in a Group. +//| """ +//| +//| def __init__( +//| self, +//| framebuffer: Union[ +//| circuitpython_typing.FrameBuffer, circuitpython_typing.WriteableBuffer +//| ], +//| ) -> None: +//| """Create a VideoBuffer that plays video into the given framebuffer +//| +//| :param framebuffer: The framebuffer to copy video frames into, or a +//| `framebufferio.FramebufferDisplay` wrapping one, or a writable +//| buffer such as a ``bytearray`` or ``memoryview`` to read the raw +//| pixels from +//| """ +//| +static mp_obj_t videocore_videobuffer_make_new(const mp_obj_type_t *type, size_t n_args, size_t n_kw, const mp_obj_t *all_args) { + enum { ARG_framebuffer, NUM_ARGS }; + static const mp_arg_t allowed_args[] = { + { MP_QSTR_framebuffer, MP_ARG_REQUIRED | MP_ARG_OBJ }, + }; + + mp_arg_val_t args[MP_ARRAY_SIZE(allowed_args)]; + MP_STATIC_ASSERT(MP_ARRAY_SIZE(allowed_args) == NUM_ARGS); + mp_arg_parse_all_kw_array(n_args, n_kw, all_args, MP_ARRAY_SIZE(allowed_args), allowed_args, args); + + videocore_videobuffer_obj_t *self = mp_obj_malloc_with_finaliser(videocore_videobuffer_obj_t, &videocore_videobuffer_type); + common_hal_videocore_videobuffer_construct(self, args[ARG_framebuffer].u_obj); + return MP_OBJ_FROM_PTR(self); +} + +//| def play(self, source: circuitpython_typing.VideoSource) -> None: +//| """Begin streaming the given video source into the target. +//| +//| The source streams continuously, writing each new frame where the +//| VideoBuffer tells it to, replacing the previous one. The source's +//| frame format must be byte aligned. When the target is a +//| framebuffer, its bits per pixel must match the framebuffer color +//| depth and the whole frame must fit within the framebuffer at the +//| current ``x``/``y`` position (rows can't wrap past its end). When +//| the target is a plain buffer, it must be large enough for a whole +//| frame. Buffer size is validated here; the +//| source's size and format are then fixed for the duration of the +//| play. Sources that change them require ``stop`` and a new +//| ``play``. +//| +//| :param ~circuitpython_typing.VideoSource source: The video source to +//| play, such as `mipicsi.Camera` +//| """ +//| +static mp_obj_t videocore_videobuffer_play(mp_obj_t self_in, mp_obj_t source) { + videocore_videobuffer_obj_t *self = MP_OBJ_TO_PTR(self_in); + common_hal_videocore_videobuffer_play(self, source); + return mp_const_none; +} +static MP_DEFINE_CONST_FUN_OBJ_2(videocore_videobuffer_play_obj, videocore_videobuffer_play); + +//| def stop(self) -> None: +//| """Stop playing the video source.""" +//| ... +//| +static mp_obj_t videocore_videobuffer_stop(mp_obj_t self_in) { + videocore_videobuffer_obj_t *self = MP_OBJ_TO_PTR(self_in); + common_hal_videocore_videobuffer_stop(self); + return mp_const_none; +} +static MP_DEFINE_CONST_FUN_OBJ_1(videocore_videobuffer_stop_obj, videocore_videobuffer_stop); + +//| playing: bool +//| """True when a video source is being played, False otherwise. (read-only)""" +//| +static mp_obj_t videocore_videobuffer_get_playing(mp_obj_t self_in) { + videocore_videobuffer_obj_t *self = MP_OBJ_TO_PTR(self_in); + return mp_obj_new_bool(common_hal_videocore_videobuffer_get_playing(self)); +} +static MP_DEFINE_CONST_FUN_OBJ_1(videocore_videobuffer_get_playing_obj, videocore_videobuffer_get_playing); +MP_PROPERTY_GETTER(videocore_videobuffer_playing_obj, + (mp_obj_t)&videocore_videobuffer_get_playing_obj); + +//| source: Optional[circuitpython_typing.VideoSource] +//| """The video source being played, or None when not playing. (read-only)""" +//| +static mp_obj_t videocore_videobuffer_get_source(mp_obj_t self_in) { + videocore_videobuffer_obj_t *self = MP_OBJ_TO_PTR(self_in); + return common_hal_videocore_videobuffer_get_source(self); +} +static MP_DEFINE_CONST_FUN_OBJ_1(videocore_videobuffer_get_source_obj, videocore_videobuffer_get_source); +MP_PROPERTY_GETTER(videocore_videobuffer_source_obj, + (mp_obj_t)&videocore_videobuffer_get_source_obj); + +//| x: int +//| """The x position of the video within the display. Framebuffer targets +//| must be at x == 0 (video sources DMA whole rows): ``ValueError`` +//| otherwise. Plain buffer targets only use it to place the video in a +//| Group: it doesn't change where frames land within the buffer.""" +//| +//| y: int +//| """The y position of the video within the display.""" +//| +//| +static void videocore_videobuffer_get_position(mp_obj_t self_in, mp_int_t *x, mp_int_t *y) { + videocore_videobuffer_obj_t *self = MP_OBJ_TO_PTR(self_in); + *x = self->x; + *y = self->y; +} + +static mp_obj_t videocore_videobuffer_get_x(mp_obj_t self_in) { + mp_int_t x, y; + videocore_videobuffer_get_position(self_in, &x, &y); + return MP_OBJ_NEW_SMALL_INT(x); +} +static MP_DEFINE_CONST_FUN_OBJ_1(videocore_videobuffer_get_x_obj, videocore_videobuffer_get_x); + +static mp_obj_t videocore_videobuffer_set_x(mp_obj_t self_in, mp_obj_t value) { + videocore_videobuffer_obj_t *self = MP_OBJ_TO_PTR(self_in); + mp_int_t x = mp_obj_get_int(value); + common_hal_videocore_videobuffer_set_position(self, x, self->y); + return mp_const_none; +} +static MP_DEFINE_CONST_FUN_OBJ_2(videocore_videobuffer_set_x_obj, videocore_videobuffer_set_x); +MP_PROPERTY_GETSET(videocore_videobuffer_x_obj, + (mp_obj_t)&videocore_videobuffer_get_x_obj, + (mp_obj_t)&videocore_videobuffer_set_x_obj); + +static mp_obj_t videocore_videobuffer_get_y(mp_obj_t self_in) { + mp_int_t x, y; + videocore_videobuffer_get_position(self_in, &x, &y); + return MP_OBJ_NEW_SMALL_INT(y); +} +static MP_DEFINE_CONST_FUN_OBJ_1(videocore_videobuffer_get_y_obj, videocore_videobuffer_get_y); + +static mp_obj_t videocore_videobuffer_set_y(mp_obj_t self_in, mp_obj_t value) { + videocore_videobuffer_obj_t *self = MP_OBJ_TO_PTR(self_in); + mp_int_t y = mp_obj_get_int(value); + common_hal_videocore_videobuffer_set_position(self, self->x, y); + return mp_const_none; +} +static MP_DEFINE_CONST_FUN_OBJ_2(videocore_videobuffer_set_y_obj, videocore_videobuffer_set_y); +MP_PROPERTY_GETSET(videocore_videobuffer_y_obj, + (mp_obj_t)&videocore_videobuffer_get_y_obj, + (mp_obj_t)&videocore_videobuffer_set_y_obj); + +//| def deinit(self) -> None: +//| """Stop playing. The video source stops itself.""" +//| ... +//| +//| def __enter__(self) -> VideoBuffer: +//| """No-op used by Context Managers.""" +//| ... +//| +//| def __exit__(self) -> None: +//| """Automatically deinitializes the hardware when exiting a context. See +//| :ref:`lifetime-and-contextmanagers` for more info.""" +//| ... +//| +static mp_obj_t videocore_videobuffer_deinit(mp_obj_t self_in) { + videocore_videobuffer_obj_t *self = MP_OBJ_TO_PTR(self_in); + common_hal_videocore_videobuffer_stop(self); + return mp_const_none; +} +static MP_DEFINE_CONST_FUN_OBJ_1(videocore_videobuffer_deinit_obj, videocore_videobuffer_deinit); + +// The finaliser must not raise (it runs during garbage collection), so stop +// without one instead of deinit(): a failed stop stays best-effort, matching +// the source's own finaliser. +static mp_obj_t videocore_videobuffer___del__(mp_obj_t self_in) { + videocore_videobuffer_obj_t *self = MP_OBJ_TO_PTR(self_in); + videocore_videobuffer_stop_quiet(self); + return mp_const_none; +} +static MP_DEFINE_CONST_FUN_OBJ_1(videocore_videobuffer___del___obj, videocore_videobuffer___del__); + +static const mp_rom_map_elem_t videocore_videobuffer_locals_dict_table[] = { + { MP_ROM_QSTR(MP_QSTR___del__), MP_ROM_PTR(&videocore_videobuffer___del___obj) }, + { MP_ROM_QSTR(MP_QSTR___enter__), MP_ROM_PTR(&default___enter___obj) }, + { MP_ROM_QSTR(MP_QSTR___exit__), MP_ROM_PTR(&default___exit___obj) }, + { MP_ROM_QSTR(MP_QSTR_deinit), MP_ROM_PTR(&videocore_videobuffer_deinit_obj) }, + { MP_ROM_QSTR(MP_QSTR_play), MP_ROM_PTR(&videocore_videobuffer_play_obj) }, + { MP_ROM_QSTR(MP_QSTR_stop), MP_ROM_PTR(&videocore_videobuffer_stop_obj) }, + { MP_ROM_QSTR(MP_QSTR_playing), MP_ROM_PTR(&videocore_videobuffer_playing_obj) }, + { MP_ROM_QSTR(MP_QSTR_source), MP_ROM_PTR(&videocore_videobuffer_source_obj) }, + { MP_ROM_QSTR(MP_QSTR_x), MP_ROM_PTR(&videocore_videobuffer_x_obj) }, + { MP_ROM_QSTR(MP_QSTR_y), MP_ROM_PTR(&videocore_videobuffer_y_obj) }, +}; + +static MP_DEFINE_CONST_DICT(videocore_videobuffer_locals_dict, videocore_videobuffer_locals_dict_table); + +MP_DEFINE_CONST_OBJ_TYPE( + videocore_videobuffer_type, + MP_QSTR_VideoBuffer, + MP_TYPE_FLAG_HAS_SPECIAL_ACCESSORS, + make_new, videocore_videobuffer_make_new, + locals_dict, &videocore_videobuffer_locals_dict, + protocol, &videocore_videobuffer_draw_protocol + ); diff --git a/shared-bindings/videocore/VideoBuffer.h b/shared-bindings/videocore/VideoBuffer.h new file mode 100644 index 00000000000..927169bc4b3 --- /dev/null +++ b/shared-bindings/videocore/VideoBuffer.h @@ -0,0 +1,11 @@ +// This file is part of the CircuitPython project: https://circuitpython.org +// +// SPDX-FileCopyrightText: Copyright (c) 2026 Scott Shawcroft for Adafruit Industries +// +// SPDX-License-Identifier: MIT + +#pragma once + +#include "shared-module/videocore/VideoBuffer.h" + +extern const mp_obj_type_t videocore_videobuffer_type; diff --git a/shared-bindings/videocore/__init__.c b/shared-bindings/videocore/__init__.c new file mode 100644 index 00000000000..057600deeb4 --- /dev/null +++ b/shared-bindings/videocore/__init__.c @@ -0,0 +1,86 @@ +// This file is part of the CircuitPython project: https://circuitpython.org +// +// SPDX-FileCopyrightText: Copyright (c) 2026 Scott Shawcroft for Adafruit Industries +// +// SPDX-License-Identifier: MIT + +#include "py/obj.h" +#include "py/runtime.h" + +#include "shared-bindings/videocore/VideoBuffer.h" +#include "shared-bindings/videocore/__init__.h" + +//| """Support for playing video sources +//| +//| This module provides the video pipeline shared between video sources and +//| video outputs. It is the video equivalent of the `audiocore` module. +//| +//| Video sources, such as `mipicsi.Camera`, implement the internal videosource +//| protocol. A `VideoBuffer` tells a video source where to put its frames in a +//| framebuffer, such as `mipidsi.Display`, and can move it as needed. +//| +//| Stream a MIPI CSI camera live into the board's display (a MIPI DSI panel +//| on ESP32-P4-Function-EV, for example): +//| +//| .. code-block:: python +//| +//| import board +//| import mipicsi +//| import sc2336 +//| import videocore +//| +//| display = board.DISPLAY() +//| sensor = sc2336.SC2336(board.I2C()) +//| sensor.mode = sc2336.Mode.HD_1280x720_RAW10_30FPS +//| # Trim the sensor output to the display size. +//| sensor.width = display.width +//| sensor.height = display.height +//| camera = mipicsi.Camera(sensor, output_format=mipicsi.PixelFormat.RGB565) +//| +//| video = videocore.VideoBuffer(display) +//| video.play(camera) # camera frames stream live into the panel +//| +//| Capture a smaller part of the sensor's image into a `bytearray` and let +//| displayio place it in a Group, together with other layers. displayio bulk +//| copies new frames into the group's area of the display (no color +//| conversion): +//| +//| .. code-block:: python +//| +//| import board +//| import displayio +//| import mipicsi +//| import sc2336 +//| import videocore +//| +//| sensor = sc2336.SC2336(board.I2C()) +//| sensor.mode = sc2336.Mode.VGA_640x480_RAW10_50FPS +//| # Trim the sensor output to a 320x240 window. +//| sensor.width = 320 +//| sensor.height = 240 +//| camera = mipicsi.Camera(sensor, output_format=mipicsi.PixelFormat.RGB565) +//| +//| capture = videocore.VideoBuffer(bytearray(320 * 240 * 2)) +//| capture.play(camera) +//| +//| root = displayio.Group() +//| root.append(capture) +//| capture.x = 350 # position of the live video within the group +//| capture.y = 180 +//| board.DISPLAY().root_group = root # live video joins the displayio scene +//| """ +//| + +static const mp_rom_map_elem_t videocore_module_globals_table[] = { + { MP_ROM_QSTR(MP_QSTR___name__), MP_ROM_QSTR(MP_QSTR_videocore) }, + { MP_ROM_QSTR(MP_QSTR_VideoBuffer), MP_ROM_PTR(&videocore_videobuffer_type) }, +}; + +static MP_DEFINE_CONST_DICT(videocore_module_globals, videocore_module_globals_table); + +const mp_obj_module_t videocore_module = { + .base = { &mp_type_module }, + .globals = (mp_obj_dict_t *)&videocore_module_globals, +}; + +MP_REGISTER_MODULE(MP_QSTR_videocore, videocore_module); diff --git a/shared-bindings/videocore/__init__.h b/shared-bindings/videocore/__init__.h new file mode 100644 index 00000000000..d309fcc838b --- /dev/null +++ b/shared-bindings/videocore/__init__.h @@ -0,0 +1,9 @@ +// This file is part of the CircuitPython project: https://circuitpython.org +// +// SPDX-FileCopyrightText: Copyright (c) 2026 Scott Shawcroft for Adafruit Industries +// +// SPDX-License-Identifier: MIT + +#pragma once + +#include "shared-module/videocore/__init__.h" diff --git a/shared-module/displayio/Group.c b/shared-module/displayio/Group.c index 84ebffdab6d..1e84da278f6 100644 --- a/shared-module/displayio/Group.c +++ b/shared-module/displayio/Group.c @@ -10,7 +10,7 @@ #include "py/objlist.h" #include "shared-bindings/displayio/TileGrid.h" -#if CIRCUITPY_VECTORIO +#if CIRCUITPY_VECTORIO || CIRCUITPY_VIDEOCORE #include "shared-bindings/vectorio/VectorShape.h" #endif @@ -52,11 +52,15 @@ void common_hal_displayio_group_set_hidden(displayio_group_t *self, bool hidden) displayio_group_set_hidden_by_parent(layer, hidden); continue; } - #if CIRCUITPY_VECTORIO + #if CIRCUITPY_VECTORIO || CIRCUITPY_VIDEOCORE const vectorio_draw_protocol_t *draw_protocol = mp_proto_get(MP_QSTR_protocol_draw, self->members->items[i]); if (draw_protocol != NULL) { layer = draw_protocol->draw_get_protocol_self(self->members->items[i]); - draw_protocol->draw_protocol_impl->draw_set_dirty(layer); + if (draw_protocol->draw_protocol_impl->draw_set_hidden != NULL) { + draw_protocol->draw_protocol_impl->draw_set_hidden(layer, hidden); + } else { + draw_protocol->draw_protocol_impl->draw_set_dirty(layer); + } continue; } #endif @@ -87,11 +91,15 @@ void displayio_group_set_hidden_by_parent(displayio_group_t *self, bool hidden) displayio_group_set_hidden_by_parent(layer, hidden); continue; } - #if CIRCUITPY_VECTORIO + #if CIRCUITPY_VECTORIO || CIRCUITPY_VIDEOCORE const vectorio_draw_protocol_t *draw_protocol = mp_proto_get(MP_QSTR_protocol_draw, self->members->items[i]); if (draw_protocol != NULL) { layer = draw_protocol->draw_get_protocol_self(self->members->items[i]); - draw_protocol->draw_protocol_impl->draw_set_dirty(layer); + if (draw_protocol->draw_protocol_impl->draw_set_hidden != NULL) { + draw_protocol->draw_protocol_impl->draw_set_hidden(layer, hidden); + } else { + draw_protocol->draw_protocol_impl->draw_set_dirty(layer); + } continue; } #endif @@ -146,7 +154,7 @@ static void _update_child_transforms(displayio_group_t *self) { } for (size_t i = 0; i < self->members->len; i++) { mp_obj_t layer; - #if CIRCUITPY_VECTORIO + #if CIRCUITPY_VECTORIO || CIRCUITPY_VIDEOCORE const vectorio_draw_protocol_t *draw_protocol = mp_proto_get(MP_QSTR_protocol_draw, self->members->items[i]); if (draw_protocol != NULL) { layer = draw_protocol->draw_get_protocol_self(self->members->items[i]); @@ -248,7 +256,7 @@ void common_hal_displayio_group_set_y(displayio_group_t *self, mp_int_t y) { static void _add_layer(displayio_group_t *self, mp_obj_t layer) { check_readonly(self); - #if CIRCUITPY_VECTORIO + #if CIRCUITPY_VECTORIO || CIRCUITPY_VIDEOCORE const vectorio_draw_protocol_t *draw_protocol = mp_proto_get(MP_QSTR_protocol_draw, layer); if (draw_protocol != NULL) { mp_obj_t protocol_self = draw_protocol->draw_get_protocol_self(layer); @@ -293,7 +301,7 @@ static void _remove_layer(displayio_group_t *self, size_t index) { mp_obj_t layer; displayio_area_t layer_area; bool rendered_last_frame = false; - #if CIRCUITPY_VECTORIO + #if CIRCUITPY_VECTORIO || CIRCUITPY_VIDEOCORE const vectorio_draw_protocol_t *draw_protocol = mp_proto_get(MP_QSTR_protocol_draw, self->members->items[index]); if (draw_protocol != NULL) { layer = draw_protocol->draw_get_protocol_self(self->members->items[index]); @@ -377,7 +385,7 @@ bool displayio_group_fill_area(displayio_group_t *self, const _displayio_colorsp if (self->hidden == false) { for (int32_t i = self->members->len - 1; i >= 0; i--) { mp_obj_t layer; - #if CIRCUITPY_VECTORIO + #if CIRCUITPY_VECTORIO || CIRCUITPY_VIDEOCORE const vectorio_draw_protocol_t *draw_protocol = mp_proto_get(MP_QSTR_protocol_draw, self->members->items[i]); if (draw_protocol != NULL) { layer = draw_protocol->draw_get_protocol_self(self->members->items[i]); @@ -412,7 +420,7 @@ void displayio_group_finish_refresh(displayio_group_t *self) { self->item_removed = false; for (int32_t i = self->members->len - 1; i >= 0; i--) { mp_obj_t layer; - #if CIRCUITPY_VECTORIO + #if CIRCUITPY_VECTORIO || CIRCUITPY_VIDEOCORE const vectorio_draw_protocol_t *draw_protocol = mp_proto_get(MP_QSTR_protocol_draw, self->members->items[i]); if (draw_protocol != NULL) { layer = draw_protocol->draw_get_protocol_self(self->members->items[i]); @@ -443,7 +451,7 @@ static displayio_area_t *group_get_refresh_areas_impl(displayio_group_t *self, d for (int32_t i = self->members->len - 1; i >= 0; i--) { mp_obj_t layer; - #if CIRCUITPY_VECTORIO + #if CIRCUITPY_VECTORIO || CIRCUITPY_VIDEOCORE const vectorio_draw_protocol_t *draw_protocol = mp_proto_get(MP_QSTR_protocol_draw, self->members->items[i]); if (draw_protocol != NULL) { layer = draw_protocol->draw_get_protocol_self(self->members->items[i]); diff --git a/shared-module/videocore/VideoBuffer.c b/shared-module/videocore/VideoBuffer.c new file mode 100644 index 00000000000..dafc942d8a1 --- /dev/null +++ b/shared-module/videocore/VideoBuffer.c @@ -0,0 +1,547 @@ +// This file is part of the CircuitPython project: https://circuitpython.org +// +// SPDX-FileCopyrightText: Copyright (c) 2026 Scott Shawcroft for Adafruit Industries +// +// SPDX-License-Identifier: MIT + +#include + +#include "shared-module/videocore/VideoBuffer.h" + +#include "py/runtime.h" + +#include "shared-bindings/displayio/Group.h" +#include "shared-bindings/videocore/VideoBuffer.h" +#include "shared-bindings/videocore/__init__.h" +#include "shared-module/displayio/Group.h" +#include "shared-module/videocore/__init__.h" + +#if CIRCUITPY_FRAMEBUFFERIO +#include "shared-bindings/framebufferio/FramebufferDisplay.h" +#endif + +#define FB_GETTER(method, default_value) \ + (self->framebuffer_protocol->method \ + ? self->framebuffer_protocol->method(self->framebuffer) \ + : (default_value)) + +// Called from interrupt context when the source lands a frame. The frame +// landed in stream_base, the buffer the accepted destination gave (set when +// the last set_destination returned true), so keep it tiny. +static void videocore_videobuffer_on_frame(void *ctx) { + videocore_videobuffer_obj_t *self = ctx; + self->frame_ready = true; +} + +// Tear down playback state: clear our fields. +static void videocore_videobuffer_clear(videocore_videobuffer_obj_t *self); + +// Recompute the rendered rect (screen coordinates) of the video from the +// x/y position and the absolute transform. Video only supports scale 1 with +// no mirroring (validated in update_transform), so the rect is the frame +// size translated into place. +static void videocore_videobuffer_update_area(videocore_videobuffer_obj_t *self) { + const displayio_buffer_transform_t *t = &self->absolute_transform; + self->current_area.x1 = self->x + t->x; + self->current_area.y1 = self->y + t->y; + self->current_area.x2 = self->current_area.x1 + self->source_width; + self->current_area.y2 = self->current_area.y1 + self->source_height; + self->current_area.next = NULL; +} + +// Compute the destination for the source in the current target, honoring the +// current x/y position and the absolute transform's translation (a video in +// a group). Raises unless the whole frame fits in the target: the camera +// DMA's whole, contiguous frames with no stride padding, so rows can't wrap +// past the buffer and there is no cropping yet. The caller validates the +// fit (this runs from Python contexts); framebuffers in groups also move +// their destination from draw contexts with clamping (see _move_dest). +static void videocore_videobuffer_compute_destination( + videocore_videobuffer_obj_t *self, videocore_destination_t *dest) { + const size_t bytes_per_pixel = self->source_bits_per_pixel / 8; + const size_t row_bytes = (size_t)self->source_width * bytes_per_pixel; + + if (self->framebuffer_protocol == NULL) { + // Raw buffer mode: frames are written compactly from the start. + dest->base = self->bufinfo.buf; + dest->stride = row_bytes; + dest->row_bytes = row_bytes; + dest->height = self->source_height; + if (self->bufinfo.len < self->source_frame_bytes) { + mp_raise_ValueError(MP_ERROR_TEXT("Buffer too small")); + } + return; + } + + mp_buffer_info_t bufinfo; + self->framebuffer_protocol->get_bufinfo(self->framebuffer, &bufinfo); + if (bufinfo.buf == NULL) { + mp_raise_ValueError(MP_ERROR_TEXT("framebuffer went away")); + } + + const mp_int_t fb_width = FB_GETTER(get_width, 0); + const mp_int_t row_stride = FB_GETTER(get_row_stride, fb_width * self->source_bits_per_pixel / 8); + const mp_int_t first_pixel_offset = FB_GETTER(get_first_pixel_offset, 0); + + // The absolute position includes a containing group's translation. The + // shift may land the frame's rows mid-row: the image won't look right, + // but the contiguous DMA stays contiguous. Only the frame's start and + // end must remain within the buffer. + const displayio_buffer_transform_t *t = &self->absolute_transform; + const mp_int_t shift_x = self->x + t->x; + const mp_int_t shift_y = self->y + t->y; + if (self->x < 0 || self->y < 0 || shift_x < 0 || shift_y < 0 || + first_pixel_offset + shift_y * row_stride + shift_x * bytes_per_pixel + + (mp_int_t)self->source_frame_bytes > + bufinfo.len) { + mp_raise_ValueError(MP_ERROR_TEXT("video doesn't fit in framebuffer")); + } + + dest->base = (uint8_t *)bufinfo.buf + first_pixel_offset + + shift_y * row_stride + shift_x * bytes_per_pixel; + dest->stride = row_stride; + dest->row_bytes = row_bytes; + dest->height = self->source_height; +} + +// Move the framebuffer destination for a new absolute position, capping the +// shift so the frame stays completely within the framebuffer. Best effort: +// never raises, because this runs from displayio's refresh. +static void videocore_videobuffer_move_dest( + videocore_videobuffer_obj_t *self, mp_int_t shift_x, mp_int_t shift_y) { + if (!self->playing) { + return; + } + mp_buffer_info_t bufinfo; + self->framebuffer_protocol->get_bufinfo(self->framebuffer, &bufinfo); + if (bufinfo.buf == NULL) { + return; + } + const size_t bytes_per_pixel = self->source_bits_per_pixel / 8; + const size_t row_bytes = (size_t)self->source_width * bytes_per_pixel; + const size_t frame_bytes = (size_t)self->source_height * row_bytes; + const mp_int_t fb_width = FB_GETTER(get_width, 0); + const mp_int_t row_stride = FB_GETTER(get_row_stride, fb_width * self->source_bits_per_pixel / 8); + const mp_int_t first_pixel_offset = FB_GETTER(get_first_pixel_offset, 0); + + if (shift_x < 0) { + shift_x = 0; + } + if (shift_y < 0) { + shift_y = 0; + } + // Cap the start and the end. Cap the rows first: enough rows must fit + // above the frame for it to fit at all. + const mp_int_t max_end = (mp_int_t)bufinfo.len - first_pixel_offset - (mp_int_t)frame_bytes; + if (shift_y > max_end / row_stride) { + shift_y = max_end / row_stride; + } + if (shift_y < 0) { + shift_y = 0; + } + const mp_int_t max_x = (max_end - shift_y * row_stride) / bytes_per_pixel; + if (shift_x > max_x) { + shift_x = max_x; + } + if (shift_x < 0) { + shift_x = 0; + } + + videocore_destination_t dest = { + .base = bufinfo.buf + first_pixel_offset + shift_y * row_stride + shift_x * bytes_per_pixel, + .stride = row_stride, + .row_bytes = row_bytes, + .height = self->source_height, + }; + // The result is ignored here: the source rejects a destination by + // returning false (e.g. a mid-row shift isn't DMA-aligned) and the + // stream continues where it was. on_frame points at the old destination + // until then, which keeps the last-known-good frame. + if (videosource_set_destination(self->source, &dest, videocore_videobuffer_on_frame, self)) { + self->stream_base = dest.base; + } +} + +void common_hal_videocore_videobuffer_construct(videocore_videobuffer_obj_t *self, mp_obj_t framebuffer_or_display) { + mp_obj_t framebuffer = framebuffer_or_display; + #if CIRCUITPY_FRAMEBUFFERIO + // Allow passing the display wrapping the framebuffer too. + mp_obj_t native_display = mp_obj_cast_to_native_base(framebuffer_or_display, &framebufferio_framebufferdisplay_type); + if (native_display != MP_OBJ_NULL) { + framebufferio_framebufferdisplay_obj_t *display = MP_OBJ_TO_PTR(native_display); + framebuffer = display->framebuffer; + } + #endif + self->framebuffer = framebuffer; + self->framebuffer_protocol = mp_proto_get(MP_QSTR_protocol_framebuffer, framebuffer); + if (self->framebuffer_protocol != NULL) { + if (self->framebuffer_protocol->get_bufinfo == NULL || + self->framebuffer_protocol->get_width == NULL || + self->framebuffer_protocol->get_height == NULL) { + mp_raise_ValueError(MP_ERROR_TEXT("framebuffer missing required functions")); + } + } else if (!mp_get_buffer(framebuffer, &self->bufinfo, MP_BUFFER_WRITE)) { + // Not a framebuffer and not a writable buffer such as a bytearray or + // memoryview. + mp_raise_TypeError_varg(MP_ERROR_TEXT("expected FrameBuffer or %q"), MP_QSTR_bytearray); + } + + self->source = mp_const_none; + self->stream_base = NULL; + self->x = 0; + self->y = 0; + self->frame_ready = false; + self->in_group = false; + self->hidden = false; + self->stale_size = false; + displayio_area_t empty = {}; + self->current_area = empty; + self->ephemeral_dirty_area = empty; + self->absolute_transform = null_transform; + self->playing = false; +} + +void common_hal_videocore_videobuffer_play(videocore_videobuffer_obj_t *self, mp_obj_t source) { + if (self->playing) { + mp_raise_RuntimeError(MP_ERROR_TEXT("already playing")); + } + const videosource_p_t *protocol = videosource_check(source); + + // Prepare and start the source first: it may reconfigure itself (e.g. a + // sensor picks its mode), which fixes what get_* report from here on. + videosource_play(source); + + mp_int_t bits_per_pixel = protocol->get_bits_per_pixel(source); + if (bits_per_pixel % 8 != 0) { + mp_raise_ValueError(MP_ERROR_TEXT("video format must be byte aligned")); + } + mp_int_t width = protocol->get_width(source); + mp_int_t height = protocol->get_height(source); + size_t frame_bytes = (size_t)((width * height * bits_per_pixel) / 8); + + if (self->framebuffer_protocol != NULL) { + mp_int_t color_depth = FB_GETTER(get_color_depth, 16); + if (color_depth % 8 != 0 || bits_per_pixel != color_depth) { + mp_raise_ValueError(MP_ERROR_TEXT("video format doesn't match framebuffer")); + } + } + + self->source = source; + self->source_width = width; + self->source_height = height; + self->source_bits_per_pixel = bits_per_pixel; + self->source_frame_bytes = frame_bytes; + + if (self->framebuffer_protocol == NULL && (mp_int_t)self->bufinfo.len < (mp_int_t)frame_bytes) { + mp_raise_ValueError(MP_ERROR_TEXT("Buffer too small")); + } + + videocore_destination_t dest; + videocore_videobuffer_compute_destination(self, &dest); + + // Tell the source where to put its pixels. The source streams from now + // on; on_frame tells us when a new frame has landed. It is the + // displayio group tree (and this object's Python references) that keep + // the VideoBuffer alive; a GC'd buffer's finaliser stops the stream. + if (!videosource_set_destination(source, &dest, videocore_videobuffer_on_frame, self)) { + mp_raise_ValueError(MP_ERROR_TEXT("video source rejected the destination")); + } + self->stream_base = dest.base; + self->playing = true; +} + +void common_hal_videocore_videobuffer_set_position(videocore_videobuffer_obj_t *self, mp_int_t x, mp_int_t y) { + if (self->framebuffer_protocol != NULL && x != 0) { + // The source DMAs whole, contiguous frames with no stride, so every + // frame row must land on a framebuffer row: the video's x must be 0. + mp_raise_ValueError(MP_ERROR_TEXT("video in a framebuffer must be at x == 0")); + } + self->ephemeral_dirty_area = self->current_area; + self->x = x; + self->y = y; + videocore_videobuffer_update_area(self); + if (!self->playing || self->framebuffer_protocol == NULL) { + // Raw buffer targets are rendered by displayio; their capture + // destination doesn't move with the position. + return; + } + // The source's frame size was fixed when play() was called. Reject + // sources that changed it (e.g. a sensor mode change) rather than + // blitting at stale geometry. + const videosource_p_t *protocol = videosource_check(self->source); + mp_int_t width = protocol->get_width(self->source); + mp_int_t height = protocol->get_height(self->source); + mp_int_t bits_per_pixel = protocol->get_bits_per_pixel(self->source); + if (width != self->source_width || height != self->source_height || + bits_per_pixel != self->source_bits_per_pixel) { + mp_raise_ValueError(MP_ERROR_TEXT("video size changed since play; stop and play again")); + } + // Move the destination between frames; the source picks up the change on + // its next frame. The result may be ignored here (the source rejects a + // destination by returning false; the stream continues where it was). + videocore_destination_t dest; + videocore_videobuffer_compute_destination(self, &dest); + (void)videosource_set_destination(self->source, &dest, videocore_videobuffer_on_frame, self); + self->stream_base = dest.base; +} + +static void videocore_videobuffer_clear(videocore_videobuffer_obj_t *self) { + self->playing = false; + self->frame_ready = false; + self->stream_base = NULL; + self->source = mp_const_none; +} + +void common_hal_videocore_videobuffer_stop(videocore_videobuffer_obj_t *self) { + // Stop the source; the camera also stops the sensor so its settings can + // be changed. That may raise: surface it, after cleaning up, since the + // stream did stop. + if (self->source != mp_const_none) { + nlr_buf_t nlr; + if (nlr_push(&nlr) == 0) { + videosource_stop(self->source); + nlr_pop(); + } else { + videocore_videobuffer_clear(self); + nlr_raise(nlr.ret_val); + } + } + videocore_videobuffer_clear(self); +} + +// Stop without raising: runs from background contexts (reload). A failed +// stop is best-effort; the source's own finaliser stops it as well. +void videocore_videobuffer_stop_quiet(videocore_videobuffer_obj_t *self) { + nlr_buf_t nlr; + if (nlr_push(&nlr) == 0) { + if (self->source != mp_const_none) { + videosource_stop(self->source); + } + nlr_pop(); + } + videocore_videobuffer_clear(self); +} + +bool common_hal_videocore_videobuffer_get_playing(videocore_videobuffer_obj_t *self) { + return self->playing; +} + +mp_obj_t common_hal_videocore_videobuffer_get_source(videocore_videobuffer_obj_t *self) { + return self->source; +} + +// Draw protocol implementation: a VideoBuffer holding a plain buffer gets +// bulk copied into the display by displayio's rendering; a framebuffer +// target's pixels are streamed into its framebuffer directly, so displayio's +// refresh only claims the video's area and pushes the dirty rows to the +// framebuffer (the fill is a no-op there). + +static mp_obj_t videocore_videobuffer_draw_protocol_self(mp_obj_t protocol_container) { + return protocol_container; +} + +static bool videocore_videobuffer_draw_fill_area(mp_obj_t self_in, const _displayio_colorspace_t *colorspace, const displayio_area_t *area, uint32_t *mask, uint32_t *buffer) { + videocore_videobuffer_obj_t *self = MP_OBJ_TO_PTR(self_in); + if (!self->playing || self->stream_base == NULL || self->hidden) { + return false; + } + + displayio_area_t overlap; + if (!displayio_area_compute_overlap(area, &self->current_area, &overlap)) { + return false; + } + const bool full_coverage = displayio_area_equal(area, &overlap); + + // Opaque: claim the pixels. + const uint16_t span_px = displayio_area_width(area); + for (int16_t y = overlap.y1; y < overlap.y2; y++) { + const size_t px = (size_t)(y - area->y1) * span_px + (overlap.x1 - area->x1); + for (size_t i = 0; i < displayio_area_width(&overlap); i++) { + mask[(px + i) / 32] |= 1u << ((px + i) % 32); + } + } + + if (self->framebuffer_protocol != NULL) { + // Framebuffer target: the source streams the pixels straight into + // its framebuffer, so the fill itself is a no-op. Claiming the area + // is what adds its rows to the display's refresh. + return full_coverage; + } + + // Raw buffer target: bulk copy, no color conversion: the display depth + // must match the video's pixel format byte for byte. + if (colorspace->depth != self->source_bits_per_pixel || colorspace->depth != 16) { + return false; + } + + const size_t bytes_per_pixel = self->source_bits_per_pixel / 8; + const size_t row_bytes = (size_t)self->source_width * bytes_per_pixel; + const uint8_t *frame = (const uint8_t *)self->stream_base; + + for (int16_t y = overlap.y1; y < overlap.y2; y++) { + const size_t px = (size_t)(y - area->y1) * span_px + (overlap.x1 - area->x1); + const int16_t fy = y - self->current_area.y1; + memcpy( + (uint16_t *)buffer + px, + frame + (size_t)fy * row_bytes + (size_t)(overlap.x1 - self->current_area.x1) * bytes_per_pixel, + (size_t)displayio_area_width(&overlap) * bytes_per_pixel); + } + return full_coverage; +} + +static bool videocore_videobuffer_draw_get_dirty_area(mp_obj_t self_in, displayio_area_t *out_area) { + videocore_videobuffer_obj_t *self = MP_OBJ_TO_PTR(self_in); + out_area->x1 = out_area->x2; + displayio_area_union(&self->ephemeral_dirty_area, &self->current_area, out_area); + return true; +} + +static void videocore_videobuffer_draw_update_transform(mp_obj_t self_in, displayio_buffer_transform_t *group_transform) { + videocore_videobuffer_obj_t *self = MP_OBJ_TO_PTR(self_in); + if (group_transform == NULL) { + self->absolute_transform = null_transform; + } else { + // Validate the transform: video frames are bulk copied (raw targets) + // or DMA'd contiguously (framebuffer targets), so only translation + // moves. + if (group_transform->scale != 0 && + (group_transform->scale != 1 || + group_transform->mirror_x || group_transform->mirror_y || + group_transform->transpose_xy || + group_transform->dx != 1 || group_transform->dy != 1)) { + mp_raise_ValueError(MP_ERROR_TEXT("VideoBuffer does not support transforms")); + } + self->absolute_transform = *group_transform; + } + videocore_videobuffer_update_area(self); + + // Framebuffer targets stream straight into the buffer: move the + // destination along with the shift. Best effort, since this can't raise: + // the shift may land the start mid-row, which won't look right but is + // obvious; the frame is capped to stay completely within the buffer. + if (self->framebuffer_protocol != NULL) { + videocore_videobuffer_move_dest( + self, + self->x + self->absolute_transform.x, + self->y + self->absolute_transform.y); + } +} + +static void videocore_videobuffer_draw_finish_refresh(mp_obj_t self_in) { + videocore_videobuffer_obj_t *self = MP_OBJ_TO_PTR(self_in); + // The frame (or move) rendered; stop flagging it. + self->frame_ready = false; + displayio_area_t empty = {}; + self->ephemeral_dirty_area = empty; +} + +static displayio_area_t *videocore_videobuffer_draw_get_refresh_areas(mp_obj_t self_in, displayio_area_t *tail) { + videocore_videobuffer_obj_t *self = MP_OBJ_TO_PTR(self_in); + if (!self->playing || self->hidden || self->stale_size) { + return tail; + } + if (!displayio_area_empty(&self->ephemeral_dirty_area)) { + // The video moved: repaint the vacated area with the lower layers. + self->ephemeral_dirty_area.next = tail; + tail = &self->ephemeral_dirty_area; + } + if (self->frame_ready) { + // New frame: bulk copy it in. + self->current_area.next = tail; + tail = &self->current_area; + } + return tail; +} + +static void videocore_videobuffer_draw_set_dirty(mp_obj_t self_in) { + (void)self_in; +} + +static void videocore_videobuffer_draw_set_hidden(mp_obj_t self_in, bool hidden) { + videocore_videobuffer_obj_t *self = MP_OBJ_TO_PTR(self_in); + if (self->hidden == hidden) { + return; + } + self->hidden = hidden; + // Repaint what the video showed or vacated. + self->ephemeral_dirty_area = self->current_area; + if (!hidden && self->playing) { + // The source is free to reconfigure while hidden. Validate its + // sizing again but quietly: if it no longer matches, don't raise; + // simply stop showing the stream until stop and play again. + const videosource_p_t *protocol = mp_proto_get(MP_QSTR_protocol_videosource, self->source); + if (protocol != NULL && + (protocol->get_width(self->source) != self->source_width || + protocol->get_height(self->source) != self->source_height || + protocol->get_bits_per_pixel(self->source) != self->source_bits_per_pixel)) { + self->stale_size = true; + } + } +} + +static bool videocore_videobuffer_draw_set_in_group(mp_obj_t self_in, bool in_group) { + videocore_videobuffer_obj_t *self = MP_OBJ_TO_PTR(self_in); + bool was_in_group = self->in_group; + self->in_group = in_group; + return was_in_group; +} + +static vectorio_draw_protocol_impl_t videocore_videobuffer_draw_protocol_impl = { + .draw_fill_area = videocore_videobuffer_draw_fill_area, + .draw_get_dirty_area = videocore_videobuffer_draw_get_dirty_area, + .draw_update_transform = videocore_videobuffer_draw_update_transform, + .draw_finish_refresh = videocore_videobuffer_draw_finish_refresh, + .draw_get_refresh_areas = videocore_videobuffer_draw_get_refresh_areas, + .draw_set_dirty = videocore_videobuffer_draw_set_dirty, + .draw_set_hidden = videocore_videobuffer_draw_set_hidden, + .draw_set_in_group = videocore_videobuffer_draw_set_in_group, +}; + +const vectorio_draw_protocol_t videocore_videobuffer_draw_protocol = { + MP_PROTO_IMPLEMENT(MP_QSTR_protocol_draw) + .draw_get_protocol_self = videocore_videobuffer_draw_protocol_self, + .draw_protocol_impl = &videocore_videobuffer_draw_protocol_impl, +}; + +// Root group validation: a VideoBuffer that streams straight into a +// framebuffer bypasses displayio rendering, so it may only be shown on a +// display that renders that same framebuffer. Plain buffer targets are drawn +// by displayio and are fine anywhere. Walks the group tree and raises when a +// VideoBuffer doesn't match the display it would render on. +static void videocore_validate_member(mp_obj_t member, mp_obj_t display) { + displayio_group_t *group = mp_obj_cast_to_native_base(member, &displayio_group_type); + if (group != MP_OBJ_NULL) { + for (size_t i = 0; i < group->members->len; i++) { + videocore_validate_member(group->members->items[i], display); + } + return; + } + const vectorio_draw_protocol_t *draw_protocol = mp_proto_get(MP_QSTR_protocol_draw, member); + if (draw_protocol == NULL) { + return; + } + mp_obj_t layer = draw_protocol->draw_get_protocol_self(member); + if (mp_obj_get_type(layer) != &videocore_videobuffer_type) { + return; + } + videocore_videobuffer_obj_t *self = MP_OBJ_TO_PTR(layer); + if (self->framebuffer_protocol == NULL || display == self->framebuffer) { + return; + } + #if CIRCUITPY_FRAMEBUFFERIO + // The display may wrap the framebuffer the video plays to (e.g. a + // framebufferio.FramebufferDisplay around the same object). + mp_obj_t native_display = mp_obj_cast_to_native_base(display, &framebufferio_framebufferdisplay_type); + if (native_display != MP_OBJ_NULL) { + framebufferio_framebufferdisplay_obj_t *fd = MP_OBJ_TO_PTR(native_display); + if (fd->framebuffer == self->framebuffer) { + return; + } + } + #endif + mp_raise_ValueError(MP_ERROR_TEXT("VideoBuffer is playing into a different framebuffer")); +} + +void videocore_validate_root_group(mp_obj_t root_group, mp_obj_t display) { + videocore_validate_member(root_group, display); +} diff --git a/shared-module/videocore/VideoBuffer.h b/shared-module/videocore/VideoBuffer.h new file mode 100644 index 00000000000..0837b34fdfb --- /dev/null +++ b/shared-module/videocore/VideoBuffer.h @@ -0,0 +1,73 @@ +// This file is part of the CircuitPython project: https://circuitpython.org +// +// SPDX-FileCopyrightText: Copyright (c) 2026 Scott Shawcroft for Adafruit Industries +// +// SPDX-License-Identifier: MIT + +#pragma once + +#include "py/obj.h" + +#include "shared-bindings/vectorio/__init__.h" // draw protocol +#include "shared-module/displayio/area.h" +#include "shared-module/framebufferio/FramebufferDisplay.h" // framebuffer protocol +#include "shared-module/videocore/__init__.h" + +typedef struct { + mp_obj_base_t base; + // Object frames are streamed into: an object implementing the framebuffer + // protocol, such as mipidsi.Display, or a writable buffer such as a + // bytearray or memoryview. + mp_obj_t framebuffer; + const framebuffer_p_t *framebuffer_protocol; + // Target buffer info in raw buffer mode (framebuffer_protocol == NULL). + mp_buffer_info_t bufinfo; + // The video source being played, or None. + mp_obj_t source; + // Destination position of the video within the target. + mp_int_t x; + mp_int_t y; + mp_int_t source_width; + mp_int_t source_height; + mp_int_t source_bits_per_pixel; + size_t source_frame_bytes; + // Where the source streams frames into: the requested destination, or a + // later, hardware-aligned position inside it. "Later in the buffer" only + // happens for plain-buffer targets; set by play() and the x/y + // properties. + const void *stream_base; + // Set from interrupt context when the source lands a frame in the + // destination; cleared by the refresh pass once the new frame has been + // rendered into the display. + volatile bool frame_ready; + // displayio draw protocol state: the rect currently rendered (in screen + // coordinates) and the rect that changed since the last refresh. + displayio_area_t current_area; + displayio_area_t ephemeral_dirty_area; + displayio_buffer_transform_t absolute_transform; + bool in_group; + // Set by the group when hidden; while hidden the video doesn't show and + // the source is free to reconfigure (it is revalidated on unhide). + bool hidden; + // True when the source's sizing no longer matches what play() captured: + // the video won't show until stop and play again. + bool stale_size; + bool playing; +} videocore_videobuffer_obj_t; + +void common_hal_videocore_videobuffer_construct(videocore_videobuffer_obj_t *self, mp_obj_t framebuffer_or_display); +void common_hal_videocore_videobuffer_play(videocore_videobuffer_obj_t *self, mp_obj_t source); +void common_hal_videocore_videobuffer_set_position(videocore_videobuffer_obj_t *self, mp_int_t x, mp_int_t y); +void common_hal_videocore_videobuffer_stop(videocore_videobuffer_obj_t *self); +// Stop without raising (from the finaliser). A failed stop is best-effort; +// the source's own finaliser stops it as well. +void videocore_videobuffer_stop_quiet(videocore_videobuffer_obj_t *self); +bool common_hal_videocore_videobuffer_get_playing(videocore_videobuffer_obj_t *self); +mp_obj_t common_hal_videocore_videobuffer_get_source(videocore_videobuffer_obj_t *self); + +// The draw protocol implementation that lets a VideoBuffer be added to a +// displayio Group: plain-buffer targets get bulk copied into the framebuffer +// by displayio's rendering; framebuffer targets claim their area (a no-op +// fill: the source streams their pixels straight into the framebuffer) so +// displayio's refresh pushes the video's rows to the framebuffer. +extern const vectorio_draw_protocol_t videocore_videobuffer_draw_protocol; diff --git a/shared-module/videocore/__init__.c b/shared-module/videocore/__init__.c new file mode 100644 index 00000000000..4bf32772cb1 --- /dev/null +++ b/shared-module/videocore/__init__.c @@ -0,0 +1,48 @@ +// This file is part of the CircuitPython project: https://circuitpython.org +// +// SPDX-FileCopyrightText: Copyright (c) 2026 Scott Shawcroft for Adafruit Industries +// +// SPDX-License-Identifier: MIT + +#include "shared-module/videocore/__init__.h" +#include "shared-module/videocore/VideoBuffer.h" + +// Playback, presentation and garbage collection are tied to the displayio +// group the VideoBuffer is in: displayio's refresh drives frame presentation +// through the draw protocol, and the group (or a Python reference) keeps the +// object alive; its finaliser stops the stream. + +const videosource_p_t *videosource_check(mp_obj_t obj) { + return (const videosource_p_t *)mp_proto_get_or_throw(MP_QSTR_protocol_videosource, obj); +} + +void videosource_play(mp_obj_t source) { + const videosource_p_t *protocol = videosource_check(source); + if (protocol->play != NULL) { + protocol->play(source); + } +} + +void videosource_stop(mp_obj_t source) { + const videosource_p_t *protocol = videosource_check(source); + if (protocol->stop != NULL) { + protocol->stop(source); + } +} + +bool videosource_set_destination(mp_obj_t source, const videocore_destination_t *dest, + videocore_on_frame_fun on_frame, void *ctx) { + return videosource_check(source)->set_destination(source, dest, on_frame, ctx); +} + +mp_int_t videosource_get_width(mp_obj_t source) { + return videosource_check(source)->get_width(source); +} + +mp_int_t videosource_get_height(mp_obj_t source) { + return videosource_check(source)->get_height(source); +} + +mp_int_t videosource_get_bits_per_pixel(mp_obj_t source) { + return videosource_check(source)->get_bits_per_pixel(source); +} diff --git a/shared-module/videocore/__init__.h b/shared-module/videocore/__init__.h new file mode 100644 index 00000000000..3db83d590d1 --- /dev/null +++ b/shared-module/videocore/__init__.h @@ -0,0 +1,96 @@ +// This file is part of the CircuitPython project: https://circuitpython.org +// +// SPDX-FileCopyrightText: Copyright (c) 2026 Scott Shawcroft for Adafruit Industries +// +// SPDX-License-Identifier: MIT + +#pragma once + +#include +#include +#include + +#include "py/obj.h" +#include "py/proto.h" + +// Destination for a video stream: where to put pixels, as a pointer to the +// first pixel and the destination row stride. row_bytes/height describe the +// source frame; the source assumes the destination holds the whole frame +// (the ESP CSI DMA is contiguous-only), so the rect is validated before +// streaming starts and when the position changes. +typedef struct { + // First pixel of the destination. + uint8_t *base; + // Destination row stride, in bytes. + size_t stride; + // Source frame row size, in bytes. + size_t row_bytes; + // Source frame height, in rows. + size_t height; +} videocore_destination_t; + +// Called (in interrupt context) when a frame has landed in the buffer +// set_destination returned. +typedef void (*videocore_on_frame_fun)(void *ctx); + +// Protocol implemented by video streamers such as mipicsi.Camera. The streams +// continuously; the target sets (and can change) where the pixels go by +// calling set_destination() at any time, including between frames. +// +// This is the video equivalent of the audiosample protocol in the audiocore +// module, inverted: instead of the target pulling frames, the source writes +// where it is told to. +typedef struct _videosource_p_t { + MP_PROTOCOL_HEAD // MP_QSTR_protocol_videosource + + // Create whatever streaming hardware the source needs and start it (e.g. + // start a sensor streaming). Called once before the first + // set_destination of a stream, from Python (VideoBuffer.play), so it may + // raise. Raises when the source is already streaming for someone else. + // May be NULL for sources that need no preparation. + void (*play)(mp_obj_t self); + + // Direct future frames into dest. The destination may be changed + // between frames; there is no NULL/stop value: stopping is stop()'s + // job. Never raises: returns whether the stream accepted the + // destination (the caller may ignore the result). on_frame(ctx) is + // called from interrupt context whenever a frame has landed in + // dest.base (the source never relocates frames elsewhere). + bool (*set_destination)(mp_obj_t self, const videocore_destination_t *dest, + videocore_on_frame_fun on_frame, void *ctx); + + // Stop streaming and release the streaming hardware (e.g. stop the + // sensor so its settings can be changed). Called once per stream, from + // Python (VideoBuffer.stop), so it may raise; exceptions there are + // clearer than implicit stops. Only called when play() succeeded. + void (*stop)(mp_obj_t self); + + // Width of the frames the source streams, in pixels. + mp_int_t (*get_width)(mp_obj_t self); + // Height of the frames the source streams, in lines. + mp_int_t (*get_height)(mp_obj_t self); + // Bits per pixel of the frames the source streams. + mp_int_t (*get_bits_per_pixel)(mp_obj_t self); +} videosource_p_t; + +// Protocol function helpers. videosource_play/stop are no-ops for sources +// whose protocol doesn't implement them. +void videosource_play(mp_obj_t source); +void videosource_stop(mp_obj_t source); + +// Return the videosource protocol of obj or raise TypeError. +const videosource_p_t *videosource_check(mp_obj_t obj); + +// Dispatch helpers mirroring the audiosample_* functions. videosource_play +// is a no-op for sources whose protocol has no play. +bool videosource_set_destination(mp_obj_t source, const videocore_destination_t *dest, + videocore_on_frame_fun on_frame, void *ctx); +mp_int_t videosource_get_width(mp_obj_t source); +mp_int_t videosource_get_height(mp_obj_t source); +mp_int_t videosource_get_bits_per_pixel(mp_obj_t source); + +// Raise when the group tree contains a VideoBuffer that streams into a +// framebuffer other than the one this display renders. Called when a group +// is made a display's root group. Implemented in shared-module/videocore/ +// VideoBuffer.c. +void videocore_validate_root_group(mp_obj_t root_group, mp_obj_t display);