From e6b0448ff238571550012a94c889c7341a585dd0 Mon Sep 17 00:00:00 2001 From: Mark Date: Thu, 29 Oct 2026 09:05:43 -0700 Subject: [PATCH 1/3] hexagon: Generalization of CONV_2D implementation for CONV_3D - New op HTP_OP_CONV_3D is wired through htp-ops.h, main.c (shares op_conv_2d) and the op remap in htp-kparams.cpp. - conv_3d runs on the existing HMX tiled (implicit GEMM) conv2d path. KD and KH are folded into one virtual kernel row (KD*KH), and output rows are counted as N*OD*OH. - htp_kparams_solve_conv3d checks the conv_3d layout and calls the tiled solver htp_kparams_conv2d_tiled, which takes an is3d flag. - The tiled kernel takes the sizes and strides that differ between conv_2d and conv_3d from a struct htp_conv_geom. Only the input stage unfolds depth. - 256-entry DMA queue can no longer overflow Assisted-by: Claude Code --- ggml/src/ggml-hexagon/htp-kparams.cpp | 78 ++++++++++++++-- ggml/src/ggml-hexagon/htp/conv2d-ops.c | 123 ++++++++++++++++++------- ggml/src/ggml-hexagon/htp/htp-ops.h | 1 + ggml/src/ggml-hexagon/htp/main.c | 1 + 4 files changed, 161 insertions(+), 42 deletions(-) diff --git a/ggml/src/ggml-hexagon/htp-kparams.cpp b/ggml/src/ggml-hexagon/htp-kparams.cpp index 822d1eb08105..bc9dac4c37a8 100644 --- a/ggml/src/ggml-hexagon/htp-kparams.cpp +++ b/ggml/src/ggml-hexagon/htp-kparams.cpp @@ -62,6 +62,7 @@ htp_op_code htp_op_remap(const struct ggml_tensor * t) { case GGML_OP_IM2COL_3D: return HTP_OP_IM2COL_3D; case GGML_OP_CONV_2D: return HTP_OP_CONV_2D; case GGML_OP_CONV_2D_DW: return HTP_OP_CONV_2D_DW; + case GGML_OP_CONV_3D: return HTP_OP_CONV_3D; case GGML_OP_ROLL: return HTP_OP_ROLL; case GGML_OP_UPSCALE: return HTP_OP_UPSCALE; case GGML_OP_LIGHTNING_INDEXER: return HTP_OP_LIGHTNING_INDEXER; @@ -3994,16 +3995,17 @@ static bool htp_kparams_conv2d_tiled( const struct ggml_tensor * src0 = dst->src[0]; const struct ggml_tensor * src1 = dst->src[1]; + const bool is3d = dst->op == GGML_OP_CONV_3D; const int32_t s0 = ggml_get_op_params_i32(dst, 0); - const int32_t d0 = ggml_get_op_params_i32(dst, 4); + const int32_t d0 = ggml_get_op_params_i32(dst, is3d ? 6 : 4); const uint32_t KW = src0->ne[0]; - const uint32_t KH = src0->ne[1]; - const uint32_t Cin = src0->ne[2]; - const uint32_t Cout = src0->ne[3]; + const uint32_t KH = is3d ? src0->ne[2] * src0->ne[1] : src0->ne[1]; + const uint32_t Cin = is3d ? ggml_get_op_params_i32(dst, 9) : src0->ne[2]; + const uint32_t Cout = is3d ? ggml_get_op_params_i32(dst, 11) : src0->ne[3]; const uint32_t OW = dst->ne[0]; - const uint32_t OH = dst->ne[1]; - const uint32_t N = dst->ne[3]; + const uint32_t OH = is3d ? dst->ne[2] * dst->ne[1] : dst->ne[1]; + const uint32_t N = is3d ? ggml_get_op_params_i32(dst, 10) : dst->ne[3]; const size_t es = ggml_type_size(src0->type); if (s0 != 1 || dst->nb[0] != sizeof(float)) { @@ -4012,10 +4014,10 @@ static bool htp_kparams_conv2d_tiled( // kernel k order must match the staged input: (cin, kh, kw) for WHCN input, (kh, kw, cin) for CWHN input. // Kernel is either k-contiguous rows per cout (WHCN only) or cout-contiguous with a uniform k stride. - const bool whcn = src1->nb[0] == sizeof(float); + const bool whcn = is3d || src1->nb[0] == sizeof(float); const bool cwhn = !whcn && src1->nb[2] == sizeof(float) && src1->nb[0] == Cin * sizeof(float); const bool co_c = Cout == 1 || src0->nb[3] == es; - const bool w_rows = whcn && src0->nb[0] == es && (KH == 1 || src0->nb[1] == KW * es) && (Cin == 1 || src0->nb[2] == KW * KH * es); + const bool w_rows = is3d || (whcn && src0->nb[0] == es && (KH == 1 || src0->nb[1] == KW * es) && (Cin == 1 || src0->nb[2] == KW * KH * es)); const size_t ks = whcn ? src0->nb[0] : src0->nb[2]; bool kmajor = false; if (whcn) { @@ -4041,6 +4043,10 @@ static bool htp_kparams_conv2d_tiled( const uint32_t nth = htp_session_n_threads(sess); const size_t tile = 2048; + if (KH > 255) { + return false; + } + uint32_t rt_block = n_rt; uint32_t ct = (std::min)(n_ct, 64u); uint32_t seg = (std::min)(8u, n_owt); @@ -4085,7 +4091,7 @@ static bool htp_kparams_conv2d_tiled( const size_t thr2 = hex_round_up((std::max)(2 * stage + ostage + prow, wpack), tile); struct htp_conv2d_vtcm_layout L2; htp_conv2d_vtcm_layout_build(&L2, rt_block, w_size, act_size, out_size, thr2, nth); - if (stage <= 128 * 1024 && L2.total_bytes <= htp_session_vtcm_size(sess)) { + if (stage <= 128 * 1024 && 2 * KH <= 255 && L2.total_bytes <= htp_session_vtcm_size(sess)) { n_stage = 2; thr_size = thr2; total = L2.total_bytes; @@ -4337,6 +4343,57 @@ static bool htp_kparams_solve_conv2d_dw( return true; } +static bool htp_kparams_solve_conv3d( + const struct ggml_hexagon_session * sess, + const struct ggml_tensor * op, + struct htp_conv2d_kernel_params * kparams +) { + memset(kparams, 0, sizeof(*kparams)); + + const struct ggml_tensor * src0 = op->src[0]; + const struct ggml_tensor * src1 = op->src[1]; + const struct ggml_tensor * dst = op; + + if ((src0->type != GGML_TYPE_F16 && src0->type != GGML_TYPE_F32) || + src1->type != GGML_TYPE_F32 || dst->type != GGML_TYPE_F32) { + return false; + } + + const int32_t IC = ggml_get_op_params_i32(op, 9); + const int32_t N = ggml_get_op_params_i32(op, 10); + const int32_t OC = ggml_get_op_params_i32(op, 11); + if (IC <= 0 || N <= 0 || OC <= 0 || src0->ne[3] != (int64_t) IC * OC || src1->ne[3] != (int64_t) IC * N || + dst->ne[3] != (int64_t) OC * N) { + return false; + } + + for (int i = 0; i < 3; i++) { + const int32_t s = ggml_get_op_params_i32(op, i); + const int32_t p = ggml_get_op_params_i32(op, 3 + i); + const int32_t d = ggml_get_op_params_i32(op, 6 + i); + if (s <= 0 || d <= 0 || p < 0 || src0->ne[i] <= 0) { + return false; + } + const int64_t o = (src1->ne[i] + 2 * p - d * (src0->ne[i] - 1) - 1) / s + 1; + if (o != dst->ne[i]) { + return false; + } + } + + const size_t es = ggml_type_size(src0->type); + const bool w_rows = src0->nb[0] == es && src0->nb[1] == src0->ne[0] * es && src0->nb[2] == src0->ne[1] * src0->nb[1] && + ((int64_t) IC * OC == 1 || src0->nb[3] == src0->ne[2] * src0->nb[2]); + if (!w_rows || src1->nb[0] != sizeof(float) || dst->nb[2] != dst->ne[1] * dst->nb[1]) { + return false; + } + + const uint32_t total_rows = (uint32_t) (dst->ne[1] * dst->ne[2] * dst->ne[3]); + kparams->total_rows = total_rows; + kparams->n_threads = total_rows == 0 ? 1 : (std::min)((uint32_t) htp_session_n_threads(sess), total_rows); + + return htp_kparams_conv2d_tiled(sess, op, kparams); +} + static bool htp_dsv4_hc_pre_can_collapse( const struct ggml_tensor * x, const struct ggml_tensor * weights, @@ -5241,6 +5298,9 @@ bool htp_kparams_solve( case HTP_OP_CONV_2D_DW: return htp_kparams_invoke(sess, op, kparams); + case HTP_OP_CONV_3D: + return htp_kparams_invoke(sess, op, kparams); + case HTP_OP_SUM_ROWS: case HTP_OP_SUM: case HTP_OP_ARGMAX: diff --git a/ggml/src/ggml-hexagon/htp/conv2d-ops.c b/ggml/src/ggml-hexagon/htp/conv2d-ops.c index 42e70e089640..1584c7c6a8cd 100644 --- a/ggml/src/ggml-hexagon/htp/conv2d-ops.c +++ b/ggml/src/ggml-hexagon/htp/conv2d-ops.c @@ -87,11 +87,11 @@ static inline HVX_Vector conv2d_hmx_rt_restore(const __fp16 * scales, uint32_t r return Q6_V_vsplat_R((127 + e) << 23); } -// Output rows [*row_base, *row_base + *n_rows) of N * OH for this device. +// Output rows [*row_base, *row_base + *n_rows) of total for this device. // The output is written by DMA only, so device ranges need no cache line alignment. -static void conv2d_mdev_rows(const struct htp_ops_context * octx, uint32_t * row_base, uint32_t * n_rows) { +static void conv2d_mdev_rows(const struct htp_ops_context * octx, uint32_t total, uint32_t * row_base, uint32_t * n_rows) { *row_base = 0; - *n_rows = octx->dst->ne[3] * octx->dst->ne[1]; + *n_rows = total; if (octx->ctx->mdev.count > 1) { const struct htp_tensor_mdev_range range = htp_tensor_mdev_partition( *n_rows, 1, octx->ctx->mdev.idx, octx->ctx->mdev.count, &octx->ctx->mdev.count_div); @@ -134,9 +134,55 @@ static void conv2d_hmx_worker(void * data) { } } +// Tiled path values whose source differs between conv_2d and conv_3d. +// conv_3d runs as conv_2d with virtual KH = KD * KHr and OH = OD * OHr, only the input stage unfolds them. +struct htp_conv_geom { + uint32_t KH, KHr, Cin, Cout; + uint32_t OH, OHr, N, ID; + int32_t s2, p0, p1, p2, d0, d1, d2; + size_t x_nb_d, x_nb_c, x_nb_n; // input strides for d, cin, n + size_t y_nb_c, y_nb_n; // dst strides for cout, n + size_t w_nb_co; // kernel cout stride + struct fastdiv_values div_khr; + struct fastdiv_values div_ohr; +}; + +// conv_3d tensors are [W, H, D, C * N] with the channel as the inner index of ne[3] +static void conv_geom_init(struct htp_conv_geom * g, const struct htp_ops_context * octx) { + const struct htp_tensor * kernel = octx->src[0]; + const struct htp_tensor * input = octx->src[1]; + const struct htp_tensor * dst = octx->dst; + const int32_t * op = octx->op_params; + + if (octx->op == HTP_OP_CONV_3D) { + const uint32_t IC = op[9]; + const uint32_t OC = op[11]; + *g = (struct htp_conv_geom) { + .KH = kernel->ne[2] * kernel->ne[1], .KHr = kernel->ne[1], .Cin = IC, .Cout = OC, + .OH = dst->ne[2] * dst->ne[1], .OHr = dst->ne[1], .N = op[10], .ID = input->ne[2], + .s2 = op[2], .p0 = op[3], .p1 = op[4], .p2 = op[5], .d0 = op[6], .d1 = op[7], .d2 = op[8], + .x_nb_d = input->nb[2], .x_nb_c = input->nb[3], .x_nb_n = (size_t) IC * input->nb[3], + .y_nb_c = dst->nb[3], .y_nb_n = (size_t) OC * dst->nb[3], + .w_nb_co = (size_t) IC * kernel->nb[3], + }; + } else { + *g = (struct htp_conv_geom) { + .KH = kernel->ne[1], .KHr = kernel->ne[1], .Cin = kernel->ne[2], .Cout = kernel->ne[3], + .OH = dst->ne[1], .OHr = dst->ne[1], .N = dst->ne[3], .ID = 1, + .s2 = 1, .p0 = op[2], .p1 = op[3], .p2 = 0, .d0 = op[4], .d1 = op[5], .d2 = 1, + .x_nb_d = 0, .x_nb_c = input->nb[2], .x_nb_n = input->nb[3], + .y_nb_c = dst->nb[2], .y_nb_n = dst->nb[3], + .w_nb_co = kernel->nb[3], + }; + } + g->div_khr = init_fastdiv_values(g->KHr); + g->div_ohr = init_fastdiv_values(g->OHr); +} + struct htp_conv2d_hmx_context { struct htp_ops_context * octx; const struct htp_conv2d_kernel_params * kp; + const struct htp_conv_geom * g; struct htp_conv2d_vtcm_layout layout; __fp16 * scales; // [rt] HMX scales __fp16 * w_tiles; @@ -169,7 +215,7 @@ static void conv2d_hmx_wpack_thread(unsigned int nth, unsigned int ith, void * d const struct htp_conv2d_kernel_params * kp = hc->kp; const struct htp_tensor * kernel = octx->src[0]; - const uint32_t Cout = kernel->ne[3]; + const uint32_t Cout = hc->g->Cout; const uint32_t K = kp->tiled.k_size; const uint32_t k_pad = kp->tiled.n_k_tiles * HMX_FP16_TILE_N_COLS; const uint32_t WK = kp->tiled.wk_chunk; @@ -200,8 +246,8 @@ static void conv2d_hmx_wpack_thread(unsigned int nth, unsigned int ith, void * d for (uint32_t i = 0; i < 2 && i * WK < k_pad; i++) { const uint32_t kc = i * WK; const uint32_t kv = MIN(WK, K - kc); - dma_queue_push(dmaq, dma_make_data(wdma[i], kernel->data + (size_t) co0 * kernel->nb[3] + (size_t) kc * es), - WK * es, kernel->nb[3], kv * es, nco); + dma_queue_push(dmaq, dma_make_data(wdma[i], kernel->data + (size_t) co0 * hc->g->w_nb_co + (size_t) kc * es), + WK * es, hc->g->w_nb_co, kv * es, nco); } for (uint32_t kc = 0; kc < k_pad; kc += WK) { @@ -234,8 +280,8 @@ static void conv2d_hmx_wpack_thread(unsigned int nth, unsigned int ith, void * d const uint32_t next_kc = kc + 2 * WK; if (next_kc < k_pad) { const uint32_t next_kv = MIN(WK, K - next_kc); - dma_queue_push(dmaq, dma_make_data(wdma[bi], kernel->data + (size_t) co0 * kernel->nb[3] + (size_t) next_kc * es), - WK * es, kernel->nb[3], next_kv * es, nco); + dma_queue_push(dmaq, dma_make_data(wdma[bi], kernel->data + (size_t) co0 * hc->g->w_nb_co + (size_t) next_kc * es), + WK * es, hc->g->w_nb_co, next_kv * es, nco); } bi ^= 1; } @@ -243,21 +289,23 @@ static void conv2d_hmx_wpack_thread(unsigned int nth, unsigned int ith, void * d } // Stage Cin*KH input rows for G col tiles: stage[(cin*KH + kh)*SR + j] = x[n][cin][ih(kh)][ow0 - p0 + j], zero outside +// For conv_3d, kh = kd * KHr + kh' and oh = od * OHr + oh' are unfolded here static void conv2d_hmx_stage(const struct htp_ops_context * octx, const struct htp_conv2d_kernel_params * kp, - dma_queue * dmaq, float * stage, uint32_t n, uint32_t oh, uint32_t ow0, uint32_t G) { + const struct htp_conv_geom * g, dma_queue * dmaq, float * stage, uint32_t n, uint32_t oh, + uint32_t ow0, uint32_t G) { const struct htp_tensor * kernel = octx->src[0]; const struct htp_tensor * input = octx->src[1]; const uint32_t KW = kernel->ne[0]; - const uint32_t KH = kernel->ne[1]; - const uint32_t Cin = kernel->ne[2]; + const uint32_t KH = g->KH; + const uint32_t Cin = g->Cin; const int32_t IW = input->ne[0]; const int32_t IH = input->ne[1]; const int32_t s1 = octx->op_params[1]; - const int32_t p0 = octx->op_params[2]; - const int32_t p1 = octx->op_params[3]; - const int32_t d0 = octx->op_params[4]; - const int32_t d1 = octx->op_params[5]; + const int32_t p0 = g->p0; + const int32_t p1 = g->p1; + const int32_t d0 = g->d0; + const int32_t d1 = g->d1; const uint32_t SR = kp->tiled.stage_row; const int32_t Wb = 32 * G + (KW - 1) * d0; @@ -265,11 +313,16 @@ static void conv2d_hmx_stage(const struct htp_ops_context * octx, const struct h const int32_t lo = MAX(0, -iw0); const int32_t hi = MIN(Wb, IW - iw0); + const uint32_t od = fastdiv(oh, &g->div_ohr); + oh -= od * g->OHr; + for (uint32_t kh = 0; kh < KH; kh++) { - const int32_t ih = (int32_t) oh * s1 + (int32_t) kh * d1 - p1; + const int32_t kd = fastdiv(kh, &g->div_khr); + const int32_t id = (int32_t) od * g->s2 + kd * g->d2 - g->p2; + const int32_t ih = (int32_t) oh * s1 + (int32_t) (kh - kd * g->KHr) * d1 - p1; float * rows = stage + (size_t) kh * SR; - if (ih < 0 || ih >= IH || hi <= lo) { + if (id < 0 || id >= (int32_t) g->ID || ih < 0 || ih >= IH || hi <= lo) { for (uint32_t c = 0; c < Cin; c++) { hvx_splat_f32_u(rows + (size_t) c * KH * SR, 0.0f, Wb); } @@ -277,8 +330,8 @@ static void conv2d_hmx_stage(const struct htp_ops_context * octx, const struct h continue; } - const dma_addr_t src = input->data + (size_t) n * input->nb[3] + (size_t) ih * input->nb[1] + (size_t) (iw0 + lo) * sizeof(float); - dma_queue_push(dmaq, dma_make_data(rows + lo, src), (size_t) KH * SR * sizeof(float), input->nb[2], + const dma_addr_t src = input->data + (size_t) n * g->x_nb_n + (size_t) id * g->x_nb_d + (size_t) ih * input->nb[1] + (size_t) (iw0 + lo) * sizeof(float); + dma_queue_push(dmaq, dma_make_data(rows + lo, src), (size_t) KH * SR * sizeof(float), g->x_nb_c, (size_t) (hi - lo) * sizeof(float), Cin); if (lo > 0 || hi < Wb) { @@ -383,8 +436,8 @@ static void conv2d_hmx_cwhn_f16(__fp16 * restrict dst, const float * restrict sr // Stage the segment of col tiles starting at t, returns its number of col tiles static uint32_t conv2d_hmx_issue_seg(const struct htp_ops_context * octx, const struct htp_conv2d_kernel_params * kp, - dma_queue * dmaq, float * stage, uint32_t t, uint32_t t1) { - const uint32_t OH = octx->dst->ne[1]; + const struct htp_conv_geom * g, dma_queue * dmaq, float * stage, uint32_t t, uint32_t t1) { + const uint32_t OH = g->OH; const uint32_t n_owt = kp->tiled.n_ow_tiles; const uint32_t row = fastdiv(t, &kp->tiled.div_ow_tiles); const uint32_t owt = t - row * n_owt; @@ -395,7 +448,7 @@ static uint32_t conv2d_hmx_issue_seg(const struct htp_ops_context * octx, const if (kp->cwhn) { conv2d_hmx_stage_cwhn(octx, kp, dmaq, stage, n, oh, owt * 32, G); } else { - conv2d_hmx_stage(octx, kp, dmaq, stage, n, oh, owt * 32, G); + conv2d_hmx_stage(octx, kp, g, dmaq, stage, n, oh, owt * 32, G); } return G; } @@ -718,9 +771,9 @@ static void conv2d_hmx_pack(const struct htp_conv2d_hmx_context * hc, uint32_t i const struct htp_tensor * kernel = octx->src[0]; const uint32_t KW = kernel->ne[0]; - const uint32_t KH = kernel->ne[1]; - const uint32_t Cin = kernel->ne[2]; - const uint32_t d0 = octx->op_params[4]; + const uint32_t KH = hc->g->KH; + const uint32_t Cin = hc->g->Cin; + const uint32_t d0 = hc->g->d0; const uint32_t n_vec = kp->tiled.n_k_tiles * (HMX_FP16_TILE_N_ELMS / 64); dma_queue * dmaq = octx->ctx->dma[ith]; @@ -750,7 +803,7 @@ static void conv2d_hmx_pack(const struct htp_conv2d_hmx_context * hc, uint32_t i uint32_t ng[2] = { 0, 0 }; uint32_t tq = t0; for (uint32_t i = 0; i < n_stage && tq < t1; i++) { - ng[i] = conv2d_hmx_issue_seg(octx, kp, dmaq, stages[i], tq, t1); + ng[i] = conv2d_hmx_issue_seg(octx, kp, hc->g, dmaq, stages[i], tq, t1); tq += ng[i]; } @@ -780,7 +833,7 @@ static void conv2d_hmx_pack(const struct htp_conv2d_hmx_context * hc, uint32_t i htp_trace_event_stop(tr, HTP_TRACE_EVT_HVX_A_PREP, (uint16_t) t); if (tq < t1) { - ng[bi] = conv2d_hmx_issue_seg(octx, kp, dmaq, stage, tq, t1); + ng[bi] = conv2d_hmx_issue_seg(octx, kp, hc->g, dmaq, stage, tq, t1); tq += ng[bi]; } t += G; @@ -794,8 +847,8 @@ static void conv2d_hmx_unpack(const struct htp_conv2d_hmx_context * hc, uint32_t const struct htp_tensor * dst = octx->dst; const uint32_t OW = dst->ne[0]; - const uint32_t OH = dst->ne[1]; - const uint32_t Cout = dst->ne[2]; + const uint32_t OH = hc->g->OH; + const uint32_t Cout = hc->g->Cout; const uint32_t n_owt = kp->tiled.n_ow_tiles; const uint32_t OS = kp->tiled.seg_tiles * 32; @@ -850,8 +903,8 @@ static void conv2d_hmx_unpack(const struct htp_conv2d_hmx_context * hc, uint32_t } htp_trace_event_stop(tr, HTP_TRACE_EVT_HVX_O_PROC, (uint16_t) t); - const dma_addr_t ddr = dst->data + (size_t) n * dst->nb[3] + (size_t) oh * dst->nb[1] + (size_t) co0 * dst->nb[2] + (size_t) ow0 * sizeof(float); - dma_queue_push(dmaq, dma_make_data(ddr, ob), dst->nb[2], (size_t) OS * sizeof(float), (size_t) nv * sizeof(float), nco); + const dma_addr_t ddr = dst->data + (size_t) n * hc->g->y_nb_n + (size_t) oh * dst->nb[1] + (size_t) co0 * hc->g->y_nb_c + (size_t) ow0 * sizeof(float); + dma_queue_push(dmaq, dma_make_data(ddr, ob), hc->g->y_nb_c, (size_t) OS * sizeof(float), (size_t) nv * sizeof(float), nco); n_pushed++; b ^= 1; } @@ -904,8 +957,11 @@ static void conv2d_hmx_run_phase(struct htp_conv2d_hmx_context * hc, int32_t pac } static int conv2d_hmx_tiled(struct htp_ops_context * octx, const struct htp_conv2d_kernel_params * kp) { + struct htp_conv_geom g; + conv_geom_init(&g, octx); + uint32_t row_base, n_rows; - conv2d_mdev_rows(octx, &row_base, &n_rows); + conv2d_mdev_rows(octx, g.N * g.OH, &row_base, &n_rows); if (n_rows == 0) { return HTP_STATUS_OK; } @@ -923,6 +979,7 @@ static int conv2d_hmx_tiled(struct htp_ops_context * octx, const struct htp_conv struct htp_conv2d_hmx_context hc = { .octx = octx, .kp = kp, + .g = &g, .layout = layout, .scales = VTCM_LAYOUT_PTR(__fp16, base, layout.off_scales), .w_tiles = VTCM_LAYOUT_PTR(__fp16, base, layout.off_w), @@ -2153,7 +2210,7 @@ static int conv2d_hmx_win(struct htp_ops_context * octx, const struct htp_conv2d const uint32_t d1 = octx->op_params[5]; uint32_t row_base, n_rows; - conv2d_mdev_rows(octx, &row_base, &n_rows); + conv2d_mdev_rows(octx, dst->ne[3] * dst->ne[1], &row_base, &n_rows); if (n_rows == 0) { return HTP_STATUS_OK; } diff --git a/ggml/src/ggml-hexagon/htp/htp-ops.h b/ggml/src/ggml-hexagon/htp/htp-ops.h index 96257c241263..bd7812486f82 100644 --- a/ggml/src/ggml-hexagon/htp/htp-ops.h +++ b/ggml/src/ggml-hexagon/htp/htp-ops.h @@ -117,6 +117,7 @@ enum htp_op_code { HTP_OP_UNARY_CEIL, HTP_OP_UNARY_SGN, HTP_OP_UPSCALE, + HTP_OP_CONV_3D, HTP_OP_DSV4_HC_PRE, HTP_OP_DSV4_HC_POST, HTP_OP_TOP_K_MASK, diff --git a/ggml/src/ggml-hexagon/htp/main.c b/ggml/src/ggml-hexagon/htp/main.c index ac8820beb02d..56567321216b 100644 --- a/ggml/src/ggml-hexagon/htp/main.c +++ b/ggml/src/ggml-hexagon/htp/main.c @@ -1030,6 +1030,7 @@ static int execute_op(struct htp_ops_context * octx) { return op_im2col_3d(octx); case HTP_OP_CONV_2D: + case HTP_OP_CONV_3D: return op_conv_2d(octx); case HTP_OP_CONV_2D_DW: From bb21a4a7016945a2e16a07a0c84edbb61064b073 Mon Sep 17 00:00:00 2001 From: Mark Date: Fri, 30 Oct 2026 17:47:30 -0700 Subject: [PATCH 2/3] hexagon: modify checking to eliminate divide-by-zero hazards --- ggml/src/ggml-hexagon/htp-kparams.cpp | 34 ++++++++++++++++++--------- tests/test-backend-ops.cpp | 7 ++++++ 2 files changed, 30 insertions(+), 11 deletions(-) diff --git a/ggml/src/ggml-hexagon/htp-kparams.cpp b/ggml/src/ggml-hexagon/htp-kparams.cpp index bc9dac4c37a8..128373920adb 100644 --- a/ggml/src/ggml-hexagon/htp-kparams.cpp +++ b/ggml/src/ggml-hexagon/htp-kparams.cpp @@ -3999,13 +3999,26 @@ static bool htp_kparams_conv2d_tiled( const int32_t s0 = ggml_get_op_params_i32(dst, 0); const int32_t d0 = ggml_get_op_params_i32(dst, is3d ? 6 : 4); + // range checks in 64 bits before narrowing, conv_3d folds KD into KH and OD into OH + const int64_t kh_w = is3d ? src0->ne[2] * src0->ne[1] : src0->ne[1]; + const int64_t cin_w = is3d ? ggml_get_op_params_i32(dst, 9) : src0->ne[2]; + const int64_t oh_w = is3d ? dst->ne[2] * dst->ne[1] : dst->ne[1]; + const int64_t n_w = is3d ? ggml_get_op_params_i32(dst, 10) : dst->ne[3]; + const int64_t k_w = src0->ne[0] * kh_w * cin_w; + const int64_t ct_w = n_w * oh_w * ((dst->ne[0] + 31) / 32); + + // each stage buffer keeps KH input DMAs in flight on a 256-entry queue (main.c), which holds at most 255 + if (kh_w < 1 || kh_w > 255 || k_w < 1 || k_w > (1 << 24) || ct_w < 1 || ct_w > UINT32_MAX) { + return false; + } + const uint32_t KW = src0->ne[0]; - const uint32_t KH = is3d ? src0->ne[2] * src0->ne[1] : src0->ne[1]; - const uint32_t Cin = is3d ? ggml_get_op_params_i32(dst, 9) : src0->ne[2]; + const uint32_t KH = kh_w; + const uint32_t Cin = cin_w; const uint32_t Cout = is3d ? ggml_get_op_params_i32(dst, 11) : src0->ne[3]; const uint32_t OW = dst->ne[0]; - const uint32_t OH = is3d ? dst->ne[2] * dst->ne[1] : dst->ne[1]; - const uint32_t N = is3d ? ggml_get_op_params_i32(dst, 10) : dst->ne[3]; + const uint32_t OH = oh_w; + const uint32_t N = n_w; const size_t es = ggml_type_size(src0->type); if (s0 != 1 || dst->nb[0] != sizeof(float)) { @@ -4043,10 +4056,6 @@ static bool htp_kparams_conv2d_tiled( const uint32_t nth = htp_session_n_threads(sess); const size_t tile = 2048; - if (KH > 255) { - return false; - } - uint32_t rt_block = n_rt; uint32_t ct = (std::min)(n_ct, 64u); uint32_t seg = (std::min)(8u, n_owt); @@ -4387,9 +4396,12 @@ static bool htp_kparams_solve_conv3d( return false; } - const uint32_t total_rows = (uint32_t) (dst->ne[1] * dst->ne[2] * dst->ne[3]); - kparams->total_rows = total_rows; - kparams->n_threads = total_rows == 0 ? 1 : (std::min)((uint32_t) htp_session_n_threads(sess), total_rows); + const int64_t total_rows = dst->ne[1] * dst->ne[2] * dst->ne[3]; + if (total_rows > UINT32_MAX) { + return false; + } + kparams->total_rows = (uint32_t) total_rows; + kparams->n_threads = total_rows == 0 ? 1 : (std::min)((uint32_t) htp_session_n_threads(sess), (uint32_t) total_rows); return htp_kparams_conv2d_tiled(sess, op, kparams); } diff --git a/tests/test-backend-ops.cpp b/tests/test-backend-ops.cpp index e35806f31523..500517ca9fe6 100644 --- a/tests/test-backend-ops.cpp +++ b/tests/test-backend-ops.cpp @@ -10883,6 +10883,13 @@ static std::vector> make_test_cases_eval() { test_cases.emplace_back(new test_conv_3d(1, 1, 2, 2, 2, 1, 1, 1, 0, 1, 1, 1, 0, 0, 0, 1, 1, 1, kernel_type)); test_cases.emplace_back(new test_conv_3d(1, 1, 2, 2, 2, 1, 1, 0, 1, 1, 1, 1, 0, 0, 0, 1, 1, 1, kernel_type)); test_cases.emplace_back(new test_conv_3d(1, 1, 2, 2, 2, 1, 0, 1, 1, 1, 1, 1, 0, 0, 0, 1, 1, 1, kernel_type)); + // input and output channel strides just over 16 MB, beyond the 24-bit DMA stride field + test_cases.emplace_back(new test_conv_3d(1, 2, 65, 256, 256, 2, 1, 1, 1, 1, 1, 1, 0, 0, 0, 1, 1, 1, kernel_type)); + // KD * KH = 127, 128, 255, 256 around the HTP DMA queue limits + test_cases.emplace_back(new test_conv_3d(1, 2, 127, 1, 40, 3, 127, 1, 1, 1, 1, 1, 0, 0, 0, 1, 1, 1, kernel_type)); + test_cases.emplace_back(new test_conv_3d(1, 2, 64, 2, 40, 3, 64, 2, 1, 1, 1, 1, 0, 0, 0, 1, 1, 1, kernel_type)); + test_cases.emplace_back(new test_conv_3d(1, 2, 85, 3, 40, 3, 85, 3, 1, 1, 1, 1, 0, 0, 0, 1, 1, 1, kernel_type)); + test_cases.emplace_back(new test_conv_3d(1, 2, 16, 16, 40, 3, 16, 16, 1, 1, 1, 1, 0, 0, 0, 1, 1, 1, kernel_type)); } for(uint32_t Cout : {1, 9}){ From fdad23a314d37e937a2583b7c8297174180cb061 Mon Sep 17 00:00:00 2001 From: Mark Date: Wed, 4 Nov 2026 09:37:07 -0800 Subject: [PATCH 3/3] hexagon: conv2d tiled: compute the conv_3d unfold divisors on the host The KH and OH fastdiv values used to unfold conv_3d kh into (kd, kh) and oh into (od, oh) were built per op on the device, each with a 64-bit divide. Pass them in the kernel params instead; they fit in the space the window variant already reserves, so the struct size does not change. Assisted-by: Claude Code --- ggml/src/ggml-hexagon/htp-kparams.cpp | 2 ++ ggml/src/ggml-hexagon/htp/conv2d-ops.c | 8 ++------ ggml/src/ggml-hexagon/htp/conv2d-ops.h | 2 ++ 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/ggml/src/ggml-hexagon/htp-kparams.cpp b/ggml/src/ggml-hexagon/htp-kparams.cpp index 128373920adb..ee5ef118b850 100644 --- a/ggml/src/ggml-hexagon/htp-kparams.cpp +++ b/ggml/src/ggml-hexagon/htp-kparams.cpp @@ -4130,6 +4130,8 @@ static bool htp_kparams_conv2d_tiled( kp->tiled.wk_chunk = wk; kp->tiled.div_ow_tiles = init_fastdiv_values(n_owt); kp->tiled.div_oh = init_fastdiv_values(OH); + kp->tiled.div_khr = init_fastdiv_values(src0->ne[1]); + kp->tiled.div_ohr = init_fastdiv_values(dst->ne[1]); return true; } diff --git a/ggml/src/ggml-hexagon/htp/conv2d-ops.c b/ggml/src/ggml-hexagon/htp/conv2d-ops.c index 1584c7c6a8cd..650e65ee4285 100644 --- a/ggml/src/ggml-hexagon/htp/conv2d-ops.c +++ b/ggml/src/ggml-hexagon/htp/conv2d-ops.c @@ -143,8 +143,6 @@ struct htp_conv_geom { size_t x_nb_d, x_nb_c, x_nb_n; // input strides for d, cin, n size_t y_nb_c, y_nb_n; // dst strides for cout, n size_t w_nb_co; // kernel cout stride - struct fastdiv_values div_khr; - struct fastdiv_values div_ohr; }; // conv_3d tensors are [W, H, D, C * N] with the channel as the inner index of ne[3] @@ -175,8 +173,6 @@ static void conv_geom_init(struct htp_conv_geom * g, const struct htp_ops_contex .w_nb_co = kernel->nb[3], }; } - g->div_khr = init_fastdiv_values(g->KHr); - g->div_ohr = init_fastdiv_values(g->OHr); } struct htp_conv2d_hmx_context { @@ -313,11 +309,11 @@ static void conv2d_hmx_stage(const struct htp_ops_context * octx, const struct h const int32_t lo = MAX(0, -iw0); const int32_t hi = MIN(Wb, IW - iw0); - const uint32_t od = fastdiv(oh, &g->div_ohr); + const uint32_t od = fastdiv(oh, &kp->tiled.div_ohr); oh -= od * g->OHr; for (uint32_t kh = 0; kh < KH; kh++) { - const int32_t kd = fastdiv(kh, &g->div_khr); + const int32_t kd = fastdiv(kh, &kp->tiled.div_khr); const int32_t id = (int32_t) od * g->s2 + kd * g->d2 - g->p2; const int32_t ih = (int32_t) oh * s1 + (int32_t) (kh - kd * g->KHr) * d1 - p1; float * rows = stage + (size_t) kh * SR; diff --git a/ggml/src/ggml-hexagon/htp/conv2d-ops.h b/ggml/src/ggml-hexagon/htp/conv2d-ops.h index a48bf3e83ffe..aefed269ea46 100644 --- a/ggml/src/ggml-hexagon/htp/conv2d-ops.h +++ b/ggml/src/ggml-hexagon/htp/conv2d-ops.h @@ -35,6 +35,8 @@ struct htp_conv2d_tiled_params { uint32_t wk_chunk; struct fastdiv_values div_ow_tiles; struct fastdiv_values div_oh; + struct fastdiv_values div_khr; // kernel ne[1], unfolds conv_3d kh = kd * KHr + kh' + struct fastdiv_values div_ohr; // dst ne[1], unfolds conv_3d oh = od * OHr + oh' }; struct htp_conv2d_window_params {