diff --git a/src/pipeline/pass_k8s.c b/src/pipeline/pass_k8s.c index 034e8984c..ce89f98a5 100644 --- a/src/pipeline/pass_k8s.c +++ b/src/pipeline/pass_k8s.c @@ -3250,6 +3250,87 @@ static char *k8s_skeleton(const char *src, int len, int *out_len) { return out; } +/* ── Dependency-manifest text encoding (#2941) ────────────────────── */ + +enum { DEP_UTF8_BOM_LEN = 3, DEP_UTF16_UNIT = 2, DEP_UTF8_MAX_PER_UNIT = 3 }; + +/* One UTF-16 code unit at `u`, little- or big-endian. */ +static uint32_t dep_utf16_unit(const unsigned char *u, bool le) { + return le ? (uint32_t)u[0] | ((uint32_t)u[1] << 8U) : ((uint32_t)u[0] << 8U) | (uint32_t)u[1]; +} + +/* Append code point `c` to `out` as UTF-8; returns the bytes written. */ +static size_t dep_put_utf8(char *out, uint32_t c) { + unsigned char *o = (unsigned char *)out; + if (c < 0x80U) { + o[0] = (unsigned char)c; + return 1; + } + if (c < 0x800U) { + o[0] = (unsigned char)(0xC0U | (c >> 6U)); + o[1] = (unsigned char)(0x80U | (c & 0x3FU)); + return 2; + } + if (c < 0x10000U) { + o[0] = (unsigned char)(0xE0U | (c >> 12U)); + o[1] = (unsigned char)(0x80U | ((c >> 6U) & 0x3FU)); + o[2] = (unsigned char)(0x80U | (c & 0x3FU)); + return 3; + } + o[0] = (unsigned char)(0xF0U | (c >> 18U)); + o[1] = (unsigned char)(0x80U | ((c >> 12U) & 0x3FU)); + o[2] = (unsigned char)(0x80U | ((c >> 6U) & 0x3FU)); + o[3] = (unsigned char)(0x80U | (c & 0x3FU)); + return 4; +} + +/* requirements.txt / go.mod as UTF-8 text. pip decodes a requirements file by + * its BOM, and Windows tools write BOMs: Windows PowerShell 5.1 saves + * `pip freeze > requirements.txt` as UTF-16 LE with a BOM, and many editors + * put a UTF-8 BOM in front. Read as raw bytes, UTF-16 stops at the NUL after + * the first letter, and a UTF-8 BOM sticks to the first package name. + * Drops a UTF-8 BOM in place, or returns a new UTF-8 buffer for UTF-16 (BOM + * dropped, `src` freed). Keeps the K8S_TS_PAD zero tail k8s_read_file adds. */ +static char *dep_manifest_utf8(char *src, int *len) { + const unsigned char *b = (const unsigned char *)src; + size_t n = (size_t)*len; + if (n >= DEP_UTF8_BOM_LEN && b[0] == 0xEFU && b[1] == 0xBBU && b[2] == 0xBFU) { + memmove(src, src + DEP_UTF8_BOM_LEN, n - DEP_UTF8_BOM_LEN + K8S_TS_PAD); + *len = (int)(n - DEP_UTF8_BOM_LEN); + return src; + } + bool le = n >= DEP_UTF16_UNIT && b[0] == 0xFFU && b[1] == 0xFEU; + bool be = n >= DEP_UTF16_UNIT && b[0] == 0xFEU && b[1] == 0xFFU; + if (!le && !be) { + return src; + } + size_t units = (n - DEP_UTF16_UNIT) / DEP_UTF16_UNIT; + char *out = cbm_alloc(CBM_MEM_CLASS_OTHER, (units * DEP_UTF8_MAX_PER_UNIT) + K8S_TS_PAD); + if (!out) { + return src; + } + const unsigned char *u = b + DEP_UTF16_UNIT; + size_t w = 0; + for (size_t i = 0; i < units; i++) { + uint32_t c = dep_utf16_unit(u + (i * DEP_UTF16_UNIT), le); + if (c >= 0xD800U && c <= 0xDBFFU && i + SKIP_ONE < units) { + uint32_t lo = dep_utf16_unit(u + ((i + SKIP_ONE) * DEP_UTF16_UNIT), le); + if (lo >= 0xDC00U && lo <= 0xDFFFU) { + c = 0x10000U + ((c - 0xD800U) << 10U) + (lo - 0xDC00U); + i++; + } + } + if (c >= 0xD800U && c <= 0xDFFFU) { + c = 0xFFFDU; /* unpaired surrogate */ + } + w += dep_put_utf8(out + w, c); + } + memset(out + w, 0, K8S_TS_PAD); + cbm_free(CBM_MEM_CLASS_OTHER, src); + *len = (int)w; + return out; +} + static void k8s_prep_file(const k8s_prep_job_t *job, int i, k8s_prep_t *out) { cbm_pipeline_ctx_t *ctx = job->ctx; const char *path = job->files[i].path; @@ -3259,6 +3340,9 @@ static void k8s_prep_file(const k8s_prep_job_t *job, int i, k8s_prep_t *out) { if (is_gomod_file(base) || lang == CBM_LANG_GOMOD || is_requirements_file(base)) { out->source = k8s_read_file(path, &out->src_len); + if (out->source) { + out->source = dep_manifest_utf8(out->source, &out->src_len); + } out->kind = out->source ? K8S_DEP : K8S_SKIP; } else if (lang == CBM_LANG_KUSTOMIZE) { /* Its `namespace:` is the namespace of the objects it builds; its diff --git a/tests/test_grammar_probe_f.c b/tests/test_grammar_probe_f.c index b51ad995e..ee6a7226a 100644 --- a/tests/test_grammar_probe_f.c +++ b/tests/test_grammar_probe_f.c @@ -6982,6 +6982,116 @@ TEST(probe_gomod_enables_go_imports) { PASS(); } +/* ══════════════════════════════════════════════════════════════════ + * GROUP 19b — requirements.txt text encodings (#2941) + * + * pip decodes requirements.txt by its BOM. Windows PowerShell 5.1 writes + * `pip freeze > requirements.txt` as UTF-16 LE with a BOM and CRLF, and many + * editors save UTF-8 with a BOM. Each package becomes one Package node. + * ══════════════════════════════════════════════════════════════════ */ + +/* `ascii` as UTF-16 (little- or big-endian) with a BOM; returns the length. */ +static size_t gpf_utf16_with_bom(const char *ascii, int big_endian, unsigned char *out, + size_t cap) { + size_t n = 0; + if (cap < 2) return 0; + out[n++] = big_endian ? 0xFE : 0xFF; + out[n++] = big_endian ? 0xFF : 0xFE; + for (const char *p = ascii; *p && n + 2 <= cap; p++) { + out[n++] = big_endian ? 0 : (unsigned char)*p; + out[n++] = big_endian ? (unsigned char)*p : 0; + } + return n; +} + +static int gpf_cmp_str(const void *a, const void *b) { + return strcmp(*(const char *const *)a, *(const char *const *)b); +} + +/* Index a project holding only requirements.txt with these raw bytes; write + * the Package node names, sorted and comma-joined, into `out`. */ +static int gpf_requirements_packages(const unsigned char *bytes, size_t len, char *out, + size_t out_sz) { + GpfProj lp; + memset(&lp, 0, sizeof(lp)); + out[0] = '\0'; + snprintf(lp.tmpdir, sizeof(lp.tmpdir), "/tmp/cbm_gpf_XXXXXX"); + if (!cbm_mkdtemp(lp.tmpdir)) return 0; + gpf_to_fwd_slashes(lp.tmpdir); + char path[700]; + snprintf(path, sizeof(path), "%s/requirements.txt", lp.tmpdir); + FILE *f = fopen(path, "wb"); + if (!f) { + th_rmtree(lp.tmpdir); + return 0; + } + size_t wrote = fwrite(bytes, 1, len, f); + fclose(f); + cbm_store_t *store = wrote == len ? gpf_open_indexed(&lp) : NULL; + int ok = 0; + cbm_node_t *nodes = NULL; + int count = 0; + if (store && + cbm_store_find_nodes_by_label(store, lp.project, "Package", &nodes, &count) == + CBM_STORE_OK) { + const char *names[64]; + int k = 0; + for (int i = 0; i < count && k < 64; i++) names[k++] = nodes[i].name; + qsort(names, (size_t)k, sizeof(names[0]), gpf_cmp_str); + for (int i = 0; i < k; i++) { + if (i) strncat(out, ",", out_sz - strlen(out) - 1); + strncat(out, names[i], out_sz - strlen(out) - 1); + } + cbm_store_free_nodes(nodes, count); + ok = 1; + } + gpf_cleanup(&lp, store); + return ok; +} + +static const char GPF_REQS[] = "requests==2.32.3\r\nuvicorn==0.30.0\r\n"; + +/* RED before #2941: the file read as one bogus package, "\xff\xfe" + "r". */ +TEST(probe_requirements_utf16le_bom_issue2941) { + unsigned char buf[256]; + size_t n = gpf_utf16_with_bom(GPF_REQS, 0, buf, sizeof(buf)); + char names[256]; + ASSERT_TRUE(gpf_requirements_packages(buf, n, names, sizeof(names))); + ASSERT_STR_EQ(names, "requests,uvicorn"); + PASS(); +} + +TEST(probe_requirements_utf16be_bom_issue2941) { + unsigned char buf[256]; + size_t n = gpf_utf16_with_bom(GPF_REQS, 1, buf, sizeof(buf)); + char names[256]; + ASSERT_TRUE(gpf_requirements_packages(buf, n, names, sizeof(names))); + ASSERT_STR_EQ(names, "requests,uvicorn"); + PASS(); +} + +/* RED before #2941: the BOM stuck to the first name ("\xEF\xBB\xBFrequests"). */ +TEST(probe_requirements_utf8_bom_issue2941) { + unsigned char buf[256]; + static const unsigned char bom[] = {0xEF, 0xBB, 0xBF}; + memcpy(buf, bom, sizeof(bom)); + memcpy(buf + sizeof(bom), GPF_REQS, sizeof(GPF_REQS) - 1); + char names[256]; + ASSERT_TRUE(gpf_requirements_packages(buf, sizeof(bom) + sizeof(GPF_REQS) - 1, names, + sizeof(names))); + ASSERT_STR_EQ(names, "requests,uvicorn"); + PASS(); +} + +/* GREEN guard: plain UTF-8 reads as before. */ +TEST(probe_requirements_utf8_plain_issue2941) { + char names[256]; + ASSERT_TRUE(gpf_requirements_packages((const unsigned char *)GPF_REQS, sizeof(GPF_REQS) - 1, + names, sizeof(names))); + ASSERT_STR_EQ(names, "requests,uvicorn"); + PASS(); +} + /* ══════════════════════════════════════════════════════════════════ * SUITE wiring * ══════════════════════════════════════════════════════════════════ */ @@ -7208,4 +7318,10 @@ SUITE(grammar_probe_f) { RUN_TEST(probe_gomod_simple); RUN_TEST(probe_gomod_depends_on_edges); RUN_TEST(probe_gomod_enables_go_imports); + + /* requirements.txt encodings (#2941) */ + RUN_TEST(probe_requirements_utf16le_bom_issue2941); + RUN_TEST(probe_requirements_utf16be_bom_issue2941); + RUN_TEST(probe_requirements_utf8_bom_issue2941); + RUN_TEST(probe_requirements_utf8_plain_issue2941); }