Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
84 changes: 84 additions & 0 deletions src/pipeline/pass_k8s.c
Original file line number Diff line number Diff line change
Expand Up @@ -3250,6 +3250,87 @@ static char *k8s_skeleton(const char *src, int len, int *out_len) {
return out;
}

/* ── Dependency-manifest text encoding (#2941) ────────────────────── */

enum { DEP_UTF8_BOM_LEN = 3, DEP_UTF16_UNIT = 2, DEP_UTF8_MAX_PER_UNIT = 3 };

/* One UTF-16 code unit at `u`, little- or big-endian. */
static uint32_t dep_utf16_unit(const unsigned char *u, bool le) {
return le ? (uint32_t)u[0] | ((uint32_t)u[1] << 8U) : ((uint32_t)u[0] << 8U) | (uint32_t)u[1];
}

/* Append code point `c` to `out` as UTF-8; returns the bytes written. */
static size_t dep_put_utf8(char *out, uint32_t c) {
unsigned char *o = (unsigned char *)out;
if (c < 0x80U) {
o[0] = (unsigned char)c;
return 1;
}
if (c < 0x800U) {
o[0] = (unsigned char)(0xC0U | (c >> 6U));
o[1] = (unsigned char)(0x80U | (c & 0x3FU));
return 2;
}
if (c < 0x10000U) {
o[0] = (unsigned char)(0xE0U | (c >> 12U));
o[1] = (unsigned char)(0x80U | ((c >> 6U) & 0x3FU));
o[2] = (unsigned char)(0x80U | (c & 0x3FU));
return 3;
}
o[0] = (unsigned char)(0xF0U | (c >> 18U));
o[1] = (unsigned char)(0x80U | ((c >> 12U) & 0x3FU));
o[2] = (unsigned char)(0x80U | ((c >> 6U) & 0x3FU));
o[3] = (unsigned char)(0x80U | (c & 0x3FU));
return 4;
}

/* requirements.txt / go.mod as UTF-8 text. pip decodes a requirements file by
* its BOM, and Windows tools write BOMs: Windows PowerShell 5.1 saves
* `pip freeze > requirements.txt` as UTF-16 LE with a BOM, and many editors
* put a UTF-8 BOM in front. Read as raw bytes, UTF-16 stops at the NUL after
* the first letter, and a UTF-8 BOM sticks to the first package name.
* Drops a UTF-8 BOM in place, or returns a new UTF-8 buffer for UTF-16 (BOM
* dropped, `src` freed). Keeps the K8S_TS_PAD zero tail k8s_read_file adds. */
static char *dep_manifest_utf8(char *src, int *len) {
const unsigned char *b = (const unsigned char *)src;
size_t n = (size_t)*len;
if (n >= DEP_UTF8_BOM_LEN && b[0] == 0xEFU && b[1] == 0xBBU && b[2] == 0xBFU) {
memmove(src, src + DEP_UTF8_BOM_LEN, n - DEP_UTF8_BOM_LEN + K8S_TS_PAD);
*len = (int)(n - DEP_UTF8_BOM_LEN);
return src;
}
bool le = n >= DEP_UTF16_UNIT && b[0] == 0xFFU && b[1] == 0xFEU;
bool be = n >= DEP_UTF16_UNIT && b[0] == 0xFEU && b[1] == 0xFFU;
if (!le && !be) {
return src;
}
size_t units = (n - DEP_UTF16_UNIT) / DEP_UTF16_UNIT;
char *out = cbm_alloc(CBM_MEM_CLASS_OTHER, (units * DEP_UTF8_MAX_PER_UNIT) + K8S_TS_PAD);
if (!out) {
return src;
}
const unsigned char *u = b + DEP_UTF16_UNIT;
size_t w = 0;
for (size_t i = 0; i < units; i++) {
uint32_t c = dep_utf16_unit(u + (i * DEP_UTF16_UNIT), le);
if (c >= 0xD800U && c <= 0xDBFFU && i + SKIP_ONE < units) {
uint32_t lo = dep_utf16_unit(u + ((i + SKIP_ONE) * DEP_UTF16_UNIT), le);
if (lo >= 0xDC00U && lo <= 0xDFFFU) {
c = 0x10000U + ((c - 0xD800U) << 10U) + (lo - 0xDC00U);
i++;
}
}
if (c >= 0xD800U && c <= 0xDFFFU) {
c = 0xFFFDU; /* unpaired surrogate */
}
w += dep_put_utf8(out + w, c);
}
memset(out + w, 0, K8S_TS_PAD);
cbm_free(CBM_MEM_CLASS_OTHER, src);
*len = (int)w;
return out;
}

static void k8s_prep_file(const k8s_prep_job_t *job, int i, k8s_prep_t *out) {
cbm_pipeline_ctx_t *ctx = job->ctx;
const char *path = job->files[i].path;
Expand All @@ -3259,6 +3340,9 @@ static void k8s_prep_file(const k8s_prep_job_t *job, int i, k8s_prep_t *out) {

if (is_gomod_file(base) || lang == CBM_LANG_GOMOD || is_requirements_file(base)) {
out->source = k8s_read_file(path, &out->src_len);
if (out->source) {
out->source = dep_manifest_utf8(out->source, &out->src_len);
}
out->kind = out->source ? K8S_DEP : K8S_SKIP;
} else if (lang == CBM_LANG_KUSTOMIZE) {
/* Its `namespace:` is the namespace of the objects it builds; its
Expand Down
116 changes: 116 additions & 0 deletions tests/test_grammar_probe_f.c
Original file line number Diff line number Diff line change
Expand Up @@ -6982,6 +6982,116 @@ TEST(probe_gomod_enables_go_imports) {
PASS();
}

/* ══════════════════════════════════════════════════════════════════
* GROUP 19b — requirements.txt text encodings (#2941)
*
* pip decodes requirements.txt by its BOM. Windows PowerShell 5.1 writes
* `pip freeze > requirements.txt` as UTF-16 LE with a BOM and CRLF, and many
* editors save UTF-8 with a BOM. Each package becomes one Package node.
* ══════════════════════════════════════════════════════════════════ */

/* `ascii` as UTF-16 (little- or big-endian) with a BOM; returns the length. */
static size_t gpf_utf16_with_bom(const char *ascii, int big_endian, unsigned char *out,
size_t cap) {
size_t n = 0;
if (cap < 2) return 0;
out[n++] = big_endian ? 0xFE : 0xFF;
out[n++] = big_endian ? 0xFF : 0xFE;
for (const char *p = ascii; *p && n + 2 <= cap; p++) {
out[n++] = big_endian ? 0 : (unsigned char)*p;
out[n++] = big_endian ? (unsigned char)*p : 0;
}
return n;
}

static int gpf_cmp_str(const void *a, const void *b) {
return strcmp(*(const char *const *)a, *(const char *const *)b);
}

/* Index a project holding only requirements.txt with these raw bytes; write
* the Package node names, sorted and comma-joined, into `out`. */
static int gpf_requirements_packages(const unsigned char *bytes, size_t len, char *out,
size_t out_sz) {
GpfProj lp;
memset(&lp, 0, sizeof(lp));
out[0] = '\0';
snprintf(lp.tmpdir, sizeof(lp.tmpdir), "/tmp/cbm_gpf_XXXXXX");
if (!cbm_mkdtemp(lp.tmpdir)) return 0;
gpf_to_fwd_slashes(lp.tmpdir);
char path[700];
snprintf(path, sizeof(path), "%s/requirements.txt", lp.tmpdir);
FILE *f = fopen(path, "wb");
if (!f) {
th_rmtree(lp.tmpdir);
return 0;
}
size_t wrote = fwrite(bytes, 1, len, f);
fclose(f);
cbm_store_t *store = wrote == len ? gpf_open_indexed(&lp) : NULL;
int ok = 0;
cbm_node_t *nodes = NULL;
int count = 0;
if (store &&
cbm_store_find_nodes_by_label(store, lp.project, "Package", &nodes, &count) ==
CBM_STORE_OK) {
const char *names[64];
int k = 0;
for (int i = 0; i < count && k < 64; i++) names[k++] = nodes[i].name;
qsort(names, (size_t)k, sizeof(names[0]), gpf_cmp_str);
for (int i = 0; i < k; i++) {
if (i) strncat(out, ",", out_sz - strlen(out) - 1);
strncat(out, names[i], out_sz - strlen(out) - 1);
}
cbm_store_free_nodes(nodes, count);
ok = 1;
}
gpf_cleanup(&lp, store);
return ok;
}

static const char GPF_REQS[] = "requests==2.32.3\r\nuvicorn==0.30.0\r\n";

/* RED before #2941: the file read as one bogus package, "\xff\xfe" + "r". */
TEST(probe_requirements_utf16le_bom_issue2941) {
unsigned char buf[256];
size_t n = gpf_utf16_with_bom(GPF_REQS, 0, buf, sizeof(buf));
char names[256];
ASSERT_TRUE(gpf_requirements_packages(buf, n, names, sizeof(names)));
ASSERT_STR_EQ(names, "requests,uvicorn");
PASS();
}

TEST(probe_requirements_utf16be_bom_issue2941) {
unsigned char buf[256];
size_t n = gpf_utf16_with_bom(GPF_REQS, 1, buf, sizeof(buf));
char names[256];
ASSERT_TRUE(gpf_requirements_packages(buf, n, names, sizeof(names)));
ASSERT_STR_EQ(names, "requests,uvicorn");
PASS();
}

/* RED before #2941: the BOM stuck to the first name ("\xEF\xBB\xBFrequests"). */
TEST(probe_requirements_utf8_bom_issue2941) {
unsigned char buf[256];
static const unsigned char bom[] = {0xEF, 0xBB, 0xBF};
memcpy(buf, bom, sizeof(bom));
memcpy(buf + sizeof(bom), GPF_REQS, sizeof(GPF_REQS) - 1);
char names[256];
ASSERT_TRUE(gpf_requirements_packages(buf, sizeof(bom) + sizeof(GPF_REQS) - 1, names,
sizeof(names)));
ASSERT_STR_EQ(names, "requests,uvicorn");
PASS();
}

/* GREEN guard: plain UTF-8 reads as before. */
TEST(probe_requirements_utf8_plain_issue2941) {
char names[256];
ASSERT_TRUE(gpf_requirements_packages((const unsigned char *)GPF_REQS, sizeof(GPF_REQS) - 1,
names, sizeof(names)));
ASSERT_STR_EQ(names, "requests,uvicorn");
PASS();
}

/* ══════════════════════════════════════════════════════════════════
* SUITE wiring
* ══════════════════════════════════════════════════════════════════ */
Expand Down Expand Up @@ -7208,4 +7318,10 @@ SUITE(grammar_probe_f) {
RUN_TEST(probe_gomod_simple);
RUN_TEST(probe_gomod_depends_on_edges);
RUN_TEST(probe_gomod_enables_go_imports);

/* requirements.txt encodings (#2941) */
RUN_TEST(probe_requirements_utf16le_bom_issue2941);
RUN_TEST(probe_requirements_utf16be_bom_issue2941);
RUN_TEST(probe_requirements_utf8_bom_issue2941);
RUN_TEST(probe_requirements_utf8_plain_issue2941);
}
Loading