Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
50 changes: 47 additions & 3 deletions internal/cbm/cbm.c
Original file line number Diff line number Diff line change
Expand Up @@ -3319,15 +3319,59 @@ CBMFileResult *cbm_extract_file(const char *source, int source_len, CBMLanguage
return r;
}

/* The grammars built on tree-sitter-c: same set as cbm_conditional_block_types
* minus C#, which has no backslash line splices. */
static bool c_splice_lang(CBMLanguage language) {
return language == CBM_LANG_C || language == CBM_LANG_CPP || language == CBM_LANG_CUDA ||
language == CBM_LANG_OBJC || language == CBM_LANG_GLSL || language == CBM_LANG_HLSL ||
language == CBM_LANG_ISPC || language == CBM_LANG_SLANG;
}

/* #1546: these grammars read a backslash line splice inside a string literal
* as an escape_sequence, `\` plus one character. With CRLF line endings that
* character is the CR, the LF then ends the literal, and the rest of the
* function, often the rest of the file, becomes one ERROR region.
* Each `\` CR LF becomes space `\` LF: the same length, the line break in the
* same place, so offsets and line numbers still refer to the file on disk and
* the next line is byte-identical. Copied only when a splice is found. The
* cross-file C/C++ LSP pass reads the file again and applies it too. */
const char *cbm_c_mask_crlf_splices(CBMArena *a, const char *source, int source_len) {
if (!a || !source) {
return source;
}
char *out = NULL;
for (int i = 0; i + PAIR_LEN < source_len; i++) {
if (source[i] != '\\' || source[i + SKIP_ONE] != '\r' || source[i + PAIR_LEN] != '\n') {
continue;
}
if (!out) {
out = (char *)cbm_arena_alloc(a, (size_t)source_len + SKIP_ONE);
if (!out) {
return source;
}
memcpy(out, source, (size_t)source_len);
out[source_len] = '\0';
}
out[i] = ' ';
out[i + SKIP_ONE] = '\\';
i += PAIR_LEN;
}
return out ? out : source;
}

/* The bytes the grammar parses. PHP that leaves PHP mode (`?> markup <?php`)
* is parsed with its markup blanked (#2000); the rewrite keeps the length and
* every line break, so offsets and line numbers still refer to the file on
* disk. Every other language parses the file as it is. */
* is parsed with its markup blanked (#2000), and a C-family line splice before
* a CRLF is rewritten (#1546); both rewrites keep the length and every line
* break, so offsets and line numbers still refer to the file on disk. Every
* other language parses the file as it is. */
static const char *extract_parse_bytes(CBMArena *a, CBMLanguage language, const char *source,
int source_len) {
if (language == CBM_LANG_PHP) {
return cbm_php_mask_inline_html(a, source, source_len);
}
if (c_splice_lang(language)) {
return cbm_c_mask_crlf_splices(a, source, source_len);
}
return source;
}

Expand Down
5 changes: 5 additions & 0 deletions internal/cbm/cbm.h
Original file line number Diff line number Diff line change
Expand Up @@ -1429,6 +1429,11 @@ void cbm_extract_dbt(CBMExtractCtx *ctx);
// parse past `?>` (#2000). Same length and line breaks as the input; returns
// `source` itself when nothing needs rewriting. Defined in php_inline_html.c.
const char *cbm_php_mask_inline_html(CBMArena *arena, const char *source, int source_len);
// C-family source with each `\` CR LF line splice rewritten as space `\` LF so
// the grammars built on tree-sitter-c read it as they read the LF splice
// (#1546). Same length and line breaks as the input; returns `source` itself
// when there is no such splice. Defined in cbm.c.
const char *cbm_c_mask_crlf_splices(CBMArena *arena, const char *source, int source_len);
void cbm_extract_imports(CBMExtractCtx *ctx);
void cbm_extract_usages(CBMExtractCtx *ctx);
void cbm_extract_semantic(CBMExtractCtx *ctx);
Expand Down
7 changes: 7 additions & 0 deletions internal/cbm/lsp/c_lsp.c
Original file line number Diff line number Diff line change
Expand Up @@ -7228,6 +7228,10 @@ static bool c_lsp_cross_with_registry_run(CBMArena *arena, const char *source, i
const CBMFileResult *test_owners,
CBMLSPStatus *status) {
*status = CBM_LSP_COMPLETE;
/* The retained or re-read source is the file on disk; read it the way the
* per-file extract did (#1546), so the owner digest, the cached tree and
* node text agree. */
source = cbm_c_mask_crlf_splices(arena, source, source_len);
if ((test_owners && test_owners->has_test_definition_owners &&
source_origin != CBM_SOURCE_ORIGIN_RAW) ||
!cbm_test_definition_owners_match(test_owners, source, source_len, cpp_mode,
Expand Down Expand Up @@ -7319,6 +7323,9 @@ static bool c_lsp_cross_run(CBMArena *arena, const char *source, int source_len,
CBMSourceOrigin source_origin, const CBMFileResult *test_owners,
CBMLSPStatus *status) {
*status = CBM_LSP_COMPLETE;
/* Same bytes as the per-file extract (#1546); see
* c_lsp_cross_with_registry_run. */
source = cbm_c_mask_crlf_splices(arena, source, source_len);
if ((test_owners && test_owners->has_test_definition_owners &&
source_origin != CBM_SOURCE_ORIGIN_RAW) ||
!cbm_test_definition_owners_match(test_owners, source, source_len, cpp_mode,
Expand Down
84 changes: 84 additions & 0 deletions tests/test_c_lsp.c
Original file line number Diff line number Diff line change
Expand Up @@ -16903,6 +16903,89 @@ TEST(clsp_issue1527_cross_file_memo_failures_propagate) {
PASS();
}

/* #1546: the cross entry points get the file as it is on disk and must read a
* `\` CR LF string splice the way the per-file extract does. Read raw, the CR
* ends the escape, the LF ends the literal, and run() falls into the ERROR
* region with its call. LF is the control; per-file, shared-registry and
* batch entry each. */
TEST(clsp_issue1546_cross_entry_points_read_crlf_string_splice) {
static const char *const srcs[2] = {"struct S { int m() { return 1; } };\n"
"void Middle()\n"
"{\n"
" const wchar_t* s = L\"line 0\\r\\n\\\n"
"line 1\\r\\n\";\n"
"}\n"
"\n"
"void run() { S s; s.m(); }\n",
"struct S { int m() { return 1; } };\r\n"
"void Middle()\r\n"
"{\r\n"
" const wchar_t* s = L\"line 0\\r\\n\\\r\n"
"line 1\\r\\n\";\r\n"
"}\r\n"
"\r\n"
"void run() { S s; s.m(); }\r\n"};
CBMLSPDef defs[] = {
{.qualified_name = "test.S", .short_name = "S", .label = "Class", .lang = CBM_LANG_CPP},
{.qualified_name = "test.S.m",
.short_name = "m",
.label = "Method",
.receiver_type = "test.S",
.return_types = "int",
.lang = CBM_LANG_CPP},
};
bool resolved[2][3] = {{false}};
for (int e = 0; e < 2; e++) {
const char *src = srcs[e];
int len = (int)strlen(src);
for (int mode = 0; mode < 3; mode++) {
CBMArena a;
cbm_arena_init(&a);
CBMTypeRegistry reg;
cbm_registry_init(&reg, &a);
reg.fallback = cbm_c_build_cross_registry(&a, defs, 2);
CBMResolvedCallArray out = {0};
CBMBatchCLSPFile file = {.source = src,
.source_len = len,
.module_qn = "test",
.cpp_mode = true,
.defs = defs,
.def_count = 2};
if (mode == 0) {
(void)cbm_run_c_lsp_cross(&a, src, len, "test", true, defs, 2, NULL, NULL, 0, NULL,
&out);
} else if (mode == 1) {
(void)cbm_run_c_lsp_cross_with_registry(&a, src, len, "test", true, &reg, NULL,
NULL, 0, NULL, &out);
} else {
(void)cbm_batch_c_lsp_cross(&a, &file, 1, &out);
}
for (int i = 0; i < out.count; i++) {
const CBMResolvedCall *rc = &out.items[i];
if (rc->caller_qn && strcmp(rc->caller_qn, "test.run") == 0 && rc->callee_qn &&
strcmp(rc->callee_qn, "test.S.m") == 0) {
resolved[e][mode] = true;
}
}
cbm_arena_destroy(&a);
}
}
for (int e = 0; e < 2; e++) {
for (int mode = 0; mode < 3; mode++) {
if (!resolved[e][mode]) {
printf(" CRLF splice diagnostic: %s entry %d did not resolve run -> S.m\n",
e ? "CRLF" : "LF", mode);
}
}
}
for (int e = 0; e < 2; e++) {
for (int mode = 0; mode < 3; mode++) {
ASSERT_TRUE(resolved[e][mode]);
}
}
PASS();
}

/* A chain's LENGTH is not nesting depth (#1527 follow-up). An operator chain
* `a + a + ... + a` and a receiver chain `b.a().a()...` are evaluated link by
* link, so neither consumes the evaluator's depth guard: the file keeps its
Expand Down Expand Up @@ -17083,6 +17166,7 @@ SUITE(c_lsp) {
RUN_TEST(clsp_issue1527_preprocessed_depth_failure_is_reported);
RUN_TEST(registry_issue1527_index_preserves_registration_order);
RUN_TEST(clsp_issue1527_cross_file_memo_failures_propagate);
RUN_TEST(clsp_issue1546_cross_entry_points_read_crlf_string_splice);
RUN_TEST(clsp_issue1527_long_binary_chain_is_not_nesting);
RUN_TEST(clsp_issue1527_long_receiver_chain_is_not_nesting);
RUN_TEST(clsp_c_reassigned_function_pointer_calls_join_exact_occurrences);
Expand Down
137 changes: 137 additions & 0 deletions tests/test_extraction.c
Original file line number Diff line number Diff line change
Expand Up @@ -401,6 +401,140 @@ TEST(extract_cpp_real_in_body_error_still_flagged_issue1071) {
PASS();
}

/* #1546: the grammars built on tree-sitter-c read a backslash line splice
* inside a string literal as `\` plus one character. With CRLF line endings
* that character was the CR, the LF then ended the literal, and every
* function after it was lost to one ERROR region. */
static const CBMDefinition *splice_def(CBMFileResult *r, const char *name) {
for (int i = 0; i < r->defs.count; i++) {
if (r->defs.items[i].name && strcmp(r->defs.items[i].name, name) == 0) {
return &r->defs.items[i];
}
}
return NULL;
}

/* The reporter's file, byte for byte. */
TEST(extract_cpp_crlf_string_splice_keeps_later_functions_issue1546) {
CBMFileResult *r = extract("void Head()\r\n"
"{\r\n"
" int h = 0;\r\n"
"}\r\n"
"\r\n"
"void Middle()\r\n"
"{\r\n"
" const wchar_t* s = L\"line 0\\r\\n\\\r\n"
"line 1\\r\\n\";\r\n"
"}\r\n"
"\r\n"
"void TrailerOne()\r\n"
"{\r\n"
" int a = 1;\r\n"
"}\r\n"
"\r\n"
"void TrailerTwo()\r\n"
"{\r\n"
" int b = 2;\r\n"
"}\r\n",
CBM_LANG_CPP, "t", "repro.cpp");
ASSERT_NOT_NULL(r);
ASSERT_FALSE(r->has_error);
ASSERT_FALSE(r->parse_incomplete);
const CBMDefinition *middle = splice_def(r, "Middle");
const CBMDefinition *one = splice_def(r, "TrailerOne");
const CBMDefinition *two = splice_def(r, "TrailerTwo");
ASSERT_NOT_NULL(middle);
ASSERT_NOT_NULL(one);
ASSERT_NOT_NULL(two);
ASSERT_EQ((int)middle->start_line, 6);
ASSERT_EQ((int)middle->end_line, 10);
ASSERT_EQ((int)one->start_line, 12);
ASSERT_EQ((int)two->start_line, 17);
cbm_free_result(r);
PASS();
}

/* Every grammar built on tree-sitter-c has the same escape rule. */
TEST(extract_c_family_crlf_string_splice_parses_clean_issue1546) {
static const CBMLanguage langs[] = {CBM_LANG_C, CBM_LANG_CPP, CBM_LANG_CUDA,
CBM_LANG_OBJC, CBM_LANG_GLSL, CBM_LANG_HLSL,
CBM_LANG_ISPC, CBM_LANG_SLANG};
const char *src = "void Head()\r\n"
"{\r\n"
" int h = 0;\r\n"
"}\r\n"
"\r\n"
"void Middle()\r\n"
"{\r\n"
" const char *s = \"line 0\\\r\n"
"line 1\";\r\n"
"}\r\n"
"\r\n"
"void TrailerOne()\r\n"
"{\r\n"
" int a = 1;\r\n"
"}\r\n";
for (size_t i = 0; i < sizeof(langs) / sizeof(langs[0]); i++) {
CBMFileResult *r = extract(src, langs[i], "t", "splice.c");
ASSERT_NOT_NULL(r);
ASSERT_FALSE(r->parse_incomplete);
const CBMDefinition *middle = splice_def(r, "Middle");
const CBMDefinition *tail = splice_def(r, "TrailerOne");
ASSERT_NOT_NULL(middle);
ASSERT_NOT_NULL(tail);
ASSERT_EQ((int)middle->end_line, 10);
ASSERT_EQ((int)tail->start_line, 12);
cbm_free_result(r);
}
PASS();
}

/* Splices in a macro and a line comment, which the grammar already read with
* CRLF, still do; the preprocessed second pass sees the rewritten bytes too. */
TEST(extract_c_crlf_splices_in_macro_comment_and_string_issue1546) {
CBMFileResult *r = extract("#define LOG2(a, b) \\\r\n"
" do { log_it(a); \\\r\n"
" log_it(b); } while (0)\r\n"
"\r\n"
"// a comment that ends with a splice \\\r\n"
" and goes on here\r\n"
"\r\n"
"static const char *usage = \"usage: tool\\n\\\r\n"
" -h help\\n\";\r\n"
"\r\n"
"int helper(int x)\r\n"
"{\r\n"
" return x + 1;\r\n"
"}\r\n"
"\r\n"
"int main(void)\r\n"
"{\r\n"
" LOG2(\"a\", \"b\");\r\n"
" return helper(2);\r\n"
"}\r\n",
CBM_LANG_C, "t", "mix.c");
ASSERT_NOT_NULL(r);
ASSERT_FALSE(r->parse_incomplete);
ASSERT(has_def(r, "Macro", "LOG2"));
const CBMDefinition *usage = splice_def(r, "usage");
const CBMDefinition *helper = splice_def(r, "helper");
ASSERT_NOT_NULL(usage);
ASSERT_NOT_NULL(helper);
ASSERT_EQ((int)usage->end_line, 9);
ASSERT_EQ((int)helper->start_line, 11);
ASSERT_EQ((int)helper->end_line, 14);
for (int i = 0; i < r->calls.count; i++) {
const CBMCall *c = &r->calls.items[i];
if (c->callee_name && strcmp(c->callee_name, "helper") == 0) {
ASSERT_NOT_NULL(c->enclosing_func_qn);
ASSERT_NOT_NULL(strstr(c->enclosing_func_qn, "main"));
}
}
ASSERT(has_call(r, "log_it"));
cbm_free_result(r);
PASS();
}

/* #1735: the #1071 check's cost. cbm_subtract_macro_invocation_regions asked
* "is this region a macro call?" before "is it inside a function?", and the
* first question walked the source from byte 0 for every region: regions x
Expand Down Expand Up @@ -15745,6 +15879,9 @@ SUITE(extraction) {
RUN_TEST(extract_cpp_macros_issue375);
RUN_TEST(extract_cpp_functionlike_macro_type_arg_no_false_parse_partial_issue1071);
RUN_TEST(extract_cpp_real_in_body_error_still_flagged_issue1071);
RUN_TEST(extract_cpp_crlf_string_splice_keeps_later_functions_issue1546);
RUN_TEST(extract_c_family_crlf_string_splice_parses_clean_issue1546);
RUN_TEST(extract_c_crlf_splices_in_macro_comment_and_string_issue1546);
RUN_TEST(macro_check_reads_no_source_for_regions_outside_functions_issue1735);
RUN_TEST(macro_check_locates_lines_with_one_table_issue1735);
RUN_TEST(extract_gdscript_issue186);
Expand Down
Loading
Loading