From 1d9590cc12f5312d817c8a7516b08b1fe3ce64ae Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=D0=9B=D0=B5=D0=BE=D0=BD=D0=B8=D0=B4=20=D0=AE=D1=80=D1=8C?= =?UTF-8?q?=D0=B5=D0=B2=20=28Leonid=20Yuriev=29?= Date: Sat, 14 Nov 2026 22:47:12 +0300 Subject: [PATCH] fix(cobol): terminate LINE_PREFIX_COMMENT scan at EOF (worker livelock) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The vendored COBOL external scanner's LINE_PREFIX_COMMENT loop advanced while get_column() <= 5 without checking the lookahead. tree-sitter's lexer advance() is a no-op at EOF -- the column never increments -- so any input reaching EOF at column <= 5 spun the scanner forever at 100% CPU: a .cob shorter than 6 bytes ("X" suffices), or a trailing newline that leaves the lexer at column 0 of the virtual empty last line in the right parser state. On a full GCC 16 index this burned nine 15-minute quiet timeouts with extract-phase restarts, quarantining 7 .cob tests and 1 .cbl per run (#2998). Also stop emitting a zero-length COMMENT_ENTRY token when start_with_word() bailed at "\n"/EOF without consuming input -- a zero-length external token is a separate parser livelock class. Reproduce-first regression tests parse the EOF shapes in a forked child under a wall-clock deadline, so a regression is a RED row on the repro board instead of a hung CI job; a control re-extracts PROGRAM-ID from the canonical valid-program fixture. RED on unfixed main, GREEN here. Fixes #2998 Signed-off-by: Леонид Юрьев (Leonid Yuriev) --- .../cbm/vendored/grammars/cobol/scanner.c | 22 ++++- tests/repro/repro_extraction.c | 88 +++++++++++++++++++ 2 files changed, 107 insertions(+), 3 deletions(-) diff --git a/internal/cbm/vendored/grammars/cobol/scanner.c b/internal/cbm/vendored/grammars/cobol/scanner.c index b0fd352773..263e7bb6e6 100644 --- a/internal/cbm/vendored/grammars/cobol/scanner.c +++ b/internal/cbm/vendored/grammars/cobol/scanner.c @@ -31,9 +31,14 @@ char* any_content_keyword[] = { "procedure division", }; -static bool start_with_word( TSLexer *lexer, char *words[], int number_of_words) { +// consumed_any reports whether this call advanced the lexer at all, so the +// caller can refuse to emit a zero-length token (tree-sitter livelock). +static bool start_with_word(TSLexer *lexer, char *words[], int number_of_words, + bool *consumed_any) { + *consumed_any = false; while(lexer->lookahead == ' ' || lexer->lookahead == '\t') { lexer->advance(lexer, true); + *consumed_any = true; } char *keyword_pointer[number_of_words]; @@ -60,6 +65,7 @@ static bool start_with_word( TSLexer *lexer, char *words[], int number_of_words) if(all_match_failed) { for(; lexer->get_column(lexer) < 71 && lexer->lookahead != '\n' && lexer->lookahead != 0; lexer->advance(lexer, true)) { + *consumed_any = true; } return false; } @@ -83,6 +89,7 @@ static bool start_with_word( TSLexer *lexer, char *words[], int number_of_words) // next character lexer->advance(lexer, true); + *consumed_any = true; } return false; @@ -106,7 +113,10 @@ bool tree_sitter_COBOL_external_scanner_scan(void *payload, TSLexer *lexer, } if(valid_symbols[LINE_PREFIX_COMMENT] && lexer->get_column(lexer) <= 5) { - while(lexer->get_column(lexer) <= 5) { + // EOF guard: tree-sitter's advance() is a no-op at EOF (the column + // never increments), so without checking lookahead this loop spins + // forever when the input ends at column <= 5. + while(lexer->get_column(lexer) <= 5 && lexer->lookahead != 0) { lexer->advance(lexer, true); } lexer->result_symbol = LINE_PREFIX_COMMENT; @@ -143,7 +153,13 @@ bool tree_sitter_COBOL_external_scanner_scan(void *payload, TSLexer *lexer, } if(valid_symbols[COMMENT_ENTRY]) { - if(!start_with_word(lexer, any_content_keyword, number_of_comment_entry_keywords)) { + bool consumed_any = false; + if(!start_with_word(lexer, any_content_keyword, number_of_comment_entry_keywords, &consumed_any)) { + // Zero-length guard: emitting a token without consuming input + // livelocks the parser. Bail out and let tree-sitter recover. + if(!consumed_any) { + return false; + } lexer->mark_end(lexer); lexer->result_symbol = COMMENT_ENTRY; return true; diff --git a/tests/repro/repro_extraction.c b/tests/repro/repro_extraction.c index 99db6954db..aa77b7e9a5 100644 --- a/tests/repro/repro_extraction.c +++ b/tests/repro/repro_extraction.c @@ -8,9 +8,16 @@ * Cluster (TIER A, in-process via cbm_extract_file): * #554 — C++ out-of-line method CALLS source = Module, not enclosing Method * (more added per wave: #495 #521 #382 #408 #523 #56 #333) + * #2998 — COBOL external scanner livelock at EOF (hang, not a wrong edge) */ #include "test_framework.h" #include "cbm.h" +#include +#if !defined(_WIN32) +#include +#include +#include +#endif /* Convenience: extract, return result (caller frees). Mirrors test_extraction.c. */ static CBMFileResult *rx(const char *src, CBMLanguage lang, const char *proj, const char *path) { @@ -87,7 +94,88 @@ TEST(repro_issue554_cpp_out_of_line_method_class_qualified) { PASS(); } +/* ─────────────────────────────────────────────────────────────────── + * #2998 — COBOL external scanner livelocks the parser at EOF. + * + * Root cause: the vendored COBOL scanner's LINE_PREFIX_COMMENT loop advanced + * while get_column() <= 5 without checking lookahead == 0. tree-sitter's + * advance() is a no-op at EOF (the column never increments), so any input that + * reaches EOF at column <= 5 — e.g. a .cob file shorter than 6 bytes, or the + * virtual empty line after a trailing "\n" in the right parser state — spun + * the scanner forever at 100% CPU. A second hazard emitted a zero-length + * COMMENT_ENTRY token when start_with_word() bailed at "\n"/EOF without + * consuming, another livelock class. + * + * Expected: extraction RETURNS (the hang itself is the failure mode; the + * fork+deadline harness below turns a regression back into a RED test instead + * of a hung CI job), and the canonical valid-program fixture still extracts + * PROGRAM-ID. Actual (buggy): never returns; last worker-log line + * "pass.start pass=definitions". + * ─────────────────────────────────────────────────────────────────── */ +#if !defined(_WIN32) +/* Parse src in a forked child under a wall-clock deadline. + * Returns the child's exit code (0 = extraction returned), -1 on timeout + * (child killed; i.e. the livelock reproduced), -2 on fork failure. */ +static int repro2998_parse_with_deadline(const char *src, CBMLanguage lang, const char *path) { + fflush(NULL); + pid_t pid = fork(); + if (pid < 0) + return -2; + if (pid == 0) { + CBMFileResult *r = cbm_extract_file(src, (int)strlen(src), lang, "t", path, 0, NULL, NULL); + if (r) + cbm_free_result(r); + _exit(0); /* reaching _exit at all is what the bug took away */ + } + int status = 0; + for (int i = 0; i < 1000; i++) { /* 10 s in 10 ms steps */ + pid_t got = waitpid(pid, &status, WNOHANG); + if (got == pid) + return WIFEXITED(status) ? WEXITSTATUS(status) : 128 + WTERMSIG(status); + usleep(10 * 1000); + } + kill(pid, SIGKILL); + waitpid(pid, &status, 0); + return -1; +} + +TEST(repro_issue2998_cobol_scanner_eof_livelock) { + /* EOF lands inside the LINE_PREFIX_COMMENT column scan: shorter than + * column 6, or a trailing newline leaving the lexer at column 0 of the + * virtual empty last line. */ + ASSERT_EQ(repro2998_parse_with_deadline("X", CBM_LANG_COBOL, "m.cob"), 0); + ASSERT_EQ(repro2998_parse_with_deadline("*<<", CBM_LANG_COBOL, "m.cob"), 0); + ASSERT_EQ(repro2998_parse_with_deadline("X\n", CBM_LANG_COBOL, "m.cob"), 0); + ASSERT_EQ(repro2998_parse_with_deadline("display 1.\n", CBM_LANG_COBOL, "m.cob"), 0); + PASS(); +} + +TEST(repro_issue2998_cobol_valid_program_still_extracts) { + /* The EOF/zero-length guards must not break the happy path: the canonical + * fixed-format fixture (same as the cobol row in grammar_cases.h) still + * yields its PROGRAM-ID definition. */ + CBMFileResult *r = rx(" IDENTIFICATION DIVISION.\n" + " PROGRAM-ID. FOO.\n" + " PROCEDURE DIVISION.\n" + " MAIN-PARA.\n" + " STOP RUN.\n", + CBM_LANG_COBOL, "t", "a.cob"); + ASSERT_NOT_NULL(r); + int saw_foo = 0; + for (int i = 0; r && i < r->defs.count; i++) + if (r->defs.items[i].name && strcmp(r->defs.items[i].name, "FOO") == 0) + saw_foo = 1; + cbm_free_result(r); + ASSERT_TRUE(saw_foo); + PASS(); +} +#endif /* !_WIN32 */ + /* ── Suite ──────────────────────────────────────────────────────── */ SUITE(repro_extraction) { RUN_TEST(repro_issue554_cpp_out_of_line_method_class_qualified); +#if !defined(_WIN32) + RUN_TEST(repro_issue2998_cobol_scanner_eof_livelock); + RUN_TEST(repro_issue2998_cobol_valid_program_still_extracts); +#endif }