diff --git a/internal/cbm/extract_defs.c b/internal/cbm/extract_defs.c index deee8a8e7..03637d709 100644 --- a/internal/cbm/extract_defs.c +++ b/internal/cbm/extract_defs.c @@ -1192,7 +1192,18 @@ static bool is_comment_node(const char *kind) { static char *extract_comment_text(CBMArena *a, TSNode node, const char *source) { char *text = cbm_node_text(a, node, source); if (text && strlen(text) > MAX_COMMENT_LEN) { - text[MAX_COMMENT_LEN] = '\0'; + /* #1017: snap the cut back to a UTF-8 character boundary so a + * multi-byte character straddling MAX_COMMENT_LEN (e.g. a 3-byte + * CJK char in a Japanese docstring) is never split — a byte-offset + * cut leaves an incomplete sequence that lands as invalid UTF-8 in + * nodes.properties, and strict decoders (Python's sqlite3) throw on + * read. Continuation bytes are 10xxxxxx; back up over them to the + * lead byte, dropping the partial character. */ + size_t cut = MAX_COMMENT_LEN; + while (cut > 0 && ((unsigned char)text[cut] & 0xC0) == 0x80) { + cut--; + } + text[cut] = '\0'; } return text; } diff --git a/tests/test_extraction.c b/tests/test_extraction.c index 0598cadb7..1c6bf02bd 100644 --- a/tests/test_extraction.c +++ b/tests/test_extraction.c @@ -1917,6 +1917,71 @@ TEST(wolfram_nested_def) { * Group I: cbm_test.go ports * ═══════════════════════════════════════════════════════════════════ */ +/* #1017: a docstring longer than MAX_COMMENT_LEN (500) that has a multi-byte + * UTF-8 character straddling the cap must be truncated at a CHARACTER + * boundary, never mid-sequence — a byte-offset cut leaves an incomplete + * UTF-8 sequence that lands as invalid UTF-8 in nodes.properties and makes + * strict decoders (Python sqlite3) throw. Verified by decoding the captured + * docstring as UTF-8. */ +static int is_valid_utf8(const char *s) { + const unsigned char *p = (const unsigned char *)s; + while (*p) { + int n; + if (*p < 0x80) { + n = 0; + } else if ((*p & 0xE0) == 0xC0) { + n = 1; + } else if ((*p & 0xF0) == 0xE0) { + n = 2; + } else if ((*p & 0xF8) == 0xF0) { + n = 3; + } else { + return 0; /* invalid lead byte (or stray continuation) */ + } + p++; + for (int i = 0; i < n; i++) { + if ((*p & 0xC0) != 0x80) { + return 0; /* missing/short continuation → truncated mid-char */ + } + p++; + } + } + return 1; +} + +TEST(python_docstring_utf8_boundary_truncation_issue1017) { + /* 495 ASCII bytes then 3-byte CJK chars (E3 81 82 = U+3042 あ): the cap at + * byte 500 lands in the middle of the second あ. */ + char src[900]; + int off = snprintf(src, sizeof(src), "def f():\n \"\"\""); + for (int i = 0; i < 495; i++) { + src[off++] = 'x'; + } + for (int i = 0; i < 5; i++) { + src[off++] = (char)0xE3; + src[off++] = (char)0x81; + src[off++] = (char)0x82; + } + off += snprintf(src + off, sizeof(src) - off, "\"\"\"\n return 1\n"); + (void)off; + CBMFileResult *r = extract(src, CBM_LANG_PYTHON, "test", "test.py"); + ASSERT_NOT_NULL(r); + const char *doc = NULL; + for (int i = 0; i < r->defs.count; i++) { + if (r->defs.items[i].name && strcmp(r->defs.items[i].name, "f") == 0) { + doc = r->defs.items[i].docstring; + } + } + ASSERT_NOT_NULL(doc); + if (!is_valid_utf8(doc)) { + fprintf(stderr, " [1017] FAIL docstring truncated mid-UTF-8-char (len=%zu)\n", + strlen(doc)); + } + ASSERT_TRUE(is_valid_utf8(doc)); + cbm_free_result(r); + PASS(); +} + TEST(python_docstring) { CBMFileResult *r = extract( "def compute(x, y):\n \"\"\"Compute the sum of x and y.\"\"\"\n return x + y\n", @@ -4723,6 +4788,7 @@ SUITE(extraction) { RUN_TEST(wolfram_nested_def); /* cbm_test.go ports */ + RUN_TEST(python_docstring_utf8_boundary_truncation_issue1017); RUN_TEST(python_docstring); RUN_TEST(go_function_extraction); RUN_TEST(js_arrow_function);