From f5ec9ede3fa677eb7b95ef31cecf74e78143f564 Mon Sep 17 00:00:00 2001 From: Ondrej Mirtes Date: Fri, 10 Jul 2026 12:51:03 +0200 Subject: [PATCH] Skip doc-comment copying and presize token arrays when tokenizing The scanner copied every doc comment into CG(doc_comment) even in non-parser mode (token_get_all/PhpToken::tokenize/highlighting), where nothing consumes it and it is immediately discarded; only do it when compiling. Also presize the tokenizer result array (one token per ~5 source bytes) and the dedup hash instead of growing them by doubling. --- Zend/zend_language_scanner.l | 6 +++++- ext/tokenizer/tokenizer.c | 6 ++++-- 2 files changed, 9 insertions(+), 3 deletions(-) diff --git a/Zend/zend_language_scanner.l b/Zend/zend_language_scanner.l index bf6bbe7f9d90..475cd1986137 100644 --- a/Zend/zend_language_scanner.l +++ b/Zend/zend_language_scanner.l @@ -2540,7 +2540,11 @@ inline_char_handler: HANDLE_NEWLINES(yytext, yyleng); if (doc_com) { - CG(doc_comment) = zend_string_init(yytext, yyleng, 0); + /* Only the compiler consumes doc comments; tokenizing/highlighting + * would copy and then discard them. */ + if (PARSER_MODE()) { + CG(doc_comment) = zend_string_init(yytext, yyleng, 0); + } RETURN_OR_SKIP_TOKEN(T_DOC_COMMENT); } diff --git a/ext/tokenizer/tokenizer.c b/ext/tokenizer/tokenizer.c index fdbdd5ddfb61..c5af60107dcf 100644 --- a/ext/tokenizer/tokenizer.c +++ b/ext/tokenizer/tokenizer.c @@ -332,8 +332,10 @@ static bool tokenize(zval *return_value, zend_string *source, zend_class_entry * zend_prepare_string_for_scanning(&source_zval, ZSTR_EMPTY_ALLOC()); LANG_SCNG(yy_state) = yycINITIAL; - zend_hash_init(&interned_strings, 0, NULL, NULL, 0); - array_init(return_value); + zend_hash_init(&interned_strings, 128, NULL, NULL, 0); + /* Rough estimate: one token per ~5 source bytes; presizing avoids + * repeated doubling of the result array. */ + array_init_size(return_value, ZSTR_LEN(source) / 5 + 8); HashTable *return_value_ht = Z_ARRVAL_P(return_value);