From 16338dcbe8c04209badcf123d23eef86a64517fe Mon Sep 17 00:00:00 2001 From: joyguoguo <599325370@qq.com> Date: Sun, 20 Jul 2025 23:03:36 +0000 Subject: [PATCH 01/15] feat: Add OSS-Fuzz submodule tracking main branch --- .gitmodules | 4 ++++ fuzz/oss-fuzz | 1 + 2 files changed, 5 insertions(+) create mode 160000 fuzz/oss-fuzz diff --git a/.gitmodules b/.gitmodules index 5b761d3..1683a3c 100644 --- a/.gitmodules +++ b/.gitmodules @@ -1,3 +1,7 @@ [submodule "UniTSyn"] path = UniTSyn url = https://github.com/SecurityLab-UCD/UniTSyn.git +[submodule "fuzz/oss-fuzz"] + path = fuzz/oss-fuzz + url = https://github.com/google/oss-fuzz.git + branch = main diff --git a/fuzz/oss-fuzz b/fuzz/oss-fuzz new file mode 160000 index 0000000..1351fe0 --- /dev/null +++ b/fuzz/oss-fuzz @@ -0,0 +1 @@ +Subproject commit 1351fe0fbefb2965b64d51411562fa6df86bcc86 From ca103e9263e14c51cff07edece6da79b638a683e Mon Sep 17 00:00:00 2001 From: joyguoguo <599325370@qq.com> Date: Mon, 21 Jul 2025 06:46:24 +0000 Subject: [PATCH 02/15] chore: Switch oss-fuzz submodule to personal fork --- fuzz/oss-fuzz | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fuzz/oss-fuzz b/fuzz/oss-fuzz index 1351fe0..c3d3fb7 160000 --- a/fuzz/oss-fuzz +++ b/fuzz/oss-fuzz @@ -1 +1 @@ -Subproject commit 1351fe0fbefb2965b64d51411562fa6df86bcc86 +Subproject commit c3d3fb70f9ece4d31e623ce0f29bc0dfd34e0af6 From f39e72744df2d08f3c3349bb8ee80520dd87c77d Mon Sep 17 00:00:00 2001 From: joyguoguo <599325370@qq.com> Date: Mon, 21 Jul 2025 06:47:24 +0000 Subject: [PATCH 03/15] Switch oss-fuzz submodule to personal fork --- .gitmodules | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitmodules b/.gitmodules index 1683a3c..82926c7 100644 --- a/.gitmodules +++ b/.gitmodules @@ -3,5 +3,5 @@ url = https://github.com/SecurityLab-UCD/UniTSyn.git [submodule "fuzz/oss-fuzz"] path = fuzz/oss-fuzz - url = https://github.com/google/oss-fuzz.git + url = https://github.com/joyguoguo/oss-fuzz.git branch = main From d691eeabc9996e62824814d22caee45ef8eed2ff Mon Sep 17 00:00:00 2001 From: joyguoguo <599325370@qq.com> Date: Mon, 21 Jul 2025 23:48:08 +0000 Subject: [PATCH 04/15] move the valid_project file --- data/valid_projects.txt | 234 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 234 insertions(+) create mode 100644 data/valid_projects.txt diff --git a/data/valid_projects.txt b/data/valid_projects.txt new file mode 100644 index 0000000..94fdcd5 --- /dev/null +++ b/data/valid_projects.txt @@ -0,0 +1,234 @@ +abseil-py +adal +aiohttp +aniso8601 +ansible +argcomplete +arrow-py +asn1crypto +asteval +astroid +asttokens +attrs +autoflake +autopep8 +azure-sdk-for-python +babel +black +botocore +bottleneck +bz2file +cachetools +cffi +chardet +charset_normalizer +click +cloud-custodian +configparser +connexion +coveragepy +croniter +cryptography +cssselect +dask +decorator +defusedxml +digest +dill +distlib +dnspython +docutils +ecdsa-python +et-xmlfile +face +filelock +filesystem_spec +flask +flask-jwt-extended +flask-restx +flask-wtf +fonttools +ftfy +g-api-auth-httplib2 +g-api-auth-library-python +g-api-pubsub +g-api-py-api-common-protos +g-api-py-oauthlib +g-api-python-bigquery-storage +g-api-python-client +g-api-python-cloud-core +g-api-python-firestore +g-api-python-tasks +g-api-resource-manager +g-api-resumable-media-python +g-api-secret-manager +g-apis-py-api-core +gast +gc-iam +gcloud-error-py +g-cloud-logging-py +gcp-python-cloud-storage +genshi +gitdb +glom +gprof2dot +g-py-bigquery +g-py-crc32c +grpc-py +gunicorn +h11 +h5py +hiredis-py +html2text +html5lib-python +httpcore +httpretty +httpx +idna +ijson +importlib_metadata +iniconfig +ipaddress +ipykernel +ipython +isodate +itsdangerous +jedi +jinja2 +jmespathpy +joblib +jsmin +jupyter-nbconvert +jupyter_server +kafka +keras +kiwisolver +lark-parser +libcst +looker-sdk +lxml +mako +markupsafe +matplotlib +mccabe +mdit-py-plugins +mdurl +more-itertools +mrab-regex +msal +msgpack-python +multidict +mutagen +nbclassic +nbformat +netaddr-py +networkx +ntlm2 +ntlm-auth +numexpr +numpy +oauth2 +oauthlib +olefile +openapi-schema-validator +opencensus-python +openpyxl +opt_einsum +oracle-py-cx +orjson +oscrypto +packaging +pandas +paramiko +parse +parsimonious +pasta +pathlib2 +pdoc +pem +pendulum +pip +ply +protobuf-python +proto-plus-python +psqlparse +psutil +psycopg2 +pyasn1 +pyasn1-modules +pycparser +pycrypto +pydantic +pydateutil +pygments +pyjson5 +pyjwt +pymysql +pynacl +pyodbc +pyparsing +pyrsistent +py-serde +pytables +pytest-py +python3-openid +python-ecdsa +python-email-validator +python-fastjsonschema +python-future +python-graphviz +python-hyperlink +python-jose +python-lz4 +python-markdown +python-markdownify +python-nameparser +python-nvd3 +python-pathspec +python-prompt-toolkit +python-pypdf +python-rison +python-rsa +python-tabulate +pytz +pyxdg +pyyaml +pyzmq +redis-py +requests +retry +rfc3967 +rich +sacremoses +scikit-learn +scipy +setuptools +sigstore-python +simplejson +six +smart_open +soupsieve +sqlalchemy_jsonfield +sqlalchemy-utils +sqlparse +stack_data +tensorflow-addons +tinycss2 +toml +tomlkit +toolbelt +toolz +tqdm +typing_extensions +underscore +uritemplate +urlextract +urllib3 +validators +w3lib +websocket-client +wheel +wtforms +xlrd +yarl +zipp From 25b01915000c45531be9e07a71cfe6d45963441f Mon Sep 17 00:00:00 2001 From: joyguoguo <599325370@qq.com> Date: Mon, 21 Jul 2025 23:48:36 +0000 Subject: [PATCH 05/15] move the .py file --- fuzz/fuzz_runner_pool.py | 270 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 270 insertions(+) create mode 100644 fuzz/fuzz_runner_pool.py diff --git a/fuzz/fuzz_runner_pool.py b/fuzz/fuzz_runner_pool.py new file mode 100644 index 0000000..abd4b30 --- /dev/null +++ b/fuzz/fuzz_runner_pool.py @@ -0,0 +1,270 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- + +""" +fuzz_runner_pool.py + +并行批量执行 OSS-Fuzz 本地测试全流程。使用 multiprocessing.Pool 将项目 +分发到多个 CPU核心上同时处理。 + +用法: python3 fuzz_runner_pool.py [项目列表文件] [--sanitizer 类型] [--workers N] +示例: python3 fuzz_runner_pool.py /home/jiayiguo/FuzzAug/data/valid_projects.txt --workers 4 +""" + +import os +import sys +import subprocess +import argparse +from datetime import datetime +from pathlib import Path +from typing import List, Optional, Tuple +from multiprocessing import Pool, cpu_count + +# --- 全局配置 (可通过命令行参数覆盖) --- +HOME_DIR = Path.home() +OSS_FUZZ_DIR = HOME_DIR / "FuzzAug" / "fuzz"/"oss-fuzz" +LOG_DIR = OSS_FUZZ_DIR / "script_pool_batch_logs" + +def setup_logging(project_name: str) -> Path: + """为单个项目创建带时间戳的日志文件.""" + LOG_DIR.mkdir(parents=True, exist_ok=True) + try: + LOG_DIR.chmod(0o777) + except PermissionError: + # 在并行环境中，这里可能会有多个进程同时尝试，打印一次警告即可 + pass + + timestamp = datetime.now().strftime("%Y%m%d%H%M%S") + log_file_path = LOG_DIR / f"oss_fuzz_{project_name}_{timestamp}.log" + return log_file_path + +def log_and_print(message: str, log_file: Path, to_stdout: bool = True): + """将消息写入日志文件，并根据需要打印到控制台。""" + if to_stdout: + # 添加进程ID，以便区分并行输出 + print(f"[PID:{os.getpid()}] {message}") + with open(log_file, "a", encoding="utf-8") as f: + f.write(message + "\n") + +def run_command( + cmd: str, + log_msg: str, + log_file: Path, + allowed_exit_codes: Optional[List[int]] = None, + auto_confirm: bool = True # 新增自动确认参数 +) -> bool: + """执行一个 shell 命令，并将输出实时流式传输到日志文件。""" + if allowed_exit_codes is None: + allowed_exit_codes = [] + + log_and_print(f"▶️ {log_msg}...", log_file, to_stdout=False) + log_and_print(f" $ {cmd}", log_file, to_stdout=False) + + try: + # 添加自动确认机制 + if auto_confirm: + cmd = f"yes | {cmd}" + + process = subprocess.Popen( + cmd, shell=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, + text=True, encoding='utf-8', errors='replace', bufsize=1 + ) + + with open(log_file, "a", encoding="utf-8") as f: + for line in iter(process.stdout.readline, ''): + f.write(line) # 只写入日志，避免控制台输出混乱 + + process.wait() + exit_code = process.returncode + + if exit_code == 0: + log_and_print(f"✅ 命令成功完成。", log_file, to_stdout=False) + return True + elif exit_code in allowed_exit_codes: + log_and_print(f"ℹ️ 命令以预期状态退出: {exit_code}", log_file, to_stdout=False) + return True + else: + log_and_print(f"❌ 命令执行失败 (退出码: {exit_code})", log_file) + return False + except Exception as e: + log_and_print(f"💥 执行命令时发生异常: {e}", log_file) + return False + +def discover_fuzz_targets(project_name: str) -> List[str]: + """自动发现 Fuzz 目标。""" + project_out_dir = OSS_FUZZ_DIR / "build" / "out" / project_name + project_src_dir = OSS_FUZZ_DIR / "projects" / project_name + targets = [] + + if project_out_dir.is_dir(): + for f in project_out_dir.iterdir(): + if (f.is_file() and os.access(f, os.X_OK) and + f.name.startswith("fuzz_") and '.' not in f.name): + targets.append(f.name) + + if not targets and project_src_dir.is_dir(): + for py_file in project_src_dir.glob("fuzz_*.py"): + try: + with open(py_file, "r", encoding="utf-8") as f: + if "atheris.Setup" in f.read(): + targets.append(py_file.stem) + except Exception as e: + # 在worker进程中，只打印到自己的日志 + print(f"⚠️ 警告: 读取文件 {py_file} 失败: {e}") + return targets + +def run_project_workflow(project_name: str, sanitizer: str) -> Tuple[bool, str]: + """ + 处理单个项目的完整工作流 (Worker Function)。 + 此函数由进程池中的每个工作进程独立执行。 + + Returns: + 一个元组 (is_success: bool, project_name: str) + """ + log_file = setup_logging(project_name) + + # 在 worker 的开头打印，以便追踪 + log_and_print("=" * 60, log_file) + log_and_print(f"🚀 开始处理项目: {project_name}", log_file) + log_and_print(f"📝 日志文件: {log_file}", log_file) + log_and_print("=" * 60, log_file) + + # 每个进程都需要设置自己的工作目录 + try: + os.chdir(OSS_FUZZ_DIR) + except FileNotFoundError: + log_and_print(f"❌ 严重错误: OSS-Fuzz 目录 '{OSS_FUZZ_DIR}' 不存在！", log_file) + return (False, project_name) + + # 步骤 1: 构建Docker镜像（启用自动确认） + if not run_command( + f"python3 infra/helper.py build_image {project_name}", + f"步骤1/5: 构建 {project_name} 的Docker镜像", log_file, + auto_confirm=True # 自动确认所有提示 + ): + log_and_print(f"❌ 项目 {project_name} 构建镜像失败", log_file) + return (False, project_name) + + # 步骤 2: 编译带检测器的fuzzer（启用自动确认） + if not run_command( + f"python3 infra/helper.py build_fuzzers --sanitizer {sanitizer} {project_name}", + f"步骤2/5: 编译 {project_name} 的fuzzer (sanitizer={sanitizer})", log_file, + auto_confirm=True # 自动确认所有提示 + ): + log_and_print(f"❌ 项目 {project_name} 编译fuzzer失败", log_file) + return (False, project_name) + + # 步骤 3: 自动发现目标 + log_and_print(f"🔍 正在为 {project_name} 自动发现fuzz目标...", log_file) + fuzz_targets = discover_fuzz_targets(project_name) + + if not fuzz_targets: + log_and_print(f"⚠️ 警告: {project_name} 未找到任何fuzz目标！跳过运行步骤。", log_file) + return (True, project_name) + + log_and_print(f"✅ 发现目标: {', '.join(fuzz_targets)}", log_file) + + # 步骤 4: 遍历运行所有目标（启用自动确认） + for i, target in enumerate(fuzz_targets, 1): + run_command( + f"python3 infra/helper.py run_fuzzer {project_name} {target} -- -max_total_time=60", + f"步骤4/{len(fuzz_targets)}: 运行目标 [{target}] (60秒)", log_file, + allowed_exit_codes=[1, 124], + auto_confirm=True # 自动确认所有提示 + ) + + # 步骤 5: 生成覆盖率报告 (暂无) + log_and_print("步骤5/5: 生成覆盖率报告 (当前版本暂未实现)", log_file) + log_and_print(f"✅ 项目 {project_name} 处理完成！", log_file) + return (True, project_name) + +def main(): + """ + 主流程函数：设置进程池并分发任务。 + """ + parser = argparse.ArgumentParser( + description="OSS-Fuzz 并行批量测试工具", + formatter_class=argparse.RawTextHelpFormatter, + epilog="示例:\n python3 fuzz_runner_parallel.py valid_projects.txt --workers 4\n python3 fuzz_runner_parallel.py my_projects.txt --sanitizer undefined" + ) + parser.add_argument( + "project_list_file", nargs="?", default="valid_projects.txt", + help="包含待测试项目列表的文本文件。(默认: valid_projects.txt)" + ) + parser.add_argument( + "--sanitizer", default="address", choices=["address", "memory", "undefined", "coverage"], + help="要使用的 sanitizer 类型。(默认: address)" + ) + parser.add_argument( + "--workers", type=int, default=cpu_count(), + help=f"并发执行的工作进程数。(默认: 系统CPU核心数, 即 {cpu_count()})" + ) + args = parser.parse_args() + + # --- 环境检查 --- + if not OSS_FUZZ_DIR.is_dir(): + print(f"❌ 错误: OSS-Fuzz 目录 '{OSS_FUZZ_DIR}' 不存在！") + sys.exit(1) + + project_list_path = Path(args.project_list_file) + if not project_list_path.is_file(): + print(f"❌ 错误: 项目列表文件 '{project_list_path}' 不存在！") + sys.exit(1) + + print(f"✅ 环境检查通过。将使用 {args.workers} 个并行工作进程。") + + # --- 读取和准备任务 --- + try: + with open(project_list_path, "r", encoding="utf-8") as f: + projects = [line.strip() for line in f if line.strip() and not line.strip().startswith("#")] + except Exception as e: + print(f"❌ 读取项目列表文件时出错: {e}") + sys.exit(1) + + if not projects: + print("⚠️ 警告: 项目列表为空，无可执行任务。") + sys.exit(0) + + # 为 starmap 准备任务参数列表，每个项目都是一个 (project_name, sanitizer) 元组 + tasks = [(project, args.sanitizer) for project in projects] + total_projects = len(tasks) + + print(f"\n🚀 即将并行处理 {total_projects} 个项目...") + + # --- 执行并行处理 --- + # 使用 with 语句确保进程池被正确关闭 + with Pool(processes=args.workers) as pool: + # starmap 会阻塞直到所有任务完成 + # 它将 tasks 列表中的每个元组解包作为参数传递给 worker 函数 + results = pool.starmap(run_project_workflow, tasks) + + # --- 收集并打印结果 --- + failed_projects = [] + for success, project_name in results: + if success: + print(f"✅ 项目 {project_name} 成功完成") + else: + print(f"❌ 项目 {project_name} 处理失败") + failed_projects.append(project_name) + + # --- 最终总结 --- + fail_count = len(failed_projects) + success_count = total_projects - fail_count + + print("\n" + "=" * 60) + print("🎉 批量处理完成！") + print(f"📊 总计: {total_projects} 个项目") + print(f"✅ 成功: {success_count}") + print(f"❌ 失败: {fail_count}") + + if failed_projects: + print("📛 失败项目列表:") + for proj in sorted(failed_projects): + print(f" • {proj}") + print("\n💡 提示: 失败项目的详细信息请查看对应的日志文件。") + print(f" 日志目录: {LOG_DIR}") + +if __name__ == "__main__": + # 在 Windows 或 macOS 的某些 Python 版本上，需要将 main 调用放在这个保护块中 + # 以防止子进程重新导入和执行主模块代码，导致无限递归。 + main() \ No newline at end of file From fcf80a94403de8b1f995b56907b410ebc4537cfd Mon Sep 17 00:00:00 2001 From: joyguoguo <599325370@qq.com> Date: Mon, 21 Jul 2025 23:50:42 +0000 Subject: [PATCH 06/15] create build_oss_fuzz.py --- fuzz/build_oss_fuzz.py | 131 +++++++++++++++++++++++++++++++++++++++++ 1 file changed, 131 insertions(+) create mode 100644 fuzz/build_oss_fuzz.py diff --git a/fuzz/build_oss_fuzz.py b/fuzz/build_oss_fuzz.py new file mode 100644 index 0000000..525c556 --- /dev/null +++ b/fuzz/build_oss_fuzz.py @@ -0,0 +1,131 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- + +""" +build_oss_fuzz.py + +并行构建 OSS-Fuzz 项目（Docker 镜像和 Fuzzer 编译）。 +使用 multiprocessing.Pool 将项目分发到多个 CPU 核心同时处理。 + +用法: python3 build_oss_fuzz.py [项目列表文件] [--sanitizer 类型] [--workers N] +示例: python3 build_oss_fuzz.py valid_projects.txt --sanitizer address --workers 4 +""" + +import os +import sys +import subprocess +import argparse +from datetime import datetime +from pathlib import Path +from typing import List, Optional, Tuple +from multiprocessing import Pool, cpu_count + +# --- 全局配置 --- +HOME_DIR = Path.home() +OSS_FUZZ_DIR = HOME_DIR / "FuzzAug" / "fuzz" / "oss-fuzz" +LOG_DIR = OSS_FUZZ_DIR / "build_logs" + +def setup_logging(project_name: str) -> Path: + """为单个项目创建带时间戳的日志文件""" + LOG_DIR.mkdir(parents=True, exist_ok=True) + timestamp = datetime.now().strftime("%Y%m%d%H%M%S") + return LOG_DIR / f"build_{project_name}_{timestamp}.log" + +def log_and_print(message: str, log_file: Path, to_stdout: bool = True): + """将消息写入日志并打印到控制台""" + if to_stdout: + print(f"[PID:{os.getpid()}] {message}") + with open(log_file, "a", encoding="utf-8") as f: + f.write(f"{datetime.now().isoformat()} {message}\n") + +def run_command( + cmd: str, + log_msg: str, + log_file: Path, + allowed_exit_codes: Optional[List[int]] = None +) -> bool: + """执行 shell 命令并实时记录输出""" + allowed_exit_codes = allowed_exit_codes or [] + log_and_print(f"▶️ {log_msg}...", log_file, to_stdout=False) + log_and_print(f" $ {cmd}", log_file, to_stdout=False) + + try: + process = subprocess.Popen( + f"yes | {cmd}", # 自动确认所有提示 + shell=True, + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + text=True, + encoding="utf-8", + errors="replace" + ) + with open(log_file, "a", encoding="utf-8") as f: + for line in iter(process.stdout.readline, ""): + f.write(line) + process.wait() + exit_code = process.returncode + if exit_code in [0, *allowed_exit_codes]: + log_and_print(f"✅ 命令成功完成", log_file, to_stdout=False) + return True + log_and_print(f"❌ 命令失败 (退出码: {exit_code})", log_file) + return False + except Exception as e: + log_and_print(f"💥 执行异常: {e}", log_file) + return False + +def build_project(project_name: str, sanitizer: str) -> Tuple[bool, str]: + """单个项目的构建工作流""" + log_file = setup_logging(project_name) + os.chdir(OSS_FUZZ_DIR) + + log_and_print("="*60, log_file) + log_and_print(f"🔨 开始构建项目: {project_name}", log_file) + log_and_print(f"📝 日志路径: {log_file}", log_file) + log_and_print("="*60, log_file) + + # 1. 构建 Docker 镜像 + if not run_command( + f"python3 infra/helper.py build_image {project_name}", + "步骤1/2: 构建 Docker 镜像", + log_file + ): + return (False, project_name) + + # 2. 编译 Fuzzer + if not run_command( + f"python3 infra/helper.py build_fuzzers --sanitizer {sanitizer} {project_name}", + f"步骤2/2: 编译 Fuzzer (sanitizer={sanitizer})", + log_file + ): + return (False, project_name) + + log_and_print(f"✅ 项目 {project_name} 构建完成", log_file) + return (True, project_name) + +def main(): + parser = argparse.ArgumentParser(description="OSS-Fuzz 并行构建工具") + parser.add_argument("project_list", help="项目列表文件路径") + parser.add_argument("--sanitizer", default="address", choices=["address", "memory", "undefined"]) + parser.add_argument("--workers", type=int, default=cpu_count()) + args = parser.parse_args() + + # 读取项目列表 + try: + with open(args.project_list, "r") as f: + projects = [line.strip() for line in f if line.strip()] + except Exception as e: + print(f"❌ 读取项目列表失败: {e}") + sys.exit(1) + + # 并行构建 + with Pool(args.workers) as pool: + results = pool.starmap(build_project, [(p, args.sanitizer) for p in projects]) + + # 输出结果 + failed = [p for success, p in results if not success] + print(f"\n📊 构建完成: 成功 {len(projects)-len(failed)}/{len(projects)}") + if failed: + print("❌ 失败项目: " + ", ".join(failed)) + +if __name__ == "__main__": + main() \ No newline at end of file From 3e8e7f4ca758b2f42ab8df7f2c371fadf4b6c06c Mon Sep 17 00:00:00 2001 From: joyguoguo <599325370@qq.com> Date: Mon, 21 Jul 2025 23:53:27 +0000 Subject: [PATCH 07/15] create run_fuzz_target.py --- fuzz/oss-fuzz | 2 +- fuzz/run_fuzz_target.py | 140 ++++++++++++++++++++++++++++++++++++++++ 2 files changed, 141 insertions(+), 1 deletion(-) create mode 100644 fuzz/run_fuzz_target.py diff --git a/fuzz/oss-fuzz b/fuzz/oss-fuzz index c3d3fb7..0b81ba5 160000 --- a/fuzz/oss-fuzz +++ b/fuzz/oss-fuzz @@ -1 +1 @@ -Subproject commit c3d3fb70f9ece4d31e623ce0f29bc0dfd34e0af6 +Subproject commit 0b81ba5d97ae3d1402744e00b1d9075fed7b7f1e diff --git a/fuzz/run_fuzz_target.py b/fuzz/run_fuzz_target.py new file mode 100644 index 0000000..ae7b036 --- /dev/null +++ b/fuzz/run_fuzz_target.py @@ -0,0 +1,140 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- + +""" +run_fuzz_target.py + +并行运行 OSS-Fuzz 测试目标。 +使用 multiprocessing.Pool 分发任务到多个 CPU 核心。 + +用法: python3 run_fuzz_target.py [项目列表文件] [--timeout 秒] [--workers N] +示例: python3 run_fuzz_target.py valid_projects.txt --timeout 120 --workers 4 +""" + +import os +import sys +import subprocess +import argparse +from datetime import datetime +from pathlib import Path +from typing import List, Optional, Tuple +from multiprocessing import Pool, cpu_count + +# --- 全局配置 --- +HOME_DIR = Path.home() +OSS_FUZZ_DIR = HOME_DIR / "FuzzAug" / "fuzz" / "oss-fuzz" +LOG_DIR = OSS_FUZZ_DIR / "run_logs" + +def setup_logging(project_name: str) -> Path: + """创建带时间戳的运行日志""" + LOG_DIR.mkdir(parents=True, exist_ok=True) + timestamp = datetime.now().strftime("%Y%m%d%H%M%S") + return LOG_DIR / f"run_{project_name}_{timestamp}.log" + +def log_and_print(message: str, log_file: Path, to_stdout: bool = True): + """日志和控制台输出""" + if to_stdout: + print(f"[PID:{os.getpid()}] {message}") + with open(log_file, "a", encoding="utf-8") as f: + f.write(f"{datetime.now().isoformat()} {message}\n") + +def run_command( + cmd: str, + log_msg: str, + log_file: Path, + allowed_exit_codes: Optional[List[int]] = None +) -> bool: + """执行命令并实时记录输出""" + allowed_exit_codes = allowed_exit_codes or [] + log_and_print(f"▶️ {log_msg}...", log_file, to_stdout=False) + log_and_print(f" $ {cmd}", log_file, to_stdout=False) + + try: + process = subprocess.Popen( + cmd, + shell=True, + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + text=True, + encoding="utf-8", + errors="replace" + ) + with open(log_file, "a", encoding="utf-8") as f: + for line in iter(process.stdout.readline, ""): + f.write(line) + process.wait() + return process.returncode in [0, *allowed_exit_codes] + except Exception as e: + log_and_print(f"💥 执行异常: {e}", log_file) + return False + +def discover_targets(project_name: str) -> List[str]: + """发现可用的 Fuzz 目标""" + out_dir = OSS_FUZZ_DIR / "build" / "out" / project_name + targets = [] + if out_dir.exists(): + for f in out_dir.iterdir(): + if f.is_file() and f.name.startswith("fuzz_") and os.access(f, os.X_OK): + targets.append(f.name) + return targets + +def run_project(project_name: str, timeout: int) -> Tuple[bool, str]: + """单个项目的测试工作流""" + log_file = setup_logging(project_name) + os.chdir(OSS_FUZZ_DIR) + + log_and_print("="*60, log_file) + log_and_print(f"🚀 开始测试项目: {project_name}", log_file) + log_and_print(f"📝 日志路径: {log_file}", log_file) + log_and_print("="*60, log_file) + + # 1. 发现测试目标 + targets = discover_targets(project_name) + if not targets: + log_and_print("⚠️ 未发现测试目标", log_file) + return (False, project_name) + log_and_print(f"🔍 发现 {len(targets)} 个测试目标", log_file) + + # 2. 运行所有目标 + all_success = True + for i, target in enumerate(targets, 1): + cmd = f"python3 infra/helper.py run_fuzzer {project_name} {target} -- -max_total_time={timeout}" + success = run_command( + cmd, + f"运行目标 [{i}/{len(targets)}] {target} (超时={timeout}s)", + log_file, + allowed_exit_codes=[1, 124] # 允许超时退出 + ) + all_success &= success + + # 3. 生成报告（占位） + log_and_print("📊 覆盖率报告生成 (当前版本暂未实现)", log_file) + return (all_success, project_name) + +def main(): + parser = argparse.ArgumentParser(description="OSS-Fuzz 并行测试工具") + parser.add_argument("project_list", help="项目列表文件路径") + parser.add_argument("--timeout", type=int, default=60, help="单目标测试超时时间（秒）") + parser.add_argument("--workers", type=int, default=cpu_count()) + args = parser.parse_args() + + # 读取项目列表 + try: + with open(args.project_list) as f: + projects = [line.strip() for line in f if line.strip()] + except Exception as e: + print(f"❌ 读取项目列表失败: {e}") + sys.exit(1) + + # 并行运行 + with Pool(args.workers) as pool: + results = pool.starmap(run_project, [(p, args.timeout) for p in projects]) + + # 输出结果 + failed = [p for success, p in results if not success] + print(f"\n📊 测试完成: 成功 {len(projects)-len(failed)}/{len(projects)}") + if failed: + print("❌ 失败项目: " + ", ".join(failed)) + +if __name__ == "__main__": + main() \ No newline at end of file From def645e95f0040ea862ce635651ada09780d37f4 Mon Sep 17 00:00:00 2001 From: joyguoguo <599325370@qq.com> Date: Tue, 22 Jul 2025 00:15:31 +0000 Subject: [PATCH 08/15] split the pool.py into build_oss_fuzz and run_fuzz_target --- fuzz/build_oss_fuzz.py | 4 +++- fuzz/fuzz_runner_pool.py | 2 +- fuzz/run_fuzz_target.py | 2 +- 3 files changed, 5 insertions(+), 3 deletions(-) diff --git a/fuzz/build_oss_fuzz.py b/fuzz/build_oss_fuzz.py index 525c556..7c60d56 100644 --- a/fuzz/build_oss_fuzz.py +++ b/fuzz/build_oss_fuzz.py @@ -8,7 +8,9 @@ 使用 multiprocessing.Pool 将项目分发到多个 CPU 核心同时处理。 用法: python3 build_oss_fuzz.py [项目列表文件] [--sanitizer 类型] [--workers N] -示例: python3 build_oss_fuzz.py valid_projects.txt --sanitizer address --workers 4 +示例: python3 fuzz/build_oss_fuzz.py data/valid_projects.txt \ + --sanitizer address \ + --workers 8 """ import os diff --git a/fuzz/fuzz_runner_pool.py b/fuzz/fuzz_runner_pool.py index abd4b30..7c40d61 100644 --- a/fuzz/fuzz_runner_pool.py +++ b/fuzz/fuzz_runner_pool.py @@ -8,7 +8,7 @@ 分发到多个 CPU核心上同时处理。 用法: python3 fuzz_runner_pool.py [项目列表文件] [--sanitizer 类型] [--workers N] -示例: python3 fuzz_runner_pool.py /home/jiayiguo/FuzzAug/data/valid_projects.txt --workers 4 +示例: python3 fuzz/fuzz_runner_pool.py data/valid_projects.txt --workers 4 """ import os diff --git a/fuzz/run_fuzz_target.py b/fuzz/run_fuzz_target.py index ae7b036..9765dbb 100644 --- a/fuzz/run_fuzz_target.py +++ b/fuzz/run_fuzz_target.py @@ -8,7 +8,7 @@ 使用 multiprocessing.Pool 分发任务到多个 CPU 核心。 用法: python3 run_fuzz_target.py [项目列表文件] [--timeout 秒] [--workers N] -示例: python3 run_fuzz_target.py valid_projects.txt --timeout 120 --workers 4 +示例: python3 fuzz/run_fuzz_target.py data/valid_projects.txt --timeout 60 --workers 4 """ import os From 1251bcd45a1d3ae84741d9217d439bc55b14756f Mon Sep 17 00:00:00 2001 From: joyguoguo <599325370@qq.com> Date: Tue, 22 Jul 2025 00:15:52 +0000 Subject: [PATCH 09/15] delete the .sh files --- script_fuzz_py_batch_final.sh | 201 ----------------------------- script_fuzz_py_final.sh | 126 ------------------ valid_projects.txt | 234 ---------------------------------- 3 files changed, 561 deletions(-) delete mode 100644 script_fuzz_py_batch_final.sh delete mode 100644 script_fuzz_py_final.sh delete mode 100644 valid_projects.txt diff --git a/script_fuzz_py_batch_final.sh b/script_fuzz_py_batch_final.sh deleted file mode 100644 index 1d7d9af..0000000 --- a/script_fuzz_py_batch_final.sh +++ /dev/null @@ -1,201 +0,0 @@ -#!/bin/bash -# script_fuzz_py_batch_final.sh -# 批量执行OSS-Fuzz本地测试全流程：从文件读取项目列表，依次为每个项目自动发现目标 -> 构建镜像 -> 编译fuzzer -> 运行测试 -> 生成覆盖率报告(暂无)) -# 用法：./script_fuzz_py_batch_final.sh [项目列表文件] [sanitizer类型] -# 示例: ./script_fuzz_py_batch_final.sh valid_projects.txt address - -# --- 全局配置 --- -PROJECT_LIST_FILE="${1:-valid_projects.txt}" # 默认项目列表文件 -SANITIZER="${2:-address}" # 默认检测器类型 -OSS_FUZZ_DIR="$HOME/oss-fuzz-py/oss-fuzz" # OSS-Fuzz目录 -LOG_DIR="$OSS_FUZZ_DIR/script_lz4_batch_logs" # 所有项目的总日志目录 -FAILED_PROJECTS=() # 存储失败项目列表 - -# --- 环境检查 --- -check_environment() { - if [ ! -d "$OSS_FUZZ_DIR" ]; then - echo "❌ 错误: OSS-Fuzz 目录 '$OSS_FUZZ_DIR' 不存在！" - return 1 - fi - if [ ! -f "$PROJECT_LIST_FILE" ]; then - echo "❌ 错误: 项目列表文件 '$PROJECT_LIST_FILE' 不存在！" - return 1 - fi - mkdir -p "$LOG_DIR" - chmod 777 "$LOG_DIR" 2>/dev/null || true - cd "$OSS_FUZZ_DIR" || return 1 - echo "✅ 环境检查通过。OSS-Fuzz 目录: $OSS_FUZZ_DIR" -} - -# --- 带日志记录的命令执行 --- -run_command() { - local cmd="$1" - local log_msg="$2" - local log_file="$3" # 日志文件作为参数传入 - local allowed_exit="${4:-}" - - echo "▶️ $log_msg..." | tee -a "$log_file" - set +e - { yes | eval "$cmd" ; } 2>&1 | tee -a "$log_file" - local exit_code=${PIPESTATUS[0]} - set -e - - if [[ -n "$allowed_exit" && ",$allowed_exit," =~ ",$exit_code," ]]; then - echo "ℹ️ 命令以预期状态退出: $exit_code" | tee -a "$log_file" - return 0 - elif [ $exit_code -ne 0 ]; then - echo "❌ 命令执行失败: $cmd (退出码: $exit_code)" | tee -a "$log_file" - return 1 # 返回错误而不是退出脚本 - fi -} - -# --- 自动发现 Fuzz 目标 --- -discover_fuzz_targets() { - local project_name="$1" - local project_dir="$OSS_FUZZ_DIR/build/out/$project_name" - local project_src="$OSS_FUZZ_DIR/projects/$project_name" - local targets=() - - if [ -d "$project_dir" ]; then - while IFS= read -r -d $'\0' file; do - filename=$(basename "$file") - if [[ -x "$file" && "$filename" =~ ^fuzz_ && ! "$file" =~ \..*$ ]]; then - targets+=("$filename") - fi - done < <(find "$project_dir" -maxdepth 1 -type f -print0) - fi - - if [ ${#targets[@]} -eq 0 ] && [ -d "$project_src" ]; then - while IFS= read -r -d $'\0' file; do - if grep -q "atheris.Setup" "$file"; then - targets+=("$(basename "${file%.*}")") - fi - done < <(find "$project_src" -name 'fuzz_*.py' -print0) - fi - - echo "${targets[@]}" -} - -# --- 单个项目的完整处理流程 --- -process_project() { - local project_name="$1" - local log_file="$LOG_DIR/oss_fuzz_${project_name}_$(date +%Y%m%d%H%M%S).log" - local project_failed=0 - - echo "============================================================" | tee -a "$log_file" - echo "🚀 开始处理项目: $project_name" | tee -a "$log_file" - echo "📝 日志文件: $log_file" | tee -a "$log_file" - echo "============================================================" | tee -a "$log_file" - - #1. 构建Docker镜像 - if ! run_command \ - "python3 infra/helper.py build_image $project_name" \ - "步骤1/5: 构建 $project_name 的Docker镜像" \ - "$log_file"; then - echo "❌ 项目 $project_name 构建镜像失败，跳过后续步骤" | tee -a "$log_file" - project_failed=1 - fi - - # 2. 编译带检测器的fuzzer (仅在构建镜像成功后执行) - if [ $project_failed -eq 0 ]; then - if ! run_command \ - "python3 infra/helper.py build_fuzzers --sanitizer $SANITIZER $project_name" \ - "步骤2/5: 编译 $project_name 的fuzzer (sanitizer=$SANITIZER)" \ - "$log_file"; then - echo "❌ 项目 $project_name 编译fuzzer失败，跳过后续步骤" | tee -a "$log_file" - project_failed=1 - fi - fi - - # 3. 自动发现目标 (仅在编译成功后执行) - if [ $project_failed -eq 0 ]; then - echo "🔍 正在为 $project_name 自动发现fuzz目标..." | tee -a "$log_file" - FUZZ_TARGETS=($(discover_fuzz_targets "$project_name")) - - if [ ${#FUZZ_TARGETS[@]} -eq 0 ]; then - echo "⚠️ 警告: 项目 $project_name 未找到任何fuzz目标！跳过运行步骤。" | tee -a "$log_file" - else - echo "✅ 发现目标: ${FUZZ_TARGETS[*]}" | tee -a "$log_file" - - # 4. 遍历运行所有目标 默认测试60秒 - for target in "${FUZZ_TARGETS[@]}"; do - if ! run_command \ - "python3 infra/helper.py run_fuzzer $project_name $target -- -max_total_time=60" \ - "步骤3/5: 运行目标 [$target] (60秒超时)" \ - "$log_file" \ - "124,1"; then # 允许超时(124)和发现崩溃(1) - echo "⚠️ 警告: 目标 [$target] 运行失败，继续下一个目标" | tee -a "$log_file" - fi - done - fi - fi - - # 5. 生成覆盖率报告 (已注释掉，与原脚本保持一致) - # [保留原有注释的覆盖率代码] - - if [ $project_failed -eq 0 ]; then - echo "✅ 项目 $project_name 处理完成！" | tee -a "$log_file" - - else - echo "❌ 项目 $project_name 处理失败！" | tee -a "$log_file" - - fi - - echo "------------------------------------------------------------" - return $project_failed -} - -# --- 主流程 --- -main() { -if ! check_environment; then - echo "❌ 环境检查失败，脚本终止" - exit 1 - fi - - # 新增：读取项目列表到数组（过滤空行和注释行） - local PROJECTS=() - while IFS= read -r line || [[ -n "$line" ]]; do - if [[ -z "$line" || "$line" =~ ^# ]]; then - continue - fi - PROJECTS+=("$line") - done < "$PROJECT_LIST_FILE" - - # 项目总数从数组长度获取（原逻辑从文件行数获取） - local total_projects=${#PROJECTS[@]} - local current_project_num=0 - local success_count=0 - local fail_count=0 - - # 新增：遍历数组处理项目（替代原while读取文件的循环） - for project_name in "${PROJECTS[@]}"; do - - current_project_num=$((current_project_num + 1)) - echo ">>> [ $current_project_num / $total_projects ] 开始处理项目: $project_name <<<" - - if process_project "$project_name"; then - echo "✅ [$current_project_num/$total_projects] 项目 $project_name 成功完成" - ( success_count=$[ $success_count + 1 ]) - else - echo "❌ [$current_project_num/$total_projects] 项目 $project_name 处理失败" - FAILED_PROJECTS+=("$project_name") - ( success_count=$[ $success_count + 1 ]) - fi - - done - - echo "============================================================" - echo "🎉 批量处理完成！" - echo "📊 总计: $total_projects 个项目" - echo "✅ 成功: $success_count" - echo "❌ 失败: $fail_count" - - if [ ${#FAILED_PROJECTS[@]} -gt 0 ]; then - echo "📛 失败项目列表:" - printf ' • %s\n' "${FAILED_PROJECTS[@]}" - echo "💡 提示: 可以重新运行失败项目，检查日志获取详细信息" - echo " 日志目录: $LOG_DIR" - fi -} - -main "$@" \ No newline at end of file diff --git a/script_fuzz_py_final.sh b/script_fuzz_py_final.sh deleted file mode 100644 index c2bf00a..0000000 --- a/script_fuzz_py_final.sh +++ /dev/null @@ -1,126 +0,0 @@ -#!/bin/bash -# script_fuzz_py_final.sh -# 执行OSS-Fuzz本地测试全流程：自动发现目标 -> 构建镜像 -> 编译fuzzer -> 运行测试 -> 生成覆盖率报告 -# 用法：script_fuzz_py_final.sh <项目名> [sanitizer类型] - -set -e # 遇到错误立即退出 - -PROJECT_NAME="${1:-abseil-py}" # 默认项目名 -SANITIZER="${2:-address}" # 默认检测器类型 -OSS_FUZZ_DIR="$HOME/oss-fuzz-py/oss-fuzz" # OSS-Fuzz目录 -LOG_DIR="$OSS_FUZZ_DIR/script_lz4_logs" -LOG_FILE="$LOG_DIR/oss_fuzz_${PROJECT_NAME}_$(date +%Y%m%d%H%M%S).log" -# 验证目录有效性 -check_environment() { - if [ ! -d "$OSS_FUZZ_DIR" ]; then - echo "❌ 错误: $OSS_FUZZ_DIR 目录不存在！" - exit 1 - fi - mkdir -p "$LOG_DIR" # 关键修复：创建日志目录 - chmod 777 "$LOG_DIR" 2>/dev/null || true # 宽松权限设置 - cd "$OSS_FUZZ_DIR" || exit 1 -} - -# 带日志记录的命令执行（支持允许的退出码） -run_command() { - local cmd="$1" - local log_msg="$2" - local allowed_exit="${3:-}" # 可选：允许的退出码（逗号分隔） - - echo "▶️ $log_msg..." | tee -a "$LOG_FILE" - set +e # 临时禁用错误退出 - eval "$cmd" 2>&1 | tee -a "$LOG_FILE" - local exit_code=${PIPESTATUS[0]} - set -e # 重新启用错误退出 - - # 检查退出码是否被允许 - if [[ -n "$allowed_exit" && ",$allowed_exit," =~ ",$exit_code," ]]; then - echo "ℹ️ 命令以预期状态退出: $exit_code" | tee -a "$LOG_FILE" - return 0 - elif [ $exit_code -ne 0 ]; then - echo "❌ 命令执行失败: $cmd (退出码: $exit_code)" | tee -a "$LOG_FILE" - exit 1 - fi -} - -# 自动发现fuzz目标 -discover_fuzz_targets() { - local project_dir="$OSS_FUZZ_DIR/build/out/$PROJECT_NAME" - local project_src="$OSS_FUZZ_DIR/projects/$PROJECT_NAME" - local targets=() - - # 编译目录扫描：仅匹配"fuzz_"开头的可执行文件 - if [ -d "$project_dir" ]; then - while IFS= read -r -d $'\0' file; do - filename=$(basename "$file") - if [[ -x "$file" && "$filename" =~ ^fuzz_ && ! "$file" =~ \..*$ ]]; then - targets+=("$filename") - fi - done < <(find "$project_dir" -maxdepth 1 -type f -print0) - fi - - # 源码目录扫描：仅匹配"fuzz_*.py"且含Atheris标识 - if [ ${#targets[@]} -eq 0 ] && [ -d "$project_src" ]; then - while IFS= read -r -d $'\0' file; do - if grep -q "atheris.Setup" "$file"; then - targets+=("$(basename "${file%.*}")") - fi - done < <(find "$project_src" -name 'fuzz_*.py' -print0) - fi - - echo "${targets[@]}" -} - -# 主流程 -main() { - check_environment - echo "==============================" - echo "🚀 开始OSS-Fuzz测试 - 项目: $PROJECT_NAME" - echo "📝 日志文件: $LOG_FILE" - echo "==============================" - - #1. 构建Docker镜像 - run_command \ - "python3 infra/helper.py build_image $PROJECT_NAME" \ - "步骤1/5: 构建Docker镜像" - - # 2. 编译带检测器的fuzzer - run_command \ - "python3 infra/helper.py build_fuzzers --sanitizer $SANITIZER $PROJECT_NAME" \ - "步骤2/5: 编译fuzzer (sanitizer=$SANITIZER)" - - # 3. 自动发现目标 - echo "🔍 自动发现fuzz目标..." - FUZZ_TARGETS=($(discover_fuzz_targets)) - - if [ ${#FUZZ_TARGETS[@]} -eq 0 ]; then - echo "❌ 未找到任何fuzz目标！检查项目配置" | tee -a "$LOG_FILE" - exit 1 - fi - - echo "✅ 发现目标: ${FUZZ_TARGETS[*]}" | tee -a "$LOG_FILE" - - # 4. 遍历运行所有目标 - for target in "${FUZZ_TARGETS[@]}"; do - run_command \ - "python3 infra/helper.py run_fuzzer $PROJECT_NAME $target -- -max_total_time=180" \ - "步骤3/5: 运行目标 [$target] (120秒超时)" \ - "124,1" # 允许超时(124)和发现崩溃(1) -done - - # 5. 生成覆盖率报告 - # run_command \ - # "python3 infra/helper.py build_fuzzers --sanitizer coverage $PROJECT_NAME" \ - # "步骤4/5: 编译覆盖率版本" - - # run_command \ - # "python3 infra/helper.py coverage --no-serve $PROJECT_NAME" \ - # "步骤5/5: 生成覆盖率报告" - - echo "✅ 所有步骤完成！结果查看:" - echo "🔍 测试日志: $LOG_FILE" - echo "📊 覆盖率报告(暂无): $OSS_FUZZ_DIR/build/out/$PROJECT_NAME/report/coverage/index.html" - echo "💥 崩溃报告: $OSS_FUZZ_DIR/build/out/$PROJECT_NAME/crashes/" -} - -main "$@" \ No newline at end of file diff --git a/valid_projects.txt b/valid_projects.txt deleted file mode 100644 index 94fdcd5..0000000 --- a/valid_projects.txt +++ /dev/null @@ -1,234 +0,0 @@ -abseil-py -adal -aiohttp -aniso8601 -ansible -argcomplete -arrow-py -asn1crypto -asteval -astroid -asttokens -attrs -autoflake -autopep8 -azure-sdk-for-python -babel -black -botocore -bottleneck -bz2file -cachetools -cffi -chardet -charset_normalizer -click -cloud-custodian -configparser -connexion -coveragepy -croniter -cryptography -cssselect -dask -decorator -defusedxml -digest -dill -distlib -dnspython -docutils -ecdsa-python -et-xmlfile -face -filelock -filesystem_spec -flask -flask-jwt-extended -flask-restx -flask-wtf -fonttools -ftfy -g-api-auth-httplib2 -g-api-auth-library-python -g-api-pubsub -g-api-py-api-common-protos -g-api-py-oauthlib -g-api-python-bigquery-storage -g-api-python-client -g-api-python-cloud-core -g-api-python-firestore -g-api-python-tasks -g-api-resource-manager -g-api-resumable-media-python -g-api-secret-manager -g-apis-py-api-core -gast -gc-iam -gcloud-error-py -g-cloud-logging-py -gcp-python-cloud-storage -genshi -gitdb -glom -gprof2dot -g-py-bigquery -g-py-crc32c -grpc-py -gunicorn -h11 -h5py -hiredis-py -html2text -html5lib-python -httpcore -httpretty -httpx -idna -ijson -importlib_metadata -iniconfig -ipaddress -ipykernel -ipython -isodate -itsdangerous -jedi -jinja2 -jmespathpy -joblib -jsmin -jupyter-nbconvert -jupyter_server -kafka -keras -kiwisolver -lark-parser -libcst -looker-sdk -lxml -mako -markupsafe -matplotlib -mccabe -mdit-py-plugins -mdurl -more-itertools -mrab-regex -msal -msgpack-python -multidict -mutagen -nbclassic -nbformat -netaddr-py -networkx -ntlm2 -ntlm-auth -numexpr -numpy -oauth2 -oauthlib -olefile -openapi-schema-validator -opencensus-python -openpyxl -opt_einsum -oracle-py-cx -orjson -oscrypto -packaging -pandas -paramiko -parse -parsimonious -pasta -pathlib2 -pdoc -pem -pendulum -pip -ply -protobuf-python -proto-plus-python -psqlparse -psutil -psycopg2 -pyasn1 -pyasn1-modules -pycparser -pycrypto -pydantic -pydateutil -pygments -pyjson5 -pyjwt -pymysql -pynacl -pyodbc -pyparsing -pyrsistent -py-serde -pytables -pytest-py -python3-openid -python-ecdsa -python-email-validator -python-fastjsonschema -python-future -python-graphviz -python-hyperlink -python-jose -python-lz4 -python-markdown -python-markdownify -python-nameparser -python-nvd3 -python-pathspec -python-prompt-toolkit -python-pypdf -python-rison -python-rsa -python-tabulate -pytz -pyxdg -pyyaml -pyzmq -redis-py -requests -retry -rfc3967 -rich -sacremoses -scikit-learn -scipy -setuptools -sigstore-python -simplejson -six -smart_open -soupsieve -sqlalchemy_jsonfield -sqlalchemy-utils -sqlparse -stack_data -tensorflow-addons -tinycss2 -toml -tomlkit -toolbelt -toolz -tqdm -typing_extensions -underscore -uritemplate -urlextract -urllib3 -validators -w3lib -websocket-client -wheel -wtforms -xlrd -yarl -zipp From 21017f1c2cf08f1929775b3480bf01320ec0315b Mon Sep 17 00:00:00 2001 From: joyguoguo <599325370@qq.com> Date: Tue, 22 Jul 2025 00:45:28 +0000 Subject: [PATCH 10/15] translate to english --- fuzz/build_oss_fuzz.py | 56 +++++++------- fuzz/fuzz_runner_pool.py | 158 +++++++++++++++++++-------------------- fuzz/run_fuzz_target.py | 62 +++++++-------- 3 files changed, 138 insertions(+), 138 deletions(-) diff --git a/fuzz/build_oss_fuzz.py b/fuzz/build_oss_fuzz.py index 7c60d56..1c78a0d 100644 --- a/fuzz/build_oss_fuzz.py +++ b/fuzz/build_oss_fuzz.py @@ -4,11 +4,11 @@ """ build_oss_fuzz.py -并行构建 OSS-Fuzz 项目（Docker 镜像和 Fuzzer 编译）。 -使用 multiprocessing.Pool 将项目分发到多个 CPU 核心同时处理。 +Parallel build of OSS-Fuzz projects (Docker images and Fuzzer compilation). +Uses multiprocessing.Pool to distribute projects across multiple CPU cores for concurrent processing. -用法: python3 build_oss_fuzz.py [项目列表文件] [--sanitizer 类型] [--workers N] -示例: python3 fuzz/build_oss_fuzz.py data/valid_projects.txt \ +Usage: python3 build_oss_fuzz.py [project_list_file] [--sanitizer type] [--workers N] +Example: python3 fuzz/build_oss_fuzz.py data/valid_projects.txt \ --sanitizer address \ --workers 8 """ @@ -22,19 +22,19 @@ from typing import List, Optional, Tuple from multiprocessing import Pool, cpu_count -# --- 全局配置 --- +# --- Global configuration --- HOME_DIR = Path.home() OSS_FUZZ_DIR = HOME_DIR / "FuzzAug" / "fuzz" / "oss-fuzz" LOG_DIR = OSS_FUZZ_DIR / "build_logs" def setup_logging(project_name: str) -> Path: - """为单个项目创建带时间戳的日志文件""" + """Create a timestamped log file for a single project""" LOG_DIR.mkdir(parents=True, exist_ok=True) timestamp = datetime.now().strftime("%Y%m%d%H%M%S") return LOG_DIR / f"build_{project_name}_{timestamp}.log" def log_and_print(message: str, log_file: Path, to_stdout: bool = True): - """将消息写入日志并打印到控制台""" + """Write message to log and print to console""" if to_stdout: print(f"[PID:{os.getpid()}] {message}") with open(log_file, "a", encoding="utf-8") as f: @@ -46,14 +46,14 @@ def run_command( log_file: Path, allowed_exit_codes: Optional[List[int]] = None ) -> bool: - """执行 shell 命令并实时记录输出""" + """Execute a shell command and stream output to log in real-time""" allowed_exit_codes = allowed_exit_codes or [] log_and_print(f"▶️ {log_msg}...", log_file, to_stdout=False) log_and_print(f" $ {cmd}", log_file, to_stdout=False) try: process = subprocess.Popen( - f"yes | {cmd}", # 自动确认所有提示 + f"yes | {cmd}", # Auto-confirm all prompts shell=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, @@ -67,67 +67,67 @@ def run_command( process.wait() exit_code = process.returncode if exit_code in [0, *allowed_exit_codes]: - log_and_print(f"✅ 命令成功完成", log_file, to_stdout=False) + log_and_print(f"✅ Command completed successfully", log_file, to_stdout=False) return True - log_and_print(f"❌ 命令失败 (退出码: {exit_code})", log_file) + log_and_print(f"❌ Command failed (exit code: {exit_code})", log_file) return False except Exception as e: - log_and_print(f"💥 执行异常: {e}", log_file) + log_and_print(f"💥 Execution exception: {e}", log_file) return False def build_project(project_name: str, sanitizer: str) -> Tuple[bool, str]: - """单个项目的构建工作流""" + """Build workflow for a single project""" log_file = setup_logging(project_name) os.chdir(OSS_FUZZ_DIR) log_and_print("="*60, log_file) - log_and_print(f"🔨 开始构建项目: {project_name}", log_file) - log_and_print(f"📝 日志路径: {log_file}", log_file) + log_and_print(f"🔨 Starting build for project: {project_name}", log_file) + log_and_print(f"📝 Log path: {log_file}", log_file) log_and_print("="*60, log_file) - # 1. 构建 Docker 镜像 + # 1. Build Docker image if not run_command( f"python3 infra/helper.py build_image {project_name}", - "步骤1/2: 构建 Docker 镜像", + "Step 1/2: Building Docker image", log_file ): return (False, project_name) - # 2. 编译 Fuzzer + # 2. Compile Fuzzers if not run_command( f"python3 infra/helper.py build_fuzzers --sanitizer {sanitizer} {project_name}", - f"步骤2/2: 编译 Fuzzer (sanitizer={sanitizer})", + f"Step 2/2: Compiling Fuzzers (sanitizer={sanitizer})", log_file ): return (False, project_name) - log_and_print(f"✅ 项目 {project_name} 构建完成", log_file) + log_and_print(f"✅ Project {project_name} build completed", log_file) return (True, project_name) def main(): - parser = argparse.ArgumentParser(description="OSS-Fuzz 并行构建工具") - parser.add_argument("project_list", help="项目列表文件路径") + parser = argparse.ArgumentParser(description="OSS-Fuzz Parallel Build Tool") + parser.add_argument("project_list", help="Project list file path") parser.add_argument("--sanitizer", default="address", choices=["address", "memory", "undefined"]) parser.add_argument("--workers", type=int, default=cpu_count()) args = parser.parse_args() - # 读取项目列表 + # Read project list try: with open(args.project_list, "r") as f: projects = [line.strip() for line in f if line.strip()] except Exception as e: - print(f"❌ 读取项目列表失败: {e}") + print(f"❌ Failed to read project list: {e}") sys.exit(1) - # 并行构建 + # Parallel build with Pool(args.workers) as pool: results = pool.starmap(build_project, [(p, args.sanitizer) for p in projects]) - # 输出结果 + # Output results failed = [p for success, p in results if not success] - print(f"\n📊 构建完成: 成功 {len(projects)-len(failed)}/{len(projects)}") + print(f"\n📊 Build completed: Success {len(projects)-len(failed)}/{len(projects)}") if failed: - print("❌ 失败项目: " + ", ".join(failed)) + print("❌ Failed projects: " + ", ".join(failed)) if __name__ == "__main__": main() \ No newline at end of file diff --git a/fuzz/fuzz_runner_pool.py b/fuzz/fuzz_runner_pool.py index 7c40d61..996b940 100644 --- a/fuzz/fuzz_runner_pool.py +++ b/fuzz/fuzz_runner_pool.py @@ -4,11 +4,11 @@ """ fuzz_runner_pool.py -并行批量执行 OSS-Fuzz 本地测试全流程。使用 multiprocessing.Pool 将项目 -分发到多个 CPU核心上同时处理。 +Parallel batch execution of the entire OSS-Fuzz local testing process. +Uses multiprocessing.Pool to distribute projects across multiple CPU cores for concurrent processing. -用法: python3 fuzz_runner_pool.py [项目列表文件] [--sanitizer 类型] [--workers N] -示例: python3 fuzz/fuzz_runner_pool.py data/valid_projects.txt --workers 4 +Usage: python3 fuzz_runner_pool.py [project_list_file] [--sanitizer type] [--workers N] +Example: python3 fuzz/fuzz_runner_pool.py data/valid_projects.txt --workers 4 """ import os @@ -20,18 +20,18 @@ from typing import List, Optional, Tuple from multiprocessing import Pool, cpu_count -# --- 全局配置 (可通过命令行参数覆盖) --- +# --- Global configuration (can be overridden by command line arguments) --- HOME_DIR = Path.home() OSS_FUZZ_DIR = HOME_DIR / "FuzzAug" / "fuzz"/"oss-fuzz" LOG_DIR = OSS_FUZZ_DIR / "script_pool_batch_logs" def setup_logging(project_name: str) -> Path: - """为单个项目创建带时间戳的日志文件.""" + """Create a timestamped log file for a single project.""" LOG_DIR.mkdir(parents=True, exist_ok=True) try: LOG_DIR.chmod(0o777) except PermissionError: - # 在并行环境中，这里可能会有多个进程同时尝试，打印一次警告即可 + # In a parallel environment, multiple processes may try simultaneously, printing a warning once is sufficient pass timestamp = datetime.now().strftime("%Y%m%d%H%M%S") @@ -39,9 +39,9 @@ def setup_logging(project_name: str) -> Path: return log_file_path def log_and_print(message: str, log_file: Path, to_stdout: bool = True): - """将消息写入日志文件，并根据需要打印到控制台。""" + """Write message to log file and optionally print to console.""" if to_stdout: - # 添加进程ID，以便区分并行输出 + # Add process ID to distinguish parallel outputs print(f"[PID:{os.getpid()}] {message}") with open(log_file, "a", encoding="utf-8") as f: f.write(message + "\n") @@ -51,9 +51,9 @@ def run_command( log_msg: str, log_file: Path, allowed_exit_codes: Optional[List[int]] = None, - auto_confirm: bool = True # 新增自动确认参数 + auto_confirm: bool = True # New auto-confirm parameter ) -> bool: - """执行一个 shell 命令，并将输出实时流式传输到日志文件。""" + """Execute a shell command and stream output to log file in real-time.""" if allowed_exit_codes is None: allowed_exit_codes = [] @@ -61,7 +61,7 @@ def run_command( log_and_print(f" $ {cmd}", log_file, to_stdout=False) try: - # 添加自动确认机制 + # Add auto-confirm mechanism if auto_confirm: cmd = f"yes | {cmd}" @@ -72,26 +72,26 @@ def run_command( with open(log_file, "a", encoding="utf-8") as f: for line in iter(process.stdout.readline, ''): - f.write(line) # 只写入日志，避免控制台输出混乱 + f.write(line) # Write to log only to avoid console clutter process.wait() exit_code = process.returncode if exit_code == 0: - log_and_print(f"✅ 命令成功完成。", log_file, to_stdout=False) + log_and_print(f"✅ Command completed successfully.", log_file, to_stdout=False) return True elif exit_code in allowed_exit_codes: - log_and_print(f"ℹ️ 命令以预期状态退出: {exit_code}", log_file, to_stdout=False) + log_and_print(f"ℹ️ Command exited with expected status: {exit_code}", log_file, to_stdout=False) return True else: - log_and_print(f"❌ 命令执行失败 (退出码: {exit_code})", log_file) + log_and_print(f"❌ Command execution failed (exit code: {exit_code})", log_file) return False except Exception as e: - log_and_print(f"💥 执行命令时发生异常: {e}", log_file) + log_and_print(f"💥 Exception occurred while executing command: {e}", log_file) return False def discover_fuzz_targets(project_name: str) -> List[str]: - """自动发现 Fuzz 目标。""" + """Automatically discover Fuzz targets.""" project_out_dir = OSS_FUZZ_DIR / "build" / "out" / project_name project_src_dir = OSS_FUZZ_DIR / "projects" / project_name targets = [] @@ -109,162 +109,162 @@ def discover_fuzz_targets(project_name: str) -> List[str]: if "atheris.Setup" in f.read(): targets.append(py_file.stem) except Exception as e: - # 在worker进程中，只打印到自己的日志 - print(f"⚠️ 警告: 读取文件 {py_file} 失败: {e}") + # In worker processes, only print to own log + print(f"⚠️ Warning: Failed to read file {py_file}: {e}") return targets def run_project_workflow(project_name: str, sanitizer: str) -> Tuple[bool, str]: """ - 处理单个项目的完整工作流 (Worker Function)。 - 此函数由进程池中的每个工作进程独立执行。 + Handle the complete workflow for a single project (Worker Function). + This function is executed independently by each worker process in the process pool. Returns: - 一个元组 (is_success: bool, project_name: str) + A tuple (is_success: bool, project_name: str) """ log_file = setup_logging(project_name) - # 在 worker 的开头打印，以便追踪 + # Print at the beginning of worker for tracking log_and_print("=" * 60, log_file) - log_and_print(f"🚀 开始处理项目: {project_name}", log_file) - log_and_print(f"📝 日志文件: {log_file}", log_file) + log_and_print(f"🚀 Starting processing for project: {project_name}", log_file) + log_and_print(f"📝 Log file: {log_file}", log_file) log_and_print("=" * 60, log_file) - # 每个进程都需要设置自己的工作目录 + # Each process needs to set its own working directory try: os.chdir(OSS_FUZZ_DIR) except FileNotFoundError: - log_and_print(f"❌ 严重错误: OSS-Fuzz 目录 '{OSS_FUZZ_DIR}' 不存在！", log_file) + log_and_print(f"❌ Critical error: OSS-Fuzz directory '{OSS_FUZZ_DIR}' does not exist!", log_file) return (False, project_name) - # 步骤 1: 构建Docker镜像（启用自动确认） + # Step 1: Build Docker image (with auto-confirm enabled) if not run_command( f"python3 infra/helper.py build_image {project_name}", - f"步骤1/5: 构建 {project_name} 的Docker镜像", log_file, - auto_confirm=True # 自动确认所有提示 + f"Step 1/5: Building Docker image for {project_name}", log_file, + auto_confirm=True # Auto-confirm all prompts ): - log_and_print(f"❌ 项目 {project_name} 构建镜像失败", log_file) + log_and_print(f"❌ Project {project_name} failed to build image", log_file) return (False, project_name) - # 步骤 2: 编译带检测器的fuzzer（启用自动确认） + # Step 2: Compile fuzzers with sanitizer (with auto-confirm enabled) if not run_command( f"python3 infra/helper.py build_fuzzers --sanitizer {sanitizer} {project_name}", - f"步骤2/5: 编译 {project_name} 的fuzzer (sanitizer={sanitizer})", log_file, - auto_confirm=True # 自动确认所有提示 + f"Step 2/5: Compiling fuzzers for {project_name} (sanitizer={sanitizer})", log_file, + auto_confirm=True # Auto-confirm all prompts ): - log_and_print(f"❌ 项目 {project_name} 编译fuzzer失败", log_file) + log_and_print(f"❌ Project {project_name} failed to compile fuzzers", log_file) return (False, project_name) - # 步骤 3: 自动发现目标 - log_and_print(f"🔍 正在为 {project_name} 自动发现fuzz目标...", log_file) + # Step 3: Automatically discover targets + log_and_print(f"🔍 Automatically discovering fuzz targets for {project_name}...", log_file) fuzz_targets = discover_fuzz_targets(project_name) if not fuzz_targets: - log_and_print(f"⚠️ 警告: {project_name} 未找到任何fuzz目标！跳过运行步骤。", log_file) + log_and_print(f"⚠️ Warning: No fuzz targets found for {project_name}! Skipping run step.", log_file) return (True, project_name) - log_and_print(f"✅ 发现目标: {', '.join(fuzz_targets)}", log_file) + log_and_print(f"✅ Targets discovered: {', '.join(fuzz_targets)}", log_file) - # 步骤 4: 遍历运行所有目标（启用自动确认） + # Step 4: Run all targets (with auto-confirm enabled) for i, target in enumerate(fuzz_targets, 1): run_command( f"python3 infra/helper.py run_fuzzer {project_name} {target} -- -max_total_time=60", - f"步骤4/{len(fuzz_targets)}: 运行目标 [{target}] (60秒)", log_file, + f"Step 4/{len(fuzz_targets)}: Running target [{target}] (60 seconds)", log_file, allowed_exit_codes=[1, 124], - auto_confirm=True # 自动确认所有提示 + auto_confirm=True # Auto-confirm all prompts ) - # 步骤 5: 生成覆盖率报告 (暂无) - log_and_print("步骤5/5: 生成覆盖率报告 (当前版本暂未实现)", log_file) - log_and_print(f"✅ 项目 {project_name} 处理完成！", log_file) + # Step 5: Generate coverage report (not implemented yet) + log_and_print("Step 5/5: Generating coverage report (not implemented in current version)", log_file) + log_and_print(f"✅ Project {project_name} processing completed!", log_file) return (True, project_name) def main(): """ - 主流程函数：设置进程池并分发任务。 + Main workflow function: Set up process pool and distribute tasks. """ parser = argparse.ArgumentParser( - description="OSS-Fuzz 并行批量测试工具", + description="OSS-Fuzz Parallel Batch Testing Tool", formatter_class=argparse.RawTextHelpFormatter, - epilog="示例:\n python3 fuzz_runner_parallel.py valid_projects.txt --workers 4\n python3 fuzz_runner_parallel.py my_projects.txt --sanitizer undefined" + epilog="Examples:\n python3 fuzz_runner_parallel.py valid_projects.txt --workers 4\n python3 fuzz_runner_parallel.py my_projects.txt --sanitizer undefined" ) parser.add_argument( "project_list_file", nargs="?", default="valid_projects.txt", - help="包含待测试项目列表的文本文件。(默认: valid_projects.txt)" + help="Text file containing list of projects to test. (Default: valid_projects.txt)" ) parser.add_argument( "--sanitizer", default="address", choices=["address", "memory", "undefined", "coverage"], - help="要使用的 sanitizer 类型。(默认: address)" + help="Type of sanitizer to use. (Default: address)" ) parser.add_argument( "--workers", type=int, default=cpu_count(), - help=f"并发执行的工作进程数。(默认: 系统CPU核心数, 即 {cpu_count()})" + help=f"Number of concurrent worker processes. (Default: system CPU count, currently {cpu_count()})" ) args = parser.parse_args() - # --- 环境检查 --- + # --- Environment checks --- if not OSS_FUZZ_DIR.is_dir(): - print(f"❌ 错误: OSS-Fuzz 目录 '{OSS_FUZZ_DIR}' 不存在！") + print(f"❌ Error: OSS-Fuzz directory '{OSS_FUZZ_DIR}' does not exist!") sys.exit(1) project_list_path = Path(args.project_list_file) if not project_list_path.is_file(): - print(f"❌ 错误: 项目列表文件 '{project_list_path}' 不存在！") + print(f"❌ Error: Project list file '{project_list_path}' does not exist!") sys.exit(1) - print(f"✅ 环境检查通过。将使用 {args.workers} 个并行工作进程。") + print(f"✅ Environment checks passed. Will use {args.workers} parallel worker processes.") - # --- 读取和准备任务 --- + # --- Read and prepare tasks --- try: with open(project_list_path, "r", encoding="utf-8") as f: projects = [line.strip() for line in f if line.strip() and not line.strip().startswith("#")] except Exception as e: - print(f"❌ 读取项目列表文件时出错: {e}") + print(f"❌ Error reading project list file: {e}") sys.exit(1) if not projects: - print("⚠️ 警告: 项目列表为空，无可执行任务。") + print("⚠️ Warning: Project list is empty, no tasks to execute.") sys.exit(0) - # 为 starmap 准备任务参数列表，每个项目都是一个 (project_name, sanitizer) 元组 + # Prepare task parameters for starmap, each project is a (project_name, sanitizer) tuple tasks = [(project, args.sanitizer) for project in projects] total_projects = len(tasks) - print(f"\n🚀 即将并行处理 {total_projects} 个项目...") + print(f"\n🚀 About to process {total_projects} projects in parallel...") - # --- 执行并行处理 --- - # 使用 with 语句确保进程池被正确关闭 + # --- Execute parallel processing --- + # Use with statement to ensure proper pool cleanup with Pool(processes=args.workers) as pool: - # starmap 会阻塞直到所有任务完成 - # 它将 tasks 列表中的每个元组解包作为参数传递给 worker 函数 + # starmap blocks until all tasks complete + # Unpacks each tuple in tasks as arguments to worker function results = pool.starmap(run_project_workflow, tasks) - # --- 收集并打印结果 --- + # --- Collect and print results --- failed_projects = [] for success, project_name in results: if success: - print(f"✅ 项目 {project_name} 成功完成") + print(f"✅ Project {project_name} completed successfully") else: - print(f"❌ 项目 {project_name} 处理失败") + print(f"❌ Project {project_name} processing failed") failed_projects.append(project_name) - # --- 最终总结 --- + # --- Final summary --- fail_count = len(failed_projects) success_count = total_projects - fail_count print("\n" + "=" * 60) - print("🎉 批量处理完成！") - print(f"📊 总计: {total_projects} 个项目") - print(f"✅ 成功: {success_count}") - print(f"❌ 失败: {fail_count}") + print("🎉 Batch processing completed!") + print(f"📊 Total: {total_projects} projects") + print(f"✅ Success: {success_count}") + print(f"❌ Failures: {fail_count}") if failed_projects: - print("📛 失败项目列表:") + print("📛 Failed projects list:") for proj in sorted(failed_projects): print(f" • {proj}") - print("\n💡 提示: 失败项目的详细信息请查看对应的日志文件。") - print(f" 日志目录: {LOG_DIR}") + print("\n💡 Tip: Detailed information for failed projects can be found in corresponding log files.") + print(f" Log directory: {LOG_DIR}") if __name__ == "__main__": - # 在 Windows 或 macOS 的某些 Python 版本上，需要将 main 调用放在这个保护块中 - # 以防止子进程重新导入和执行主模块代码，导致无限递归。 - main() \ No newline at end of file + # On some Python versions for Windows or macOS, main call needs to be in this guard + # To prevent child processes from re-importing and executing main module code, causing infinite recursion. + main() diff --git a/fuzz/run_fuzz_target.py b/fuzz/run_fuzz_target.py index 9765dbb..5b43a9e 100644 --- a/fuzz/run_fuzz_target.py +++ b/fuzz/run_fuzz_target.py @@ -4,11 +4,11 @@ """ run_fuzz_target.py -并行运行 OSS-Fuzz 测试目标。 -使用 multiprocessing.Pool 分发任务到多个 CPU 核心。 +Run OSS-Fuzz test targets in parallel. +Uses multiprocessing.Pool to distribute tasks to multiple CPU cores. -用法: python3 run_fuzz_target.py [项目列表文件] [--timeout 秒] [--workers N] -示例: python3 fuzz/run_fuzz_target.py data/valid_projects.txt --timeout 60 --workers 4 +Usage: python3 run_fuzz_target.py [project_list_file] [--timeout seconds] [--workers N] +Example: python3 fuzz/run_fuzz_target.py data/valid_projects.txt --timeout 60 --workers 4 """ import os @@ -20,19 +20,19 @@ from typing import List, Optional, Tuple from multiprocessing import Pool, cpu_count -# --- 全局配置 --- +# --- Global configuration --- HOME_DIR = Path.home() OSS_FUZZ_DIR = HOME_DIR / "FuzzAug" / "fuzz" / "oss-fuzz" LOG_DIR = OSS_FUZZ_DIR / "run_logs" def setup_logging(project_name: str) -> Path: - """创建带时间戳的运行日志""" + """Create a timestamped run log""" LOG_DIR.mkdir(parents=True, exist_ok=True) timestamp = datetime.now().strftime("%Y%m%d%H%M%S") return LOG_DIR / f"run_{project_name}_{timestamp}.log" def log_and_print(message: str, log_file: Path, to_stdout: bool = True): - """日志和控制台输出""" + """Log and console output""" if to_stdout: print(f"[PID:{os.getpid()}] {message}") with open(log_file, "a", encoding="utf-8") as f: @@ -44,7 +44,7 @@ def run_command( log_file: Path, allowed_exit_codes: Optional[List[int]] = None ) -> bool: - """执行命令并实时记录输出""" + """Execute command and log output in real-time""" allowed_exit_codes = allowed_exit_codes or [] log_and_print(f"▶️ {log_msg}...", log_file, to_stdout=False) log_and_print(f" $ {cmd}", log_file, to_stdout=False) @@ -65,11 +65,11 @@ def run_command( process.wait() return process.returncode in [0, *allowed_exit_codes] except Exception as e: - log_and_print(f"💥 执行异常: {e}", log_file) + log_and_print(f"💥 Execution exception: {e}", log_file) return False def discover_targets(project_name: str) -> List[str]: - """发现可用的 Fuzz 目标""" + """Discover available Fuzz targets""" out_dir = OSS_FUZZ_DIR / "build" / "out" / project_name targets = [] if out_dir.exists(): @@ -79,62 +79,62 @@ def discover_targets(project_name: str) -> List[str]: return targets def run_project(project_name: str, timeout: int) -> Tuple[bool, str]: - """单个项目的测试工作流""" + """Testing workflow for a single project""" log_file = setup_logging(project_name) os.chdir(OSS_FUZZ_DIR) log_and_print("="*60, log_file) - log_and_print(f"🚀 开始测试项目: {project_name}", log_file) - log_and_print(f"📝 日志路径: {log_file}", log_file) + log_and_print(f"🚀 Starting testing for project: {project_name}", log_file) + log_and_print(f"📝 Log path: {log_file}", log_file) log_and_print("="*60, log_file) - # 1. 发现测试目标 + # 1. Discover test targets targets = discover_targets(project_name) if not targets: - log_and_print("⚠️ 未发现测试目标", log_file) + log_and_print("⚠️ No test targets found", log_file) return (False, project_name) - log_and_print(f"🔍 发现 {len(targets)} 个测试目标", log_file) + log_and_print(f"🔍 Discovered {len(targets)} test targets", log_file) - # 2. 运行所有目标 + # 2. Run all targets all_success = True for i, target in enumerate(targets, 1): cmd = f"python3 infra/helper.py run_fuzzer {project_name} {target} -- -max_total_time={timeout}" success = run_command( cmd, - f"运行目标 [{i}/{len(targets)}] {target} (超时={timeout}s)", + f"Running target [{i}/{len(targets)}] {target} (timeout={timeout}s)", log_file, - allowed_exit_codes=[1, 124] # 允许超时退出 + allowed_exit_codes=[1, 124] # Allow timeout exit codes ) all_success &= success - # 3. 生成报告（占位） - log_and_print("📊 覆盖率报告生成 (当前版本暂未实现)", log_file) + # 3. Generate report (placeholder) + log_and_print("📊 Coverage report generation (not implemented in current version)", log_file) return (all_success, project_name) def main(): - parser = argparse.ArgumentParser(description="OSS-Fuzz 并行测试工具") - parser.add_argument("project_list", help="项目列表文件路径") - parser.add_argument("--timeout", type=int, default=60, help="单目标测试超时时间（秒）") + parser = argparse.ArgumentParser(description="OSS-Fuzz Parallel Testing Tool") + parser.add_argument("project_list", help="Project list file path") + parser.add_argument("--timeout", type=int, default=60, help="Timeout per target test (seconds)") parser.add_argument("--workers", type=int, default=cpu_count()) args = parser.parse_args() - # 读取项目列表 + # Read project list try: with open(args.project_list) as f: projects = [line.strip() for line in f if line.strip()] except Exception as e: - print(f"❌ 读取项目列表失败: {e}") + print(f"❌ Failed to read project list: {e}") sys.exit(1) - # 并行运行 + # Parallel execution with Pool(args.workers) as pool: results = pool.starmap(run_project, [(p, args.timeout) for p in projects]) - # 输出结果 + # Output results failed = [p for success, p in results if not success] - print(f"\n📊 测试完成: 成功 {len(projects)-len(failed)}/{len(projects)}") + print(f"\n📊 Testing completed: Success {len(projects)-len(failed)}/{len(projects)}") if failed: - print("❌ 失败项目: " + ", ".join(failed)) + print("❌ Failed projects: " + ", ".join(failed)) if __name__ == "__main__": - main() \ No newline at end of file + main() From bb5f14aa0f1c8956f2ab19d9e2bacb0a0a59d2d6 Mon Sep 17 00:00:00 2001 From: joyguoguo <599325370@qq.com> Date: Tue, 22 Jul 2025 03:15:29 +0000 Subject: [PATCH 11/15] fuzz_runner_pool.py:74 --- fuzz/fuzz_runner_pool.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/fuzz/fuzz_runner_pool.py b/fuzz/fuzz_runner_pool.py index 996b940..3580930 100644 --- a/fuzz/fuzz_runner_pool.py +++ b/fuzz/fuzz_runner_pool.py @@ -71,8 +71,9 @@ def run_command( ) with open(log_file, "a", encoding="utf-8") as f: - for line in iter(process.stdout.readline, ''): - f.write(line) # Write to log only to avoid console clutter + if process.stdout is not None: # 显式检查 + for line in iter(process.stdout.readline, ""): + f.write(line) # Write to log only to avoid console clutter process.wait() exit_code = process.returncode From 1b9b0101071faf8b66a06051adfdb16d68644e44 Mon Sep 17 00:00:00 2001 From: joyguoguo <599325370@qq.com> Date: Tue, 22 Jul 2025 03:19:18 +0000 Subject: [PATCH 12/15] edit stdout --- fuzz/fuzz_runner_pool.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fuzz/fuzz_runner_pool.py b/fuzz/fuzz_runner_pool.py index 3580930..6a2e4dd 100644 --- a/fuzz/fuzz_runner_pool.py +++ b/fuzz/fuzz_runner_pool.py @@ -49,7 +49,7 @@ def log_and_print(message: str, log_file: Path, to_stdout: bool = True): def run_command( cmd: str, log_msg: str, - log_file: Path, + log_file: TextIO , allowed_exit_codes: Optional[List[int]] = None, auto_confirm: bool = True # New auto-confirm parameter ) -> bool: From 49e9dddf1d45fc0ddd240197b92aa6e910b98d67 Mon Sep 17 00:00:00 2001 From: joyguoguo <599325370@qq.com> Date: Tue, 22 Jul 2025 03:23:41 +0000 Subject: [PATCH 13/15] =?UTF-8?q?=E6=B7=BB=E5=8A=A0=E7=A9=BA=E5=80=BC?= =?UTF-8?q?=E6=A3=80=E6=9F=A5?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- fuzz/build_oss_fuzz.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/fuzz/build_oss_fuzz.py b/fuzz/build_oss_fuzz.py index 1c78a0d..fdd894e 100644 --- a/fuzz/build_oss_fuzz.py +++ b/fuzz/build_oss_fuzz.py @@ -62,8 +62,11 @@ def run_command( errors="replace" ) with open(log_file, "a", encoding="utf-8") as f: - for line in iter(process.stdout.readline, ""): - f.write(line) + if process.stdout is not None: + for line in iter(process.stdout.readline, ""): + f.write(line) + else: + log_and_print("⚠️ Warning: process.stdout is None", log_file) process.wait() exit_code = process.returncode if exit_code in [0, *allowed_exit_codes]: From 6e5221dc52b5234b59e2fe42665e92347071a815 Mon Sep 17 00:00:00 2001 From: joyguoguo <599325370@qq.com> Date: Tue, 22 Jul 2025 03:27:29 +0000 Subject: [PATCH 14/15] modify stdout, delete pool.py --- fuzz/fuzz_runner_pool.py | 271 --------------------------------------- fuzz/run_fuzz_target.py | 7 +- 2 files changed, 5 insertions(+), 273 deletions(-) delete mode 100644 fuzz/fuzz_runner_pool.py diff --git a/fuzz/fuzz_runner_pool.py b/fuzz/fuzz_runner_pool.py deleted file mode 100644 index 6a2e4dd..0000000 --- a/fuzz/fuzz_runner_pool.py +++ /dev/null @@ -1,271 +0,0 @@ -#!/usr/bin/env python3 -# -*- coding: utf-8 -*- - -""" -fuzz_runner_pool.py - -Parallel batch execution of the entire OSS-Fuzz local testing process. -Uses multiprocessing.Pool to distribute projects across multiple CPU cores for concurrent processing. - -Usage: python3 fuzz_runner_pool.py [project_list_file] [--sanitizer type] [--workers N] -Example: python3 fuzz/fuzz_runner_pool.py data/valid_projects.txt --workers 4 -""" - -import os -import sys -import subprocess -import argparse -from datetime import datetime -from pathlib import Path -from typing import List, Optional, Tuple -from multiprocessing import Pool, cpu_count - -# --- Global configuration (can be overridden by command line arguments) --- -HOME_DIR = Path.home() -OSS_FUZZ_DIR = HOME_DIR / "FuzzAug" / "fuzz"/"oss-fuzz" -LOG_DIR = OSS_FUZZ_DIR / "script_pool_batch_logs" - -def setup_logging(project_name: str) -> Path: - """Create a timestamped log file for a single project.""" - LOG_DIR.mkdir(parents=True, exist_ok=True) - try: - LOG_DIR.chmod(0o777) - except PermissionError: - # In a parallel environment, multiple processes may try simultaneously, printing a warning once is sufficient - pass - - timestamp = datetime.now().strftime("%Y%m%d%H%M%S") - log_file_path = LOG_DIR / f"oss_fuzz_{project_name}_{timestamp}.log" - return log_file_path - -def log_and_print(message: str, log_file: Path, to_stdout: bool = True): - """Write message to log file and optionally print to console.""" - if to_stdout: - # Add process ID to distinguish parallel outputs - print(f"[PID:{os.getpid()}] {message}") - with open(log_file, "a", encoding="utf-8") as f: - f.write(message + "\n") - -def run_command( - cmd: str, - log_msg: str, - log_file: TextIO , - allowed_exit_codes: Optional[List[int]] = None, - auto_confirm: bool = True # New auto-confirm parameter -) -> bool: - """Execute a shell command and stream output to log file in real-time.""" - if allowed_exit_codes is None: - allowed_exit_codes = [] - - log_and_print(f"▶️ {log_msg}...", log_file, to_stdout=False) - log_and_print(f" $ {cmd}", log_file, to_stdout=False) - - try: - # Add auto-confirm mechanism - if auto_confirm: - cmd = f"yes | {cmd}" - - process = subprocess.Popen( - cmd, shell=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, - text=True, encoding='utf-8', errors='replace', bufsize=1 - ) - - with open(log_file, "a", encoding="utf-8") as f: - if process.stdout is not None: # 显式检查 - for line in iter(process.stdout.readline, ""): - f.write(line) # Write to log only to avoid console clutter - - process.wait() - exit_code = process.returncode - - if exit_code == 0: - log_and_print(f"✅ Command completed successfully.", log_file, to_stdout=False) - return True - elif exit_code in allowed_exit_codes: - log_and_print(f"ℹ️ Command exited with expected status: {exit_code}", log_file, to_stdout=False) - return True - else: - log_and_print(f"❌ Command execution failed (exit code: {exit_code})", log_file) - return False - except Exception as e: - log_and_print(f"💥 Exception occurred while executing command: {e}", log_file) - return False - -def discover_fuzz_targets(project_name: str) -> List[str]: - """Automatically discover Fuzz targets.""" - project_out_dir = OSS_FUZZ_DIR / "build" / "out" / project_name - project_src_dir = OSS_FUZZ_DIR / "projects" / project_name - targets = [] - - if project_out_dir.is_dir(): - for f in project_out_dir.iterdir(): - if (f.is_file() and os.access(f, os.X_OK) and - f.name.startswith("fuzz_") and '.' not in f.name): - targets.append(f.name) - - if not targets and project_src_dir.is_dir(): - for py_file in project_src_dir.glob("fuzz_*.py"): - try: - with open(py_file, "r", encoding="utf-8") as f: - if "atheris.Setup" in f.read(): - targets.append(py_file.stem) - except Exception as e: - # In worker processes, only print to own log - print(f"⚠️ Warning: Failed to read file {py_file}: {e}") - return targets - -def run_project_workflow(project_name: str, sanitizer: str) -> Tuple[bool, str]: - """ - Handle the complete workflow for a single project (Worker Function). - This function is executed independently by each worker process in the process pool. - - Returns: - A tuple (is_success: bool, project_name: str) - """ - log_file = setup_logging(project_name) - - # Print at the beginning of worker for tracking - log_and_print("=" * 60, log_file) - log_and_print(f"🚀 Starting processing for project: {project_name}", log_file) - log_and_print(f"📝 Log file: {log_file}", log_file) - log_and_print("=" * 60, log_file) - - # Each process needs to set its own working directory - try: - os.chdir(OSS_FUZZ_DIR) - except FileNotFoundError: - log_and_print(f"❌ Critical error: OSS-Fuzz directory '{OSS_FUZZ_DIR}' does not exist!", log_file) - return (False, project_name) - - # Step 1: Build Docker image (with auto-confirm enabled) - if not run_command( - f"python3 infra/helper.py build_image {project_name}", - f"Step 1/5: Building Docker image for {project_name}", log_file, - auto_confirm=True # Auto-confirm all prompts - ): - log_and_print(f"❌ Project {project_name} failed to build image", log_file) - return (False, project_name) - - # Step 2: Compile fuzzers with sanitizer (with auto-confirm enabled) - if not run_command( - f"python3 infra/helper.py build_fuzzers --sanitizer {sanitizer} {project_name}", - f"Step 2/5: Compiling fuzzers for {project_name} (sanitizer={sanitizer})", log_file, - auto_confirm=True # Auto-confirm all prompts - ): - log_and_print(f"❌ Project {project_name} failed to compile fuzzers", log_file) - return (False, project_name) - - # Step 3: Automatically discover targets - log_and_print(f"🔍 Automatically discovering fuzz targets for {project_name}...", log_file) - fuzz_targets = discover_fuzz_targets(project_name) - - if not fuzz_targets: - log_and_print(f"⚠️ Warning: No fuzz targets found for {project_name}! Skipping run step.", log_file) - return (True, project_name) - - log_and_print(f"✅ Targets discovered: {', '.join(fuzz_targets)}", log_file) - - # Step 4: Run all targets (with auto-confirm enabled) - for i, target in enumerate(fuzz_targets, 1): - run_command( - f"python3 infra/helper.py run_fuzzer {project_name} {target} -- -max_total_time=60", - f"Step 4/{len(fuzz_targets)}: Running target [{target}] (60 seconds)", log_file, - allowed_exit_codes=[1, 124], - auto_confirm=True # Auto-confirm all prompts - ) - - # Step 5: Generate coverage report (not implemented yet) - log_and_print("Step 5/5: Generating coverage report (not implemented in current version)", log_file) - log_and_print(f"✅ Project {project_name} processing completed!", log_file) - return (True, project_name) - -def main(): - """ - Main workflow function: Set up process pool and distribute tasks. - """ - parser = argparse.ArgumentParser( - description="OSS-Fuzz Parallel Batch Testing Tool", - formatter_class=argparse.RawTextHelpFormatter, - epilog="Examples:\n python3 fuzz_runner_parallel.py valid_projects.txt --workers 4\n python3 fuzz_runner_parallel.py my_projects.txt --sanitizer undefined" - ) - parser.add_argument( - "project_list_file", nargs="?", default="valid_projects.txt", - help="Text file containing list of projects to test. (Default: valid_projects.txt)" - ) - parser.add_argument( - "--sanitizer", default="address", choices=["address", "memory", "undefined", "coverage"], - help="Type of sanitizer to use. (Default: address)" - ) - parser.add_argument( - "--workers", type=int, default=cpu_count(), - help=f"Number of concurrent worker processes. (Default: system CPU count, currently {cpu_count()})" - ) - args = parser.parse_args() - - # --- Environment checks --- - if not OSS_FUZZ_DIR.is_dir(): - print(f"❌ Error: OSS-Fuzz directory '{OSS_FUZZ_DIR}' does not exist!") - sys.exit(1) - - project_list_path = Path(args.project_list_file) - if not project_list_path.is_file(): - print(f"❌ Error: Project list file '{project_list_path}' does not exist!") - sys.exit(1) - - print(f"✅ Environment checks passed. Will use {args.workers} parallel worker processes.") - - # --- Read and prepare tasks --- - try: - with open(project_list_path, "r", encoding="utf-8") as f: - projects = [line.strip() for line in f if line.strip() and not line.strip().startswith("#")] - except Exception as e: - print(f"❌ Error reading project list file: {e}") - sys.exit(1) - - if not projects: - print("⚠️ Warning: Project list is empty, no tasks to execute.") - sys.exit(0) - - # Prepare task parameters for starmap, each project is a (project_name, sanitizer) tuple - tasks = [(project, args.sanitizer) for project in projects] - total_projects = len(tasks) - - print(f"\n🚀 About to process {total_projects} projects in parallel...") - - # --- Execute parallel processing --- - # Use with statement to ensure proper pool cleanup - with Pool(processes=args.workers) as pool: - # starmap blocks until all tasks complete - # Unpacks each tuple in tasks as arguments to worker function - results = pool.starmap(run_project_workflow, tasks) - - # --- Collect and print results --- - failed_projects = [] - for success, project_name in results: - if success: - print(f"✅ Project {project_name} completed successfully") - else: - print(f"❌ Project {project_name} processing failed") - failed_projects.append(project_name) - - # --- Final summary --- - fail_count = len(failed_projects) - success_count = total_projects - fail_count - - print("\n" + "=" * 60) - print("🎉 Batch processing completed!") - print(f"📊 Total: {total_projects} projects") - print(f"✅ Success: {success_count}") - print(f"❌ Failures: {fail_count}") - - if failed_projects: - print("📛 Failed projects list:") - for proj in sorted(failed_projects): - print(f" • {proj}") - print("\n💡 Tip: Detailed information for failed projects can be found in corresponding log files.") - print(f" Log directory: {LOG_DIR}") - -if __name__ == "__main__": - # On some Python versions for Windows or macOS, main call needs to be in this guard - # To prevent child processes from re-importing and executing main module code, causing infinite recursion. - main() diff --git a/fuzz/run_fuzz_target.py b/fuzz/run_fuzz_target.py index 5b43a9e..aa6bafb 100644 --- a/fuzz/run_fuzz_target.py +++ b/fuzz/run_fuzz_target.py @@ -60,8 +60,11 @@ def run_command( errors="replace" ) with open(log_file, "a", encoding="utf-8") as f: - for line in iter(process.stdout.readline, ""): - f.write(line) + if process.stdout is not None: + for line in iter(process.stdout.readline, ""): + f.write(line) + else: + log_and_print("⚠️ Warning: process.stdout is None", log_file) process.wait() return process.returncode in [0, *allowed_exit_codes] except Exception as e: From 4a5befabedf3cc425b7bc78fbb0d07a51ba62c1e Mon Sep 17 00:00:00 2001 From: joyguoguo <599325370@qq.com> Date: Tue, 22 Jul 2025 03:32:03 +0000 Subject: [PATCH 15/15] indentation level check --- fuzz/oss-fuzz | 2 +- fuzz/run_fuzz_target.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/fuzz/oss-fuzz b/fuzz/oss-fuzz index 0b81ba5..f73b405 160000 --- a/fuzz/oss-fuzz +++ b/fuzz/oss-fuzz @@ -1 +1 @@ -Subproject commit 0b81ba5d97ae3d1402744e00b1d9075fed7b7f1e +Subproject commit f73b405d84e886bac90f8b15200230f08a2709c9 diff --git a/fuzz/run_fuzz_target.py b/fuzz/run_fuzz_target.py index aa6bafb..4a9f15a 100644 --- a/fuzz/run_fuzz_target.py +++ b/fuzz/run_fuzz_target.py @@ -60,7 +60,7 @@ def run_command( errors="replace" ) with open(log_file, "a", encoding="utf-8") as f: - if process.stdout is not None: + if process.stdout is not None: for line in iter(process.stdout.readline, ""): f.write(line) else: