From d3738a33b907dac23f171f96e8153a4909ec6575 Mon Sep 17 00:00:00 2001 From: cc06 Date: Thu, 25 Jun 2026 18:16:48 +0800 Subject: [PATCH 1/7] Add Venus IPS log query service package --- services/bin/octobus-tentacles.js | 4 + services/bin/venus-ips.js | 10 + services/package.json | 3 + services/venus__ips/README.md | 72 ++++++ services/venus__ips/bin/venus-ips.js | 6 + services/venus__ips/config.schema.json | 15 ++ services/venus__ips/package.json | 12 + services/venus__ips/proto/venus_ips.proto | 41 ++++ services/venus__ips/secret.schema.json | 10 + services/venus__ips/service.json | 20 ++ services/venus__ips/src/service.js | 7 + services/venus__ips/src/venus-ips.js | 250 +++++++++++++++++++++ services/venus__ips/test/mock_upstream.js | 47 ++++ services/venus__ips/test/venus-ips.test.js | 194 ++++++++++++++++ 14 files changed, 691 insertions(+) create mode 100644 services/bin/venus-ips.js create mode 100644 services/venus__ips/README.md create mode 100644 services/venus__ips/bin/venus-ips.js create mode 100644 services/venus__ips/config.schema.json create mode 100644 services/venus__ips/package.json create mode 100644 services/venus__ips/proto/venus_ips.proto create mode 100644 services/venus__ips/secret.schema.json create mode 100644 services/venus__ips/service.json create mode 100644 services/venus__ips/src/service.js create mode 100644 services/venus__ips/src/venus-ips.js create mode 100644 services/venus__ips/test/mock_upstream.js create mode 100644 services/venus__ips/test/venus-ips.test.js diff --git a/services/bin/octobus-tentacles.js b/services/bin/octobus-tentacles.js index ea8155f4..919414fb 100755 --- a/services/bin/octobus-tentacles.js +++ b/services/bin/octobus-tentacles.js @@ -461,6 +461,10 @@ const services = { entryFile: "../topsec__edr/bin/topsec-edr.js", serviceModule: "../topsec__edr/src/service.js", }, + "venus-ips": { + entryFile: "../venus__ips/bin/venus-ips.js", + serviceModule: "../venus__ips/src/service.js", + }, "venus-ads-v3-6": { entryFile: "../venus__ads_v3-6/bin/venus-ads-v3-6.js", serviceModule: "../venus__ads_v3-6/src/service.js", diff --git a/services/bin/venus-ips.js b/services/bin/venus-ips.js new file mode 100644 index 00000000..42608733 --- /dev/null +++ b/services/bin/venus-ips.js @@ -0,0 +1,10 @@ +#!/usr/bin/env node + +import { fileURLToPath } from "node:url"; +import { runServiceMain } from "@chaitin-ai/octobus-sdk"; + +import { service } from "../venus__ips/src/service.js"; + +runServiceMain(service, { + entryFile: fileURLToPath(new URL("../venus__ips/bin/venus-ips.js", import.meta.url)), +}); diff --git a/services/package.json b/services/package.json index 6764a88d..882725f6 100644 --- a/services/package.json +++ b/services/package.json @@ -127,6 +127,7 @@ "leadsec-waf": "bin/leadsec-waf.js", "venus-ips-v6079": "bin/venus-ips-v6079.js", "venus-tar": "bin/venus-tar.js", + "venus-ips": "bin/venus-ips.js", "wangsu-label-ip": "bin/wangsu-label-ip.js", "wd-k01": "bin/wd-k01.js", "threatbook-hfish": "bin/threatbook-hfish.js", @@ -289,6 +290,7 @@ "bin/leadsec-waf.js", "bin/venus-ips-v6079.js", "bin/venus-tar.js", + "bin/venus-ips.js", "bin/wd-k01.js", "bin/threatbook-hfish.js", "bin/dongtai-iast.js", @@ -408,6 +410,7 @@ "leadsec__waf", "venus__ips_v6079", "venus__tar", + "venus__ips", "wd__k01", "huoxian__dongtai-iast", "filigran__opencti", diff --git a/services/venus__ips/README.md b/services/venus__ips/README.md new file mode 100644 index 00000000..15e2c3b9 --- /dev/null +++ b/services/venus__ips/README.md @@ -0,0 +1,72 @@ +# Venustech IPS + +启明星辰 IPS(入侵防御系统)攻击日志查询的 OctoBus service package。 +属于「流量检测 / NIPS」类。设备日志页返回 HTML,本包解析其中的日志表行为结构化条目。 + +## 支持版本 + +启明星辰 IPS(web 控制台,日志页 `/log/memorylog/ipslog.php`)。请求/响应按真机抓包对齐。 + +## 认证方式(web 会话 Cookie) + +控制台以 **会话 Cookie** 鉴权(浏览器 `credentials: include`)。请求头 `Cookie: <会话cookie>`, +cookie 经 `secret.cookie` 外部传入。 + +> ⚠️ cookie 有时效,过期需更换。会话失效时设备会以 200 返回登录页;本包用日志页标记 +> (`ips_log_filter`)识别,识别失败时报 `FAILED_PRECONDITION`,避免把登录页当成空结果。 + +## 配置 + +```json +// config +{ "host": "https://192.168.1.10", "timeoutMs": 5000, "skipTlsVerify": true } +// secret +{ "cookie": "PHPSESSID=<会话id>" } +``` + +## 方法 + +| RPC | 上游接口 | +| --- | --- | +| `QueryIpsLog` | `GET /log/memorylog/ipslog.php` | + +### 请求 / 响应 + +- 请求:`limit`(返回条目上限,客户端侧截断;<=0 表示全部)。 +- 响应:`http_status`、`total`(解析到的条目数)、`entries[]`。每条 `entries` 含: + `name`(名称)、`src_ip`/`src_port`、`dst_ip`/`dst_port`、`protocol`、`time`、`type`(类型)、 + `severity`(事件级别)、`priority`(优先级)、`action`(动作)、`policy_id`(策略ID)、`count`(发生次数)、`content`(内容)。 + +> 注:当前抓包为不带过滤的全量 GET;按源/目的 IP/时间过滤(设备 `ips_log_filter` 表单)未实现, +> 待补对应抓包后扩展。响应原始 HTML(较大且含内网地址)不回传,仅返回结构化条目。 + +## 风险边界 + +- 本方法为**只读查询**,无写操作,风险面低。 +- 会话 cookie 等同登录态,泄露即会话失陷;仅放 `secret`,勿写入 `config`、日志或截图。 +- 默认校验 TLS;私有自签部署需 `skipTlsVerify: true`。 + +## 错误映射 + +| 场景 | gRPC code | +| --- | --- | +| 缺 host/cookie | `INVALID_ARGUMENT` | +| 上游 401/403 | `PERMISSION_DENIED` | +| 其它 4xx / 会话失效(返回登录页) | `FAILED_PRECONDITION` | +| 网络错误/超时/5xx | `UNAVAILABLE` | + +## 建议 capset + +`query-ips-log`(只读),可直接授权给 AI SOC / 工作流做告警拉取与研判。 + +## 验证方式 + +```bash +cd services +npm run validate -- --service-dir venus__ips +npm test -- --service-dir venus__ips --coverage +npm run pack:check +``` + +真机验证:用一个有效会话 cookie 调 `query-ips-log`,确认返回 IPS 告警条目(名称/源IP/目的IP/时间/级别/动作 等)。 +PR 附**真机验证截图**(cookie / host / 响应中的内网 IP 等敏感数据已打码)。**代码/测试/截图里不得出现真实 cookie、内网地址或业务数据。** diff --git a/services/venus__ips/bin/venus-ips.js b/services/venus__ips/bin/venus-ips.js new file mode 100644 index 00000000..8e0066ef --- /dev/null +++ b/services/venus__ips/bin/venus-ips.js @@ -0,0 +1,6 @@ +#!/usr/bin/env node +import { runServiceMain } from '@chaitin-ai/octobus-sdk'; + +import { service } from '../src/service.js'; + +runServiceMain(service); diff --git a/services/venus__ips/config.schema.json b/services/venus__ips/config.schema.json new file mode 100644 index 00000000..a7cccd60 --- /dev/null +++ b/services/venus__ips/config.schema.json @@ -0,0 +1,15 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "type": "object", + "additionalProperties": true, + "properties": { + "host": { "type": "string", "description": "Venus IPS console base URL with scheme, e.g. https://192.168.1.10." }, + "restBaseUrl": { "type": "string", "description": "Alias for host." }, + "baseUrl": { "type": "string", "description": "Alias for host." }, + "timeoutMs": { "type": "integer", "minimum": 1, "default": 5000, "description": "HTTP timeout in milliseconds." }, + "skipTlsVerify": { "type": "boolean", "default": false, "description": "Skip TLS certificate verification for private deployments." }, + "tlsInsecureSkipVerify": { "type": "boolean", "default": false, "description": "Legacy alias for skipTlsVerify." }, + "insecureSkipVerify": { "type": "boolean", "default": false, "description": "Alias for skipTlsVerify." }, + "headers": { "type": "object", "additionalProperties": { "type": "string" }, "description": "Optional additional HTTP headers." } + } +} diff --git a/services/venus__ips/package.json b/services/venus__ips/package.json new file mode 100644 index 00000000..4bc8fa38 --- /dev/null +++ b/services/venus__ips/package.json @@ -0,0 +1,12 @@ +{ + "name": "venus-ips", + "version": "0.0.0", + "private": true, + "type": "module", + "bin": { + "venus-ips": "bin/venus-ips.js" + }, + "dependencies": { + "@chaitin-ai/octobus-sdk": "^0.5.0" + } +} diff --git a/services/venus__ips/proto/venus_ips.proto b/services/venus__ips/proto/venus_ips.proto new file mode 100644 index 00000000..8a6a083f --- /dev/null +++ b/services/venus__ips/proto/venus_ips.proto @@ -0,0 +1,41 @@ +syntax = "proto3"; + +package VENUS_IPS; + +option go_package = "miner/grpc-service/VENUS_IPS"; + +// 启明星辰 IPS(入侵防御系统)攻击日志查询。 +// 认证为 web 会话 Cookie:GET /log/memorylog/ipslog.php，浏览器 credentials=include。 +// 设备返回 HTML 日志页,本 service 解析其中的日志表行,映射为结构化条目。 +service VENUS_IPS { + // 查询 IPS 攻击日志(内存日志): GET /log/memorylog/ipslog.php + rpc QueryIpsLog(QueryIpsLogRequest) returns (QueryIpsLogResponse) {} +} + +message QueryIpsLogRequest { + int32 limit = 1; // 返回条目上限(客户端侧截断)，<=0 表示全部 +} + +// 对应日志表列:名称/源IP/源端口/目的IP/目的端口/协议/时间/类型/级别/优先级/动作/策略ID/次数/内容。 +message IpsLogEntry { + string name = 1; // 名称 + string src_ip = 2; // 源IP + string src_port = 3; // 源端口 + string dst_ip = 4; // 目的IP + string dst_port = 5; // 目的端口 + string protocol = 6; // 协议类型 + string time = 7; // 时间 + string type = 8; // 类型 + string severity = 9; // 事件级别 + string priority = 10; // 优先级 + string action = 11; // 动作 + string policy_id = 12;// 入侵防御策略ID + string count = 13; // 发生次数 + string content = 14; // 内容 +} + +message QueryIpsLogResponse { + int32 http_status = 1; // 上游 HTTP 状态码 + int32 total = 2; // 解析到的条目数 + repeated IpsLogEntry entries = 3; // 结构化日志条目 +} diff --git a/services/venus__ips/secret.schema.json b/services/venus__ips/secret.schema.json new file mode 100644 index 00000000..c7dd01c4 --- /dev/null +++ b/services/venus__ips/secret.schema.json @@ -0,0 +1,10 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "type": "object", + "additionalProperties": true, + "properties": { + "cookie": { "type": "string", "description": "Venus IPS web session cookie header value (e.g. 'PHPSESSID=...'). Time-limited; refresh when expired." }, + "sessionCookie": { "type": "string", "description": "Alias for cookie." }, + "session_cookie": { "type": "string", "description": "Alias for cookie." } + } +} diff --git a/services/venus__ips/service.json b/services/venus__ips/service.json new file mode 100644 index 00000000..323ea322 --- /dev/null +++ b/services/venus__ips/service.json @@ -0,0 +1,20 @@ +{ + "schema": "chaitin.octobus.service.v1", + "name": "venus-ips", + "displayName": "Venustech IPS", + "description": "OctoBus package for Venustech IPS attack-log query (GET /log/memorylog/ipslog.php, web session cookie, HTML log page parsed into structured entries).", + "runtime": { "mode": "long-running" }, + "proto": { "roots": ["proto"], "files": ["proto/venus_ips.proto"] }, + "configSchema": "config.schema.json", + "secretSchema": "secret.schema.json", + "sdk": { + "cli": { + "commands": { + "VENUS_IPS.VENUS_IPS/QueryIpsLog": { + "name": "query-ips-log", + "description": "Query Venus IPS attack logs (memory log)." + } + } + } + } +} diff --git a/services/venus__ips/src/service.js b/services/venus__ips/src/service.js new file mode 100644 index 00000000..0fe48e87 --- /dev/null +++ b/services/venus__ips/src/service.js @@ -0,0 +1,7 @@ +import { defineService } from '@chaitin-ai/octobus-sdk'; + +import { handlers } from './venus-ips.js'; + +export { handlers } from './venus-ips.js'; + +export const service = defineService({ handlers }); diff --git a/services/venus__ips/src/venus-ips.js b/services/venus__ips/src/venus-ips.js new file mode 100644 index 00000000..f91cf9e8 --- /dev/null +++ b/services/venus__ips/src/venus-ips.js @@ -0,0 +1,250 @@ +// 启明星辰 IPS 攻击日志查询适配。 +// 认证:web 会话 Cookie。GET /log/memorylog/ipslog.php 返回 HTML 日志页，解析表行为结构化条目。 +import { GrpcError, grpcStatus } from '@chaitin-ai/octobus-sdk'; + +const SVC = 'VENUS_IPS.VENUS_IPS'; +export const QUERY_IPS_LOG_PATH = `/${SVC}/QueryIpsLog`; +export const METHOD_QUERY_IPS_LOG_FULL = `${SVC}/QueryIpsLog`; + +export const IPS_LOG_URI = '/log/memorylog/ipslog.php'; +export const LOG_PAGE_MARKER = 'ips_log_filter'; +export const DEFAULT_TIMEOUT_MS = 5000; + +// 日志表 14 个有 title 的数据单元格，按列顺序映射。 +const ENTRY_FIELDS = [ + 'name', 'src_ip', 'src_port', 'dst_ip', 'dst_port', 'protocol', + 'time', 'type', 'severity', 'priority', 'action', 'policy_id', 'count', 'content', +]; +const DATETIME_RE = /\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}/; + +const grpcCodeFor = (code) => ({ + FAILED_PRECONDITION: grpcStatus.FAILED_PRECONDITION, + INVALID_ARGUMENT: grpcStatus.INVALID_ARGUMENT, + PERMISSION_DENIED: grpcStatus.PERMISSION_DENIED, + UNAVAILABLE: grpcStatus.UNAVAILABLE, + UNKNOWN: grpcStatus.UNKNOWN, +})[code] ?? grpcStatus.UNKNOWN; + +const errorWithCode = (code, message) => { + const err = new GrpcError(grpcCodeFor(code), `${code}: ${message}`); + err.legacyCode = code; + return err; +}; + +const hasOwn = (obj, key) => Object.prototype.hasOwnProperty.call(obj ?? {}, key); + +const unwrapScalar = (value) => { + if (value === undefined || value === null) return undefined; + if (typeof value === 'object' && hasOwn(value, 'value')) return unwrapScalar(value.value); + return value; +}; + +const pickFirstString = (values = []) => { + for (const value of values) { + const raw = unwrapScalar(value); + if (raw === undefined || raw === null) continue; + const str = String(raw).trim(); + if (str) return str; + } + return ''; +}; + +const pickStringFrom = (source = {}, keys = []) => { + for (const key of keys) { + if (!hasOwn(source, key)) continue; + const raw = unwrapScalar(source[key]); + if (raw === undefined || raw === null) continue; + const value = String(raw).trim(); + if (value) return value; + } + return ''; +}; + +const pickInt = (source = {}, keys = [], fallback = 0) => { + for (const key of keys) { + if (!hasOwn(source, key)) continue; + const raw = unwrapScalar(source[key]); + if (raw === undefined || raw === null || raw === '') continue; + const num = Number(raw); + if (Number.isFinite(num)) return Math.trunc(num); + } + return fallback; +}; + +const pickBoolean = (value) => { + const raw = unwrapScalar(value); + if (raw === undefined || raw === null) return undefined; + if (typeof raw === 'boolean') return raw; + if (typeof raw === 'number') return Number.isNaN(raw) ? undefined : raw !== 0; + if (typeof raw === 'string') { + const normalized = raw.trim().toLowerCase(); + if (['true', '1', 'yes', 'y', 'on'].includes(normalized)) return true; + if (['false', '0', 'no', 'n', 'off', ''].includes(normalized)) return false; + } + return undefined; +}; + +const pickFirstBoolean = (values = []) => { + for (const value of values) { + const bool = pickBoolean(value); + if (bool !== undefined) return bool; + } + return undefined; +}; + +const normalizeBaseUrl = (value) => { + const raw = String(unwrapScalar(value) || '').trim(); + if (!/^https?:\/\//i.test(raw)) return ''; + return raw.replace(/\/+$/, ''); +}; + +const resolveCallContext = (ctx = {}) => ({ + ...ctx, + bindings: { + ...(ctx.config ?? {}), + ...(ctx.secret ?? {}), + ...(ctx.bindings ?? {}), + }, + limits: ctx.limits ?? {}, + meta: ctx.meta ?? {}, + req: ctx.req ?? ctx.request ?? {}, +}); + +const resolveHost = (bindings = {}) => normalizeBaseUrl(pickFirstString([bindings.host, bindings.restBaseUrl, bindings.baseUrl])); +const resolveCookie = (bindings = {}) => pickStringFrom(bindings, ['cookie', 'sessionCookie', 'session_cookie']); + +const resolveTimeoutMs = (ctx = {}) => { + const raw = Number(unwrapScalar(ctx.limits?.timeoutMs ?? ctx.bindings?.timeoutMs ?? DEFAULT_TIMEOUT_MS)); + return Number.isFinite(raw) && raw > 0 ? raw : DEFAULT_TIMEOUT_MS; +}; + +const buildTlsOptions = (bindings = {}) => { + const enabled = pickFirstBoolean([bindings.skipTlsVerify, bindings.tlsInsecureSkipVerify, bindings.insecureSkipVerify]) || false; + return enabled ? { skipTlsVerify: true, tlsInsecureSkipVerify: true, insecureSkipVerify: true } : {}; +}; + +const sanitizeHeaders = (headers) => { + const raw = unwrapScalar(headers); + if (!raw || typeof raw !== 'object' || Array.isArray(raw)) return {}; + return Object.fromEntries(Object.entries(raw).filter(([key]) => key).map(([key, value]) => [key, String(unwrapScalar(value) ?? '')])); +}; + +const buildHeaders = (bindings = {}, meta = {}, cookie = '') => ({ + ...sanitizeHeaders(bindings.headers), + accept: 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8', + cookie, + 'x-engine-instance': pickFirstString([meta.instance_id, meta.instanceId, 'unknown']), + 'x-request-id': pickFirstString([meta.request_id, meta.requestId, 'unknown']), +}); + +const throwForHttpStatus = (status, text) => { + if (status === 401 || status === 403) throw errorWithCode('PERMISSION_DENIED', `upstream http ${status}: ${text}`); + if (status >= 400 && status < 500) throw errorWithCode('FAILED_PRECONDITION', `upstream http ${status}: ${text}`); + throw errorWithCode('UNAVAILABLE', `upstream http ${status}: ${text}`); +}; + +const requireBindings = (ctx = {}) => { + const callCtx = resolveCallContext(ctx); + const bindings = callCtx.bindings || {}; + const host = resolveHost(bindings); + if (!host) throw errorWithCode('INVALID_ARGUMENT', 'bindings.host is required'); + const cookie = resolveCookie(bindings); + if (!cookie) throw errorWithCode('INVALID_ARGUMENT', 'bindings.cookie (web session cookie) is required'); + return { ...callCtx, bindings, host, cookie }; +}; + +const decodeEntities = (s) => String(s) + .replace(/&/g, '&') + .replace(/</g, '<') + .replace(/>/g, '>') + .replace(/"/g, '"') + .replace(/�?39;/g, "'") + .replace(/ /g, ' '); + +// 从单个 中按顺序取出带 title 的 文本。 +const rowTitles = (rowHtml) => { + const titles = []; + const tdRe = /]*\btitle="([^"]*)"[^>]*>/gi; + let c; + while ((c = tdRe.exec(rowHtml)) !== null) titles.push(decodeEntities(c[1])); + return titles; +}; + +// 解析 HTML 日志页为结构化条目:数据行需含时间且至少 13 个 title 单元格。 +const parseIpsLog = (html, limit = 0) => { + const entries = []; + const rowRe = /]*>([\s\S]*?)<\/tr>/gi; + let m; + while ((m = rowRe.exec(html)) !== null) { + const titles = rowTitles(m[1]); + if (titles.length < 13) continue; + if (!titles.some((v) => DATETIME_RE.test(v))) continue; + const entry = {}; + ENTRY_FIELDS.forEach((key, i) => { entry[key] = titles[i] ?? ''; }); + entries.push(entry); + if (limit > 0 && entries.length >= limit) break; + } + return entries; +}; + +const runQueryIpsLog = async (req = {}, ctx = {}) => { + const bound = requireBindings(ctx); + const request = bound.req ? { ...bound.req, ...req } : req; + const limit = Math.max(0, pickInt(request, ['limit'], 0)); + let response; + try { + response = await fetch(`${bound.host}${IPS_LOG_URI}`, { + method: 'GET', + timeoutMs: resolveTimeoutMs(bound), + ...buildTlsOptions(bound.bindings), + headers: buildHeaders(bound.bindings, bound.meta, bound.cookie), + }); + } catch (err) { + throw errorWithCode('UNAVAILABLE', err?.cause?.message || err?.message || 'fetch failed'); + } + const text = await response.text(); + const status = Number(response.status); + if (!response.ok) throwForHttpStatus(status, text); + // 会话失效时设备会重定向到登录页(同样 200),用日志页标记区分。 + if (!String(text || '').includes(LOG_PAGE_MARKER)) { + throw errorWithCode('FAILED_PRECONDITION', 'unexpected response (session may be expired or not the IPS log page)'); + } + const entries = parseIpsLog(text, limit); + return { http_status: status, total: entries.length, entries }; +}; + +export function rpcdef(ctx = {}) { + const callCtx = resolveCallContext(ctx); + return { + [QUERY_IPS_LOG_PATH]: async (req) => runQueryIpsLog(req ?? callCtx.req, callCtx), + }; +} + +export const handlers = { + [METHOD_QUERY_IPS_LOG_FULL]: (req, ctx = {}) => runQueryIpsLog(req, ctx), +}; + +export const _test = { + buildHeaders, + buildTlsOptions, + decodeEntities, + errorWithCode, + grpcCodeFor, + hasOwn, + normalizeBaseUrl, + parseIpsLog, + pickBoolean, + pickFirstBoolean, + pickFirstString, + pickInt, + pickStringFrom, + requireBindings, + resolveCallContext, + resolveCookie, + resolveHost, + resolveTimeoutMs, + rowTitles, + sanitizeHeaders, + throwForHttpStatus, + unwrapScalar, +}; diff --git a/services/venus__ips/test/mock_upstream.js b/services/venus__ips/test/mock_upstream.js new file mode 100644 index 00000000..a90e2df8 --- /dev/null +++ b/services/venus__ips/test/mock_upstream.js @@ -0,0 +1,47 @@ +/* node:coverage disable */ +import http from 'node:http'; + +// 合成的 IPS 日志 HTML 页(结构与真机一致: 数据单元格 + ips_log_filter 标记), +// 使用文档保留地址段(198.51.100.x / 203.0.113.x),不含任何真实数据。 +const FIELDS_ROWS = [ + ['TCP_可疑行为_安全风险_MYSQL_查询系统变量', '198.51.100.10', '60782', '203.0.113.5', '3883', 'TCP', '2026-06-25 17:49:45', '可疑行为', '中', '警示', 'PASS', '1', '3', ''], + ['UDP_扫描_端口扫描', '198.51.100.11', '53', '203.0.113.6', '161', 'UDP', '2026-06-25 17:10:44', '扫描', '高', '严重', 'DROP', '2', '1', '备注X'], +]; + +const dataRow = (cells) => + `#${cells.map((v) => `${v}`).join('')}操作`; + +const buildLogHtml = () => ` +
+ + + ${FIELDS_ROWS.map(dataRow).join('\n ')} +
#名称源IP源端口目的IP目的端口协议类型时间类型事件级别优先级动作策略ID次数内容操作
`; + +// 登录页(无 ips_log_filter 标记),用于模拟会话失效后的 200 重定向。 +const LOGIN_HTML = '
'; + +export const createMockServer = async ({ cookie = 'PHPSESSID=abc123' } = {}) => { + const state = { requests: [] }; + + const server = http.createServer((req, res) => { + state.requests.push({ url: req.url, method: req.method, cookie: req.headers.cookie }); + if (req.url !== '/log/memorylog/ipslog.php' || req.method !== 'GET') { + res.writeHead(404, { 'content-type': 'text/html' }); res.end('not found'); return; + } + // 未带正确 cookie -> 返回登录页(会话失效) + const body = req.headers.cookie === cookie ? buildLogHtml() : LOGIN_HTML; + res.writeHead(200, { 'content-type': 'text/html; charset=utf-8' }); + res.end(body); + }); + + await new Promise((resolve) => server.listen(0, '127.0.0.1', resolve)); + const { port } = server.address(); + return { + state, + host: `http://127.0.0.1:${port}`, + cookie, + rowCount: FIELDS_ROWS.length, + async close() { await new Promise((resolve) => server.close(resolve)); }, + }; +}; diff --git a/services/venus__ips/test/venus-ips.test.js b/services/venus__ips/test/venus-ips.test.js new file mode 100644 index 00000000..518d6778 --- /dev/null +++ b/services/venus__ips/test/venus-ips.test.js @@ -0,0 +1,194 @@ +import assert from 'node:assert/strict'; +import test from 'node:test'; + +import { GrpcError, grpcStatus } from '@chaitin-ai/octobus-sdk'; + +import { + QUERY_IPS_LOG_PATH, + METHOD_QUERY_IPS_LOG_FULL, + IPS_LOG_URI, + _test, + handlers, + rpcdef, +} from '../src/venus-ips.js'; +import { service } from '../src/service.js'; +import { createMockServer } from './mock_upstream.js'; + +const originalFetch = globalThis.fetch; +let seq = 0; +const nextId = () => `inst-${++seq}`; + +const buildCtx = (mock, overrides = {}) => ({ + bindings: { host: mock?.host, cookie: mock?.cookie, ...(overrides.bindings || {}) }, + config: overrides.config || {}, + secret: overrides.secret || {}, + limits: { timeoutMs: 10_000, ...(overrides.limits || {}) }, + meta: { instance_id: nextId(), request_id: 'req', ...(overrides.meta || {}) }, + req: overrides.req || {}, +}); + +const createHeaders = (entries = {}) => { + const map = new Map(); + for (const [k, v] of Object.entries(entries)) map.set(String(k).toLowerCase(), Array.isArray(v) ? v.map(String) : [String(v)]); + return { get(n) { const x = map.get(String(n).toLowerCase()); return x?.length ? x.join(', ') : null; } }; +}; +const fakeResponse = (status, body, ok = status >= 200 && status < 300) => ({ status, ok, headers: createHeaders(), text: async () => body }); +const withFetch = (impl) => { globalThis.fetch = impl; }; + +test.afterEach(() => { globalThis.fetch = originalFetch; }); + +// ---------- end-to-end against mock ---------- + +test('parses IPS log HTML into structured entries', async () => { + const mock = await createMockServer(); + try { + const out = await rpcdef(buildCtx(mock))[QUERY_IPS_LOG_PATH]({}); + assert.equal(out.http_status, 200); + assert.equal(out.total, mock.rowCount); + const first = out.entries[0]; + assert.equal(first.name, 'TCP_可疑行为_安全风险_MYSQL_查询系统变量'); + assert.equal(first.src_ip, '198.51.100.10'); + assert.equal(first.src_port, '60782'); + assert.equal(first.dst_ip, '203.0.113.5'); + assert.equal(first.protocol, 'TCP'); + assert.equal(first.time, '2026-06-25 17:49:45'); + assert.equal(first.severity, '中'); + assert.equal(first.action, 'PASS'); + assert.equal(first.count, '3'); + assert.equal(out.entries[1].content, '备注X'); + // request shape + const r = mock.state.requests[0]; + assert.equal(r.method, 'GET'); + assert.equal(r.url, IPS_LOG_URI); + assert.equal(r.cookie, mock.cookie); + } finally { + await mock.close(); + } +}); + +test('limit caps the number of returned entries', async () => { + const mock = await createMockServer(); + try { + const out = await handlers[METHOD_QUERY_IPS_LOG_FULL]({ limit: 1 }, buildCtx(mock)); + assert.equal(out.total, 1); + assert.equal(out.entries.length, 1); + } finally { + await mock.close(); + } +}); + +test('expired session (login page, no marker) -> FAILED_PRECONDITION', async () => { + const mock = await createMockServer(); + try { + await assert.rejects( + () => handlers[METHOD_QUERY_IPS_LOG_FULL]({}, buildCtx(mock, { bindings: { cookie: 'PHPSESSID=wrong' } })), + (e) => e.legacyCode === 'FAILED_PRECONDITION', + ); + } finally { + await mock.close(); + } +}); + +// ---------- validation ---------- + +test('binding validation', async () => { + await assert.rejects(() => handlers[METHOD_QUERY_IPS_LOG_FULL]({}, buildCtx({ host: '' })), (e) => e.legacyCode === 'INVALID_ARGUMENT'); + await assert.rejects(() => handlers[METHOD_QUERY_IPS_LOG_FULL]({}, buildCtx({ host: 'https://h', cookie: '' })), (e) => e.legacyCode === 'INVALID_ARGUMENT'); +}); + +// ---------- error mapping ---------- + +test('error mapping: network / http', async () => { + const ctx = buildCtx({ host: 'https://ips', cookie: 'c=1' }); + withFetch(async () => { throw new Error('ECONNREFUSED'); }); + await assert.rejects(() => handlers[METHOD_QUERY_IPS_LOG_FULL]({}, ctx), (e) => e.legacyCode === 'UNAVAILABLE'); + withFetch(async () => fakeResponse(401, 'no', false)); + await assert.rejects(() => handlers[METHOD_QUERY_IPS_LOG_FULL]({}, ctx), (e) => e.legacyCode === 'PERMISSION_DENIED'); + withFetch(async () => fakeResponse(404, 'no', false)); + await assert.rejects(() => handlers[METHOD_QUERY_IPS_LOG_FULL]({}, ctx), (e) => e.legacyCode === 'FAILED_PRECONDITION'); + withFetch(async () => fakeResponse(500, 'no', false)); + await assert.rejects(() => handlers[METHOD_QUERY_IPS_LOG_FULL]({}, ctx), (e) => e.legacyCode === 'UNAVAILABLE'); +}); + +test('fetch error fallback message', async () => { + const ctx = buildCtx({ host: 'https://ips', cookie: 'c=1' }); + withFetch(async () => { throw {}; }); + await assert.rejects(() => handlers[METHOD_QUERY_IPS_LOG_FULL]({}, ctx), (e) => /fetch failed/.test(e.message)); + withFetch(async () => { const e = new Error('m'); e.cause = { message: 'deep' }; throw e; }); + await assert.rejects(() => handlers[METHOD_QUERY_IPS_LOG_FULL]({}, ctx), (e) => /deep/.test(e.message)); +}); + +test('valid log page with zero data rows returns empty entries', async () => { + const ctx = buildCtx({ host: 'https://ips', cookie: 'c=1' }); + withFetch(async () => fakeResponse(200, '
名称
')); + const out = await handlers[METHOD_QUERY_IPS_LOG_FULL]({}, ctx); + assert.equal(out.total, 0); + assert.deepEqual(out.entries, []); +}); + +// ---------- service surface + helpers ---------- + +test('service exposes the QueryIpsLog handler', () => { + assert.equal(typeof service.handlers[METHOD_QUERY_IPS_LOG_FULL], 'function'); +}); + +test('helper coverage', () => { + const h = _test; + assert.equal(h.normalizeBaseUrl('https://h/'), 'https://h'); + assert.equal(h.normalizeBaseUrl('ftp://x'), ''); + assert.equal(h.resolveCookie({ session_cookie: 'c' }), 'c'); + assert.equal(h.resolveCookie({ sessionCookie: 'c2' }), 'c2'); + assert.equal(h.decodeEntities('a&b<c>"' d'), 'a&b"\' d'); + assert.equal(h.pickBoolean(true), true); + assert.equal(h.pickBoolean(0), false); + assert.equal(h.pickBoolean(undefined), undefined); + + // rowTitles + parseIpsLog + const row = '#名称X1.1.1.1'; + assert.deepEqual(h.rowTitles(row), ['名称X', '1.1.1.1']); + const html = 'h' + + '#' + Array.from({ length: 14 }, (_, i) => `v${i}`).join('').replace('v6', '2026-01-02 03:04:05') + ''; + const parsed = h.parseIpsLog(html); + assert.equal(parsed.length, 1); + assert.equal(parsed[0].name, 'v0'); + assert.equal(parsed[0].time, '2026-01-02 03:04:05'); + // a row without a datetime is skipped + assert.equal(h.parseIpsLog('' + Array.from({ length: 14 }, (_, i) => `x`).join('') + '').length, 0); + // limit + const two = '' + Array.from({ length: 14 }, (_, i) => `a`).join('') + ''; + assert.equal(h.parseIpsLog(two + two, 1).length, 1); + + assert.equal(h.pickInt({ a: '5' }, ['a'], 0), 5); + assert.equal(h.pickInt({ a: '' }, ['a'], 9), 9); + assert.equal(h.pickFirstString([null, '', 'y']), 'y'); + assert.equal(h.pickBoolean('off'), false); + assert.equal(h.pickBoolean('maybe'), undefined); + assert.equal(h.pickFirstBoolean(['x', 'true']), true); + assert.equal(h.unwrapScalar({ value: { value: 2 } }), 2); + assert.deepEqual(h.sanitizeHeaders({ A: 1, '': 2 }), { A: '1' }); + assert.deepEqual(h.sanitizeHeaders('x'), {}); + assert.equal(h.buildTlsOptions({ skipTlsVerify: true }).skipTlsVerify, true); + assert.deepEqual(h.buildTlsOptions({}), {}); + assert.equal(h.resolveTimeoutMs({ limits: { timeoutMs: 0 } }), 5000); + assert.equal(h.resolveTimeoutMs({ limits: { timeoutMs: 321 } }), 321); + assert.equal(h.grpcCodeFor('NOPE'), grpcStatus.UNKNOWN); + assert.ok(h.errorWithCode('UNAVAILABLE', 'x') instanceof GrpcError); + assert.throws(() => h.throwForHttpStatus(403, 'x'), (e) => e.legacyCode === 'PERMISSION_DENIED'); + assert.throws(() => h.throwForHttpStatus(400, 'x'), (e) => e.legacyCode === 'FAILED_PRECONDITION'); + assert.throws(() => h.throwForHttpStatus(500, 'x'), (e) => e.legacyCode === 'UNAVAILABLE'); + const hdr = h.buildHeaders({ headers: { 'X-A': '1' } }, { instance_id: 'i', request_id: 'r' }, 'c=1'); + assert.equal(hdr.cookie, 'c=1'); + assert.equal(hdr['X-A'], '1'); + assert.deepEqual(h.resolveCallContext({ request: { a: 1 } }).req, { a: 1 }); + assert.deepEqual(h.resolveCallContext({}).req, {}); +}); + +test('rpcdef falls back to ctx.req when called without an argument', async () => { + const mock = await createMockServer(); + try { + const out = await rpcdef(buildCtx(mock, { req: { limit: 1 } }))[QUERY_IPS_LOG_PATH](); + assert.equal(out.total, 1); + } finally { + await mock.close(); + } +}); From da4ecbfbea08e1675797f70a4b46c982941e94da Mon Sep 17 00:00:00 2001 From: kingfs Date: Mon, 17 Aug 2026 10:27:09 +0800 Subject: [PATCH 2/7] Harden Venus IPS service for L2 --- services/bin/venus-ips.js | 0 services/venus__ips/README.md | 6 +- services/venus__ips/bin/venus-ips.js | 0 services/venus__ips/config.schema.json | 3 +- services/venus__ips/package.json | 2 +- services/venus__ips/proto/venus_ips.proto | 9 ++ services/venus__ips/secret.schema.json | 1 + services/venus__ips/service.json | 4 + services/venus__ips/src/venus-ips.js | 132 +++++++++++++++++++-- services/venus__ips/test/smoke.json | 12 ++ services/venus__ips/test/venus-ips.test.js | 106 ++++++++++++++--- 11 files changed, 240 insertions(+), 35 deletions(-) mode change 100644 => 100755 services/bin/venus-ips.js mode change 100644 => 100755 services/venus__ips/bin/venus-ips.js create mode 100644 services/venus__ips/test/smoke.json diff --git a/services/bin/venus-ips.js b/services/bin/venus-ips.js old mode 100644 new mode 100755 diff --git a/services/venus__ips/README.md b/services/venus__ips/README.md index 15e2c3b9..8e4c7315 100644 --- a/services/venus__ips/README.md +++ b/services/venus__ips/README.md @@ -19,7 +19,7 @@ cookie 经 `secret.cookie` 外部传入。 ```json // config -{ "host": "https://192.168.1.10", "timeoutMs": 5000, "skipTlsVerify": true } +{ "host": "https://192.168.1.10", "timeoutMs": 5000, "maxResponseBytes": 2097152, "skipTlsVerify": true } // secret { "cookie": "PHPSESSID=<会话id>" } ``` @@ -28,6 +28,7 @@ cookie 经 `secret.cookie` 外部传入。 | RPC | 上游接口 | | --- | --- | +| `ProbeConnectivity` | `GET /log/memorylog/ipslog.php`（仅报告 HTTP 可达性，不读取正文） | | `QueryIpsLog` | `GET /log/memorylog/ipslog.php` | ### 请求 / 响应 @@ -45,6 +46,7 @@ cookie 经 `secret.cookie` 外部传入。 - 本方法为**只读查询**,无写操作,风险面低。 - 会话 cookie 等同登录态,泄露即会话失陷;仅放 `secret`,勿写入 `config`、日志或截图。 - 默认校验 TLS;私有自签部署需 `skipTlsVerify: true`。 +- 禁止跨站重定向，响应体默认最多 2 MiB；上游正文、网络异常详情和 cookie 不写入错误信息。 ## 错误映射 @@ -69,4 +71,4 @@ npm run pack:check ``` 真机验证:用一个有效会话 cookie 调 `query-ips-log`,确认返回 IPS 告警条目(名称/源IP/目的IP/时间/级别/动作 等)。 -PR 附**真机验证截图**(cookie / host / 响应中的内网 IP 等敏感数据已打码)。**代码/测试/截图里不得出现真实 cookie、内网地址或业务数据。** +原作者说明实现按设备抓包对齐，但本 PR 当前没有可独立审计的脱敏真机截图；合并前仍需维护者核验实际设备兼容性。**代码/测试/截图里不得出现真实 cookie、内网地址或业务数据。** diff --git a/services/venus__ips/bin/venus-ips.js b/services/venus__ips/bin/venus-ips.js old mode 100644 new mode 100755 diff --git a/services/venus__ips/config.schema.json b/services/venus__ips/config.schema.json index a7cccd60..a911f21e 100644 --- a/services/venus__ips/config.schema.json +++ b/services/venus__ips/config.schema.json @@ -7,9 +7,10 @@ "restBaseUrl": { "type": "string", "description": "Alias for host." }, "baseUrl": { "type": "string", "description": "Alias for host." }, "timeoutMs": { "type": "integer", "minimum": 1, "default": 5000, "description": "HTTP timeout in milliseconds." }, + "maxResponseBytes": { "type": "integer", "minimum": 1024, "maximum": 8388608, "default": 2097152, "description": "Maximum upstream response size in bytes." }, "skipTlsVerify": { "type": "boolean", "default": false, "description": "Skip TLS certificate verification for private deployments." }, "tlsInsecureSkipVerify": { "type": "boolean", "default": false, "description": "Legacy alias for skipTlsVerify." }, "insecureSkipVerify": { "type": "boolean", "default": false, "description": "Alias for skipTlsVerify." }, - "headers": { "type": "object", "additionalProperties": { "type": "string" }, "description": "Optional additional HTTP headers." } + "headers": { "type": "object", "additionalProperties": { "type": "string" }, "description": "Optional non-sensitive HTTP headers. Authentication and hop-by-hop headers are ignored." } } } diff --git a/services/venus__ips/package.json b/services/venus__ips/package.json index 4bc8fa38..e77c3b2d 100644 --- a/services/venus__ips/package.json +++ b/services/venus__ips/package.json @@ -7,6 +7,6 @@ "venus-ips": "bin/venus-ips.js" }, "dependencies": { - "@chaitin-ai/octobus-sdk": "^0.5.0" + "@chaitin-ai/octobus-sdk": "^0.6.0" } } diff --git a/services/venus__ips/proto/venus_ips.proto b/services/venus__ips/proto/venus_ips.proto index 8a6a083f..9f201473 100644 --- a/services/venus__ips/proto/venus_ips.proto +++ b/services/venus__ips/proto/venus_ips.proto @@ -8,10 +8,19 @@ option go_package = "miner/grpc-service/VENUS_IPS"; // 认证为 web 会话 Cookie:GET /log/memorylog/ipslog.php，浏览器 credentials=include。 // 设备返回 HTML 日志页,本 service 解析其中的日志表行,映射为结构化条目。 service VENUS_IPS { + // 探测控制台 HTTP 可达性；不读取或返回响应正文，也不代表会话一定有效。 + rpc ProbeConnectivity(ProbeConnectivityRequest) returns (ProbeConnectivityResponse) {} // 查询 IPS 攻击日志(内存日志): GET /log/memorylog/ipslog.php rpc QueryIpsLog(QueryIpsLogRequest) returns (QueryIpsLogResponse) {} } +message ProbeConnectivityRequest {} + +message ProbeConnectivityResponse { + bool reachable = 1; + int32 http_status = 2; +} + message QueryIpsLogRequest { int32 limit = 1; // 返回条目上限(客户端侧截断)，<=0 表示全部 } diff --git a/services/venus__ips/secret.schema.json b/services/venus__ips/secret.schema.json index c7dd01c4..c344c96b 100644 --- a/services/venus__ips/secret.schema.json +++ b/services/venus__ips/secret.schema.json @@ -2,6 +2,7 @@ "$schema": "https://json-schema.org/draft/2020-12/schema", "type": "object", "additionalProperties": true, + "required": ["cookie"], "properties": { "cookie": { "type": "string", "description": "Venus IPS web session cookie header value (e.g. 'PHPSESSID=...'). Time-limited; refresh when expired." }, "sessionCookie": { "type": "string", "description": "Alias for cookie." }, diff --git a/services/venus__ips/service.json b/services/venus__ips/service.json index 323ea322..fd56a072 100644 --- a/services/venus__ips/service.json +++ b/services/venus__ips/service.json @@ -10,6 +10,10 @@ "sdk": { "cli": { "commands": { + "VENUS_IPS.VENUS_IPS/ProbeConnectivity": { + "name": "probe-connectivity", + "description": "Probe Venus IPS console HTTP connectivity without returning response content." + }, "VENUS_IPS.VENUS_IPS/QueryIpsLog": { "name": "query-ips-log", "description": "Query Venus IPS attack logs (memory log)." diff --git a/services/venus__ips/src/venus-ips.js b/services/venus__ips/src/venus-ips.js index f91cf9e8..19d100fd 100644 --- a/services/venus__ips/src/venus-ips.js +++ b/services/venus__ips/src/venus-ips.js @@ -1,3 +1,5 @@ +import { Buffer } from 'node:buffer'; + // 启明星辰 IPS 攻击日志查询适配。 // 认证:web 会话 Cookie。GET /log/memorylog/ipslog.php 返回 HTML 日志页，解析表行为结构化条目。 import { GrpcError, grpcStatus } from '@chaitin-ai/octobus-sdk'; @@ -5,10 +7,19 @@ import { GrpcError, grpcStatus } from '@chaitin-ai/octobus-sdk'; const SVC = 'VENUS_IPS.VENUS_IPS'; export const QUERY_IPS_LOG_PATH = `/${SVC}/QueryIpsLog`; export const METHOD_QUERY_IPS_LOG_FULL = `${SVC}/QueryIpsLog`; +export const PROBE_CONNECTIVITY_PATH = `/${SVC}/ProbeConnectivity`; +export const METHOD_PROBE_CONNECTIVITY_FULL = `${SVC}/ProbeConnectivity`; export const IPS_LOG_URI = '/log/memorylog/ipslog.php'; export const LOG_PAGE_MARKER = 'ips_log_filter'; export const DEFAULT_TIMEOUT_MS = 5000; +export const DEFAULT_MAX_RESPONSE_BYTES = 2 * 1024 * 1024; +export const MAX_LIMIT = 10_000; + +const BLOCKED_HEADERS = new Set([ + 'authorization', 'cookie', 'host', 'connection', 'content-length', 'proxy-authorization', + 'transfer-encoding', 'upgrade', 'x-engine-instance', 'x-request-id', +]); // 日志表 14 个有 title 的数据单元格，按列顺序映射。 const ENTRY_FIELDS = [ @@ -21,6 +32,7 @@ const grpcCodeFor = (code) => ({ FAILED_PRECONDITION: grpcStatus.FAILED_PRECONDITION, INVALID_ARGUMENT: grpcStatus.INVALID_ARGUMENT, PERMISSION_DENIED: grpcStatus.PERMISSION_DENIED, + RESOURCE_EXHAUSTED: grpcStatus.RESOURCE_EXHAUSTED, UNAVAILABLE: grpcStatus.UNAVAILABLE, UNKNOWN: grpcStatus.UNKNOWN, })[code] ?? grpcStatus.UNKNOWN; @@ -94,8 +106,15 @@ const pickFirstBoolean = (values = []) => { const normalizeBaseUrl = (value) => { const raw = String(unwrapScalar(value) || '').trim(); - if (!/^https?:\/\//i.test(raw)) return ''; - return raw.replace(/\/+$/, ''); + try { + const url = new URL(raw); + if (!['http:', 'https:'].includes(url.protocol) + || url.username || url.password || url.search || url.hash + || (url.pathname && url.pathname !== '/')) return ''; + return url.origin; + } catch { + return ''; + } }; const resolveCallContext = (ctx = {}) => ({ @@ -110,7 +129,13 @@ const resolveCallContext = (ctx = {}) => ({ req: ctx.req ?? ctx.request ?? {}, }); -const resolveHost = (bindings = {}) => normalizeBaseUrl(pickFirstString([bindings.host, bindings.restBaseUrl, bindings.baseUrl])); +const resolveHost = (bindings = {}) => { + for (const candidate of [bindings.host, bindings.restBaseUrl, bindings.baseUrl]) { + const normalized = normalizeBaseUrl(candidate); + if (normalized) return normalized; + } + return ''; +}; const resolveCookie = (bindings = {}) => pickStringFrom(bindings, ['cookie', 'sessionCookie', 'session_cookie']); const resolveTimeoutMs = (ctx = {}) => { @@ -118,6 +143,12 @@ const resolveTimeoutMs = (ctx = {}) => { return Number.isFinite(raw) && raw > 0 ? raw : DEFAULT_TIMEOUT_MS; }; +const resolveMaxResponseBytes = (ctx = {}) => { + const raw = Number(unwrapScalar(ctx.bindings?.maxResponseBytes ?? DEFAULT_MAX_RESPONSE_BYTES)); + return Number.isFinite(raw) && raw >= 1024 && raw <= 8 * 1024 * 1024 + ? Math.trunc(raw) : DEFAULT_MAX_RESPONSE_BYTES; +}; + const buildTlsOptions = (bindings = {}) => { const enabled = pickFirstBoolean([bindings.skipTlsVerify, bindings.tlsInsecureSkipVerify, bindings.insecureSkipVerify]) || false; return enabled ? { skipTlsVerify: true, tlsInsecureSkipVerify: true, insecureSkipVerify: true } : {}; @@ -126,7 +157,14 @@ const buildTlsOptions = (bindings = {}) => { const sanitizeHeaders = (headers) => { const raw = unwrapScalar(headers); if (!raw || typeof raw !== 'object' || Array.isArray(raw)) return {}; - return Object.fromEntries(Object.entries(raw).filter(([key]) => key).map(([key, value]) => [key, String(unwrapScalar(value) ?? '')])); + const result = {}; + for (const [key, value] of Object.entries(raw)) { + const normalized = key.trim().toLowerCase(); + const text = String(unwrapScalar(value) ?? ''); + if (!normalized || BLOCKED_HEADERS.has(normalized) || /[\r\n]/.test(key) || /[\r\n]/.test(text)) continue; + result[key] = text; + } + return result; }; const buildHeaders = (bindings = {}, meta = {}, cookie = '') => ({ @@ -137,10 +175,43 @@ const buildHeaders = (bindings = {}, meta = {}, cookie = '') => ({ 'x-request-id': pickFirstString([meta.request_id, meta.requestId, 'unknown']), }); -const throwForHttpStatus = (status, text) => { - if (status === 401 || status === 403) throw errorWithCode('PERMISSION_DENIED', `upstream http ${status}: ${text}`); - if (status >= 400 && status < 500) throw errorWithCode('FAILED_PRECONDITION', `upstream http ${status}: ${text}`); - throw errorWithCode('UNAVAILABLE', `upstream http ${status}: ${text}`); +const throwForHttpStatus = (status) => { + if (status === 401 || status === 403) throw errorWithCode('PERMISSION_DENIED', `upstream rejected authentication (HTTP ${status})`); + if (status >= 300 && status < 400) throw errorWithCode('FAILED_PRECONDITION', 'upstream redirect refused (session may be expired)'); + if (status >= 400 && status < 500) throw errorWithCode('FAILED_PRECONDITION', `upstream rejected request (HTTP ${status})`); + throw errorWithCode('UNAVAILABLE', `upstream unavailable (HTTP ${status})`); +}; + +const readBoundedText = async (response, maxBytes) => { + const declared = Number(response.headers?.get?.('content-length')); + if (Number.isFinite(declared) && declared > maxBytes) { + throw errorWithCode('RESOURCE_EXHAUSTED', 'upstream response exceeds configured limit'); + } + if (response.body?.getReader) { + const reader = response.body.getReader(); + const chunks = []; + let size = 0; + try { + while (true) { + const { done, value } = await reader.read(); + if (done) break; + size += value.byteLength; + if (size > maxBytes) { + await reader.cancel?.(); + throw errorWithCode('RESOURCE_EXHAUSTED', 'upstream response exceeds configured limit'); + } + chunks.push(Buffer.from(value)); + } + } finally { + reader.releaseLock?.(); + } + return Buffer.concat(chunks).toString('utf8'); + } + const text = await response.text(); + if (Buffer.byteLength(String(text), 'utf8') > maxBytes) { + throw errorWithCode('RESOURCE_EXHAUSTED', 'upstream response exceeds configured limit'); + } + return String(text); }; const requireBindings = (ctx = {}) => { @@ -150,6 +221,7 @@ const requireBindings = (ctx = {}) => { if (!host) throw errorWithCode('INVALID_ARGUMENT', 'bindings.host is required'); const cookie = resolveCookie(bindings); if (!cookie) throw errorWithCode('INVALID_ARGUMENT', 'bindings.cookie (web session cookie) is required'); + if (cookie.length > 8192 || /[\r\n]/.test(cookie)) throw errorWithCode('INVALID_ARGUMENT', 'bindings.cookie is invalid'); return { ...callCtx, bindings, host, cookie }; }; @@ -190,21 +262,31 @@ const parseIpsLog = (html, limit = 0) => { const runQueryIpsLog = async (req = {}, ctx = {}) => { const bound = requireBindings(ctx); const request = bound.req ? { ...bound.req, ...req } : req; - const limit = Math.max(0, pickInt(request, ['limit'], 0)); + const rawLimit = pickInt(request, ['limit'], 0); + if (rawLimit < 0 || rawLimit > MAX_LIMIT) throw errorWithCode('INVALID_ARGUMENT', `limit must be between 0 and ${MAX_LIMIT}`); + const limit = rawLimit; let response; try { response = await fetch(`${bound.host}${IPS_LOG_URI}`, { method: 'GET', + redirect: 'manual', timeoutMs: resolveTimeoutMs(bound), ...buildTlsOptions(bound.bindings), headers: buildHeaders(bound.bindings, bound.meta, bound.cookie), }); } catch (err) { - throw errorWithCode('UNAVAILABLE', err?.cause?.message || err?.message || 'fetch failed'); + if (err instanceof GrpcError) throw err; + throw errorWithCode('UNAVAILABLE', 'upstream request failed'); } - const text = await response.text(); const status = Number(response.status); - if (!response.ok) throwForHttpStatus(status, text); + if (!response.ok) throwForHttpStatus(status); + let text; + try { + text = await readBoundedText(response, resolveMaxResponseBytes(bound)); + } catch (err) { + if (err instanceof GrpcError) throw err; + throw errorWithCode('UNAVAILABLE', 'failed to read upstream response'); + } // 会话失效时设备会重定向到登录页(同样 200),用日志页标记区分。 if (!String(text || '').includes(LOG_PAGE_MARKER)) { throw errorWithCode('FAILED_PRECONDITION', 'unexpected response (session may be expired or not the IPS log page)'); @@ -213,15 +295,37 @@ const runQueryIpsLog = async (req = {}, ctx = {}) => { return { http_status: status, total: entries.length, entries }; }; +const runProbeConnectivity = async (ctx = {}) => { + const bound = requireBindings(ctx); + let response; + try { + response = await fetch(`${bound.host}${IPS_LOG_URI}`, { + method: 'GET', + redirect: 'manual', + timeoutMs: resolveTimeoutMs(bound), + ...buildTlsOptions(bound.bindings), + headers: buildHeaders(bound.bindings, bound.meta, bound.cookie), + }); + } catch { + throw errorWithCode('UNAVAILABLE', 'upstream request failed'); + } + const status = Number(response.status); + if (!response.ok) throwForHttpStatus(status); + response.body?.cancel?.().catch?.(() => {}); + return { reachable: true, http_status: status }; +}; + export function rpcdef(ctx = {}) { const callCtx = resolveCallContext(ctx); return { + [PROBE_CONNECTIVITY_PATH]: async () => runProbeConnectivity(callCtx), [QUERY_IPS_LOG_PATH]: async (req) => runQueryIpsLog(req ?? callCtx.req, callCtx), }; } export const handlers = { - [METHOD_QUERY_IPS_LOG_FULL]: (req, ctx = {}) => runQueryIpsLog(req, ctx), + [METHOD_PROBE_CONNECTIVITY_FULL]: (ctx = {}) => runProbeConnectivity(ctx), + [METHOD_QUERY_IPS_LOG_FULL]: (ctx = {}) => runQueryIpsLog(ctx.request ?? ctx.req ?? {}, ctx), }; export const _test = { @@ -242,8 +346,10 @@ export const _test = { resolveCallContext, resolveCookie, resolveHost, + resolveMaxResponseBytes, resolveTimeoutMs, rowTitles, + readBoundedText, sanitizeHeaders, throwForHttpStatus, unwrapScalar, diff --git a/services/venus__ips/test/smoke.json b/services/venus__ips/test/smoke.json new file mode 100644 index 00000000..a6f83fc7 --- /dev/null +++ b/services/venus__ips/test/smoke.json @@ -0,0 +1,12 @@ +{ + "method": "VENUS_IPS.VENUS_IPS/ProbeConnectivity", + "request": {}, + "expectUpstream": true, + "requireBusinessSuccess": true, + "requireUpstreamPerProtocol": true, + "protocols": ["connect", "grpc", "mcp"], + "upstream": { + "method": "GET", + "path": "/log/memorylog/ipslog.php" + } +} diff --git a/services/venus__ips/test/venus-ips.test.js b/services/venus__ips/test/venus-ips.test.js index 518d6778..9035b189 100644 --- a/services/venus__ips/test/venus-ips.test.js +++ b/services/venus__ips/test/venus-ips.test.js @@ -5,6 +5,8 @@ import { GrpcError, grpcStatus } from '@chaitin-ai/octobus-sdk'; import { QUERY_IPS_LOG_PATH, + PROBE_CONNECTIVITY_PATH, + METHOD_PROBE_CONNECTIVITY_FULL, METHOD_QUERY_IPS_LOG_FULL, IPS_LOG_URI, _test, @@ -34,6 +36,7 @@ const createHeaders = (entries = {}) => { }; const fakeResponse = (status, body, ok = status >= 200 && status < 300) => ({ status, ok, headers: createHeaders(), text: async () => body }); const withFetch = (impl) => { globalThis.fetch = impl; }; +const invoke = (request, ctx) => handlers[METHOD_QUERY_IPS_LOG_FULL]({ ...ctx, request }); test.afterEach(() => { globalThis.fetch = originalFetch; }); @@ -66,10 +69,38 @@ test('parses IPS log HTML into structured entries', async () => { } }); +test('connectivity probe uses the same hardened upstream request', async () => { + const mock = await createMockServer(); + try { + const ctx = buildCtx(mock); + const out = await handlers[METHOD_PROBE_CONNECTIVITY_FULL](ctx); + assert.deepEqual(out, { reachable: true, http_status: 200 }); + assert.equal(mock.state.requests[0].cookie, mock.cookie); + const viaRpcdef = await rpcdef(ctx)[PROBE_CONNECTIVITY_PATH](); + assert.equal(viaRpcdef.reachable, true); + } finally { + await mock.close(); + } +}); + +test('connectivity probe maps network and HTTP failures without leaking details', async () => { + const ctx = buildCtx({ host: 'https://ips', cookie: 'c=1' }); + withFetch(async () => { throw new Error('secret network detail'); }); + await assert.rejects( + () => handlers[METHOD_PROBE_CONNECTIVITY_FULL](ctx), + (e) => e.legacyCode === 'UNAVAILABLE' && !e.message.includes('secret'), + ); + withFetch(async () => fakeResponse(401, 'secret response', false)); + await assert.rejects( + () => handlers[METHOD_PROBE_CONNECTIVITY_FULL](ctx), + (e) => e.legacyCode === 'PERMISSION_DENIED' && !e.message.includes('secret'), + ); +}); + test('limit caps the number of returned entries', async () => { const mock = await createMockServer(); try { - const out = await handlers[METHOD_QUERY_IPS_LOG_FULL]({ limit: 1 }, buildCtx(mock)); + const out = await invoke({ limit: 1 }, buildCtx(mock)); assert.equal(out.total, 1); assert.equal(out.entries.length, 1); } finally { @@ -81,7 +112,7 @@ test('expired session (login page, no marker) -> FAILED_PRECONDITION', async () const mock = await createMockServer(); try { await assert.rejects( - () => handlers[METHOD_QUERY_IPS_LOG_FULL]({}, buildCtx(mock, { bindings: { cookie: 'PHPSESSID=wrong' } })), + () => invoke({}, buildCtx(mock, { bindings: { cookie: 'PHPSESSID=wrong' } })), (e) => e.legacyCode === 'FAILED_PRECONDITION', ); } finally { @@ -92,8 +123,11 @@ test('expired session (login page, no marker) -> FAILED_PRECONDITION', async () // ---------- validation ---------- test('binding validation', async () => { - await assert.rejects(() => handlers[METHOD_QUERY_IPS_LOG_FULL]({}, buildCtx({ host: '' })), (e) => e.legacyCode === 'INVALID_ARGUMENT'); - await assert.rejects(() => handlers[METHOD_QUERY_IPS_LOG_FULL]({}, buildCtx({ host: 'https://h', cookie: '' })), (e) => e.legacyCode === 'INVALID_ARGUMENT'); + await assert.rejects(() => invoke({}, buildCtx({ host: '' })), (e) => e.legacyCode === 'INVALID_ARGUMENT'); + await assert.rejects(() => invoke({}, buildCtx({ host: 'https://h', cookie: '' })), (e) => e.legacyCode === 'INVALID_ARGUMENT'); + await assert.rejects(() => invoke({ limit: -1 }, buildCtx({ host: 'https://h', cookie: 'c' })), (e) => e.legacyCode === 'INVALID_ARGUMENT'); + await assert.rejects(() => invoke({ limit: 10_001 }, buildCtx({ host: 'https://h', cookie: 'c' })), (e) => e.legacyCode === 'INVALID_ARGUMENT'); + await assert.rejects(() => invoke({}, buildCtx({ host: 'https://h', cookie: 'bad\r\nx: y' })), (e) => e.legacyCode === 'INVALID_ARGUMENT'); }); // ---------- error mapping ---------- @@ -101,41 +135,75 @@ test('binding validation', async () => { test('error mapping: network / http', async () => { const ctx = buildCtx({ host: 'https://ips', cookie: 'c=1' }); withFetch(async () => { throw new Error('ECONNREFUSED'); }); - await assert.rejects(() => handlers[METHOD_QUERY_IPS_LOG_FULL]({}, ctx), (e) => e.legacyCode === 'UNAVAILABLE'); + await assert.rejects(() => invoke({}, ctx), (e) => e.legacyCode === 'UNAVAILABLE' && !e.message.includes('ECONNREFUSED')); withFetch(async () => fakeResponse(401, 'no', false)); - await assert.rejects(() => handlers[METHOD_QUERY_IPS_LOG_FULL]({}, ctx), (e) => e.legacyCode === 'PERMISSION_DENIED'); + await assert.rejects(() => invoke({}, ctx), (e) => e.legacyCode === 'PERMISSION_DENIED' && !e.message.includes('no')); withFetch(async () => fakeResponse(404, 'no', false)); - await assert.rejects(() => handlers[METHOD_QUERY_IPS_LOG_FULL]({}, ctx), (e) => e.legacyCode === 'FAILED_PRECONDITION'); + await assert.rejects(() => invoke({}, ctx), (e) => e.legacyCode === 'FAILED_PRECONDITION'); withFetch(async () => fakeResponse(500, 'no', false)); - await assert.rejects(() => handlers[METHOD_QUERY_IPS_LOG_FULL]({}, ctx), (e) => e.legacyCode === 'UNAVAILABLE'); + await assert.rejects(() => invoke({}, ctx), (e) => e.legacyCode === 'UNAVAILABLE'); + withFetch(async () => fakeResponse(302, 'location secret', false)); + await assert.rejects(() => invoke({}, ctx), (e) => e.legacyCode === 'FAILED_PRECONDITION' && !e.message.includes('secret')); }); -test('fetch error fallback message', async () => { +test('fetch errors are redacted', async () => { const ctx = buildCtx({ host: 'https://ips', cookie: 'c=1' }); - withFetch(async () => { throw {}; }); - await assert.rejects(() => handlers[METHOD_QUERY_IPS_LOG_FULL]({}, ctx), (e) => /fetch failed/.test(e.message)); - withFetch(async () => { const e = new Error('m'); e.cause = { message: 'deep' }; throw e; }); - await assert.rejects(() => handlers[METHOD_QUERY_IPS_LOG_FULL]({}, ctx), (e) => /deep/.test(e.message)); + withFetch(async () => { throw new Error('https://ips/?cookie=secret'); }); + await assert.rejects(() => invoke({}, ctx), (e) => e.legacyCode === 'UNAVAILABLE' && !e.message.includes('secret')); }); test('valid log page with zero data rows returns empty entries', async () => { const ctx = buildCtx({ host: 'https://ips', cookie: 'c=1' }); withFetch(async () => fakeResponse(200, '
名称
')); - const out = await handlers[METHOD_QUERY_IPS_LOG_FULL]({}, ctx); + const out = await invoke({}, ctx); assert.equal(out.total, 0); assert.deepEqual(out.entries, []); }); +test('response size and read failures are bounded and redacted', async () => { + const ctx = buildCtx({ host: 'https://ips', cookie: 'c=1' }, { bindings: { maxResponseBytes: 1024 } }); + withFetch(async () => ({ status: 200, ok: true, headers: createHeaders({ 'content-length': '2048' }), text: async () => 'not read' })); + await assert.rejects(() => invoke({}, ctx), (e) => e.legacyCode === 'RESOURCE_EXHAUSTED'); + + withFetch(async (_url, options) => { + assert.equal(options.redirect, 'manual'); + return fakeResponse(200, 'x'.repeat(1025)); + }); + await assert.rejects(() => invoke({}, ctx), (e) => e.legacyCode === 'RESOURCE_EXHAUSTED'); + + withFetch(async () => ({ + status: 200, ok: true, headers: createHeaders(), + text: async () => { throw new Error('secret response failure'); }, + })); + await assert.rejects(() => invoke({}, ctx), (e) => e.legacyCode === 'UNAVAILABLE' && !e.message.includes('secret')); + + let cancelled = false; + const values = [new Uint8Array(700), new Uint8Array(700)]; + withFetch(async () => ({ + status: 200, ok: true, headers: createHeaders(), + body: { getReader: () => ({ + read: async () => (values.length ? { done: false, value: values.shift() } : { done: true }), + cancel: async () => { cancelled = true; }, + releaseLock: () => {}, + }) }, + })); + await assert.rejects(() => invoke({}, ctx), (e) => e.legacyCode === 'RESOURCE_EXHAUSTED'); + assert.equal(cancelled, true); +}); + // ---------- service surface + helpers ---------- test('service exposes the QueryIpsLog handler', () => { assert.equal(typeof service.handlers[METHOD_QUERY_IPS_LOG_FULL], 'function'); + assert.equal(typeof service.handlers[METHOD_PROBE_CONNECTIVITY_FULL], 'function'); }); test('helper coverage', () => { const h = _test; assert.equal(h.normalizeBaseUrl('https://h/'), 'https://h'); assert.equal(h.normalizeBaseUrl('ftp://x'), ''); + assert.equal(h.normalizeBaseUrl('https://user:pass@h'), ''); + assert.equal(h.normalizeBaseUrl('https://h/path'), ''); assert.equal(h.resolveCookie({ session_cookie: 'c' }), 'c'); assert.equal(h.resolveCookie({ sessionCookie: 'c2' }), 'c2'); assert.equal(h.decodeEntities('a&b<c>"' d'), 'a&b"\' d'); @@ -165,17 +233,19 @@ test('helper coverage', () => { assert.equal(h.pickBoolean('maybe'), undefined); assert.equal(h.pickFirstBoolean(['x', 'true']), true); assert.equal(h.unwrapScalar({ value: { value: 2 } }), 2); - assert.deepEqual(h.sanitizeHeaders({ A: 1, '': 2 }), { A: '1' }); + assert.deepEqual(h.sanitizeHeaders({ A: 1, '': 2, Cookie: 'bad', 'X-B': 'bad\r\nx: y' }), { A: '1' }); assert.deepEqual(h.sanitizeHeaders('x'), {}); assert.equal(h.buildTlsOptions({ skipTlsVerify: true }).skipTlsVerify, true); assert.deepEqual(h.buildTlsOptions({}), {}); assert.equal(h.resolveTimeoutMs({ limits: { timeoutMs: 0 } }), 5000); assert.equal(h.resolveTimeoutMs({ limits: { timeoutMs: 321 } }), 321); + assert.equal(h.resolveMaxResponseBytes({ bindings: { maxResponseBytes: 2048 } }), 2048); + assert.equal(h.resolveMaxResponseBytes({ bindings: { maxResponseBytes: 1 } }), 2 * 1024 * 1024); assert.equal(h.grpcCodeFor('NOPE'), grpcStatus.UNKNOWN); assert.ok(h.errorWithCode('UNAVAILABLE', 'x') instanceof GrpcError); - assert.throws(() => h.throwForHttpStatus(403, 'x'), (e) => e.legacyCode === 'PERMISSION_DENIED'); - assert.throws(() => h.throwForHttpStatus(400, 'x'), (e) => e.legacyCode === 'FAILED_PRECONDITION'); - assert.throws(() => h.throwForHttpStatus(500, 'x'), (e) => e.legacyCode === 'UNAVAILABLE'); + assert.throws(() => h.throwForHttpStatus(403), (e) => e.legacyCode === 'PERMISSION_DENIED'); + assert.throws(() => h.throwForHttpStatus(400), (e) => e.legacyCode === 'FAILED_PRECONDITION'); + assert.throws(() => h.throwForHttpStatus(500), (e) => e.legacyCode === 'UNAVAILABLE'); const hdr = h.buildHeaders({ headers: { 'X-A': '1' } }, { instance_id: 'i', request_id: 'r' }, 'c=1'); assert.equal(hdr.cookie, 'c=1'); assert.equal(hdr['X-A'], '1'); From 53b234524d71f65f379eafa737a88b549439bcc3 Mon Sep 17 00:00:00 2001 From: kingfs Date: Mon, 17 Aug 2026 20:55:36 +0800 Subject: [PATCH 3/7] Fix Venus IPS transport controls --- services/venus__ips/src/venus-ips.js | 48 ++++++++++++++-------- services/venus__ips/test/venus-ips.test.js | 18 +++++++- 2 files changed, 49 insertions(+), 17 deletions(-) diff --git a/services/venus__ips/src/venus-ips.js b/services/venus__ips/src/venus-ips.js index 19d100fd..29f13ed6 100644 --- a/services/venus__ips/src/venus-ips.js +++ b/services/venus__ips/src/venus-ips.js @@ -1,4 +1,5 @@ import { Buffer } from 'node:buffer'; +import { Agent } from 'undici'; // 启明星辰 IPS 攻击日志查询适配。 // 认证:web 会话 Cookie。GET /log/memorylog/ipslog.php 返回 HTML 日志页，解析表行为结构化条目。 @@ -27,6 +28,7 @@ const ENTRY_FIELDS = [ 'time', 'type', 'severity', 'priority', 'action', 'policy_id', 'count', 'content', ]; const DATETIME_RE = /\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}/; +const insecureTlsDispatcher = new Agent({ connect: { rejectUnauthorized: false } }); const grpcCodeFor = (code) => ({ FAILED_PRECONDITION: grpcStatus.FAILED_PRECONDITION, @@ -151,7 +153,22 @@ const resolveMaxResponseBytes = (ctx = {}) => { const buildTlsOptions = (bindings = {}) => { const enabled = pickFirstBoolean([bindings.skipTlsVerify, bindings.tlsInsecureSkipVerify, bindings.insecureSkipVerify]) || false; - return enabled ? { skipTlsVerify: true, tlsInsecureSkipVerify: true, insecureSkipVerify: true } : {}; + return enabled ? { dispatcher: insecureTlsDispatcher } : {}; +}; + +const buildRequestOptions = (bound) => { + const controller = new AbortController(); + const timer = setTimeout(() => controller.abort(), resolveTimeoutMs(bound)); + return { + options: { + method: 'GET', + redirect: 'manual', + signal: controller.signal, + ...buildTlsOptions(bound.bindings), + headers: buildHeaders(bound.bindings, bound.meta, bound.cookie), + }, + cleanup: () => clearTimeout(timer), + }; }; const sanitizeHeaders = (headers) => { @@ -266,26 +283,27 @@ const runQueryIpsLog = async (req = {}, ctx = {}) => { if (rawLimit < 0 || rawLimit > MAX_LIMIT) throw errorWithCode('INVALID_ARGUMENT', `limit must be between 0 and ${MAX_LIMIT}`); const limit = rawLimit; let response; + const upstreamRequest = buildRequestOptions(bound); try { - response = await fetch(`${bound.host}${IPS_LOG_URI}`, { - method: 'GET', - redirect: 'manual', - timeoutMs: resolveTimeoutMs(bound), - ...buildTlsOptions(bound.bindings), - headers: buildHeaders(bound.bindings, bound.meta, bound.cookie), - }); + response = await fetch(`${bound.host}${IPS_LOG_URI}`, upstreamRequest.options); } catch (err) { + upstreamRequest.cleanup(); if (err instanceof GrpcError) throw err; throw errorWithCode('UNAVAILABLE', 'upstream request failed'); } const status = Number(response.status); - if (!response.ok) throwForHttpStatus(status); + if (!response.ok) { + upstreamRequest.cleanup(); + throwForHttpStatus(status); + } let text; try { text = await readBoundedText(response, resolveMaxResponseBytes(bound)); } catch (err) { if (err instanceof GrpcError) throw err; throw errorWithCode('UNAVAILABLE', 'failed to read upstream response'); + } finally { + upstreamRequest.cleanup(); } // 会话失效时设备会重定向到登录页(同样 200),用日志页标记区分。 if (!String(text || '').includes(LOG_PAGE_MARKER)) { @@ -298,16 +316,13 @@ const runQueryIpsLog = async (req = {}, ctx = {}) => { const runProbeConnectivity = async (ctx = {}) => { const bound = requireBindings(ctx); let response; + const request = buildRequestOptions(bound); try { - response = await fetch(`${bound.host}${IPS_LOG_URI}`, { - method: 'GET', - redirect: 'manual', - timeoutMs: resolveTimeoutMs(bound), - ...buildTlsOptions(bound.bindings), - headers: buildHeaders(bound.bindings, bound.meta, bound.cookie), - }); + response = await fetch(`${bound.host}${IPS_LOG_URI}`, request.options); } catch { throw errorWithCode('UNAVAILABLE', 'upstream request failed'); + } finally { + request.cleanup(); } const status = Number(response.status); if (!response.ok) throwForHttpStatus(status); @@ -330,6 +345,7 @@ export const handlers = { export const _test = { buildHeaders, + buildRequestOptions, buildTlsOptions, decodeEntities, errorWithCode, diff --git a/services/venus__ips/test/venus-ips.test.js b/services/venus__ips/test/venus-ips.test.js index 9035b189..4dedc025 100644 --- a/services/venus__ips/test/venus-ips.test.js +++ b/services/venus__ips/test/venus-ips.test.js @@ -152,6 +152,22 @@ test('fetch errors are redacted', async () => { await assert.rejects(() => invoke({}, ctx), (e) => e.legacyCode === 'UNAVAILABLE' && !e.message.includes('secret')); }); +test('timeout uses AbortSignal and insecure TLS uses an undici dispatcher', async () => { + const ctx = buildCtx( + { host: 'https://ips', cookie: 'c=1' }, + { bindings: { skipTlsVerify: true }, limits: { timeoutMs: 5 } }, + ); + withFetch(async (_url, options) => { + assert.ok(options.signal instanceof AbortSignal); + assert.ok(options.dispatcher); + assert.equal('timeoutMs' in options, false); + assert.equal('skipTlsVerify' in options, false); + await new Promise((resolve) => options.signal.addEventListener('abort', resolve, { once: true })); + throw new DOMException('aborted', 'AbortError'); + }); + await assert.rejects(() => invoke({}, ctx), (e) => e.legacyCode === 'UNAVAILABLE'); +}); + test('valid log page with zero data rows returns empty entries', async () => { const ctx = buildCtx({ host: 'https://ips', cookie: 'c=1' }); withFetch(async () => fakeResponse(200, '
名称
')); @@ -235,7 +251,7 @@ test('helper coverage', () => { assert.equal(h.unwrapScalar({ value: { value: 2 } }), 2); assert.deepEqual(h.sanitizeHeaders({ A: 1, '': 2, Cookie: 'bad', 'X-B': 'bad\r\nx: y' }), { A: '1' }); assert.deepEqual(h.sanitizeHeaders('x'), {}); - assert.equal(h.buildTlsOptions({ skipTlsVerify: true }).skipTlsVerify, true); + assert.ok(h.buildTlsOptions({ skipTlsVerify: true }).dispatcher); assert.deepEqual(h.buildTlsOptions({}), {}); assert.equal(h.resolveTimeoutMs({ limits: { timeoutMs: 0 } }), 5000); assert.equal(h.resolveTimeoutMs({ limits: { timeoutMs: 321 } }), 321); From 52cf323e4e244214d679d37754883031843fb97b Mon Sep 17 00:00:00 2001 From: kingfs Date: Mon, 17 Aug 2026 21:02:07 +0800 Subject: [PATCH 4/7] Align Venus IPS response handling --- services/venus__ips/README.md | 2 +- services/venus__ips/proto/venus_ips.proto | 2 +- services/venus__ips/src/venus-ips.js | 16 ++++++++++++- services/venus__ips/test/venus-ips.test.js | 26 ++++++++++++++++++++++ 4 files changed, 43 insertions(+), 3 deletions(-) diff --git a/services/venus__ips/README.md b/services/venus__ips/README.md index 8e4c7315..1325f2de 100644 --- a/services/venus__ips/README.md +++ b/services/venus__ips/README.md @@ -33,7 +33,7 @@ cookie 经 `secret.cookie` 外部传入。 ### 请求 / 响应 -- 请求:`limit`(返回条目上限,客户端侧截断;<=0 表示全部)。 +- 请求:`limit`(返回条目上限,客户端侧截断；`0` 表示全部，负值返回 `INVALID_ARGUMENT`)。 - 响应:`http_status`、`total`(解析到的条目数)、`entries[]`。每条 `entries` 含: `name`(名称)、`src_ip`/`src_port`、`dst_ip`/`dst_port`、`protocol`、`time`、`type`(类型)、 `severity`(事件级别)、`priority`(优先级)、`action`(动作)、`policy_id`(策略ID)、`count`(发生次数)、`content`(内容)。 diff --git a/services/venus__ips/proto/venus_ips.proto b/services/venus__ips/proto/venus_ips.proto index 9f201473..d046b2d9 100644 --- a/services/venus__ips/proto/venus_ips.proto +++ b/services/venus__ips/proto/venus_ips.proto @@ -22,7 +22,7 @@ message ProbeConnectivityResponse { } message QueryIpsLogRequest { - int32 limit = 1; // 返回条目上限(客户端侧截断)，<=0 表示全部 + int32 limit = 1; // 返回条目上限(客户端侧截断)，0 表示全部；负值返回 INVALID_ARGUMENT } // 对应日志表列:名称/源IP/源端口/目的IP/目的端口/协议/时间/类型/级别/优先级/动作/策略ID/次数/内容。 diff --git a/services/venus__ips/src/venus-ips.js b/services/venus__ips/src/venus-ips.js index 29f13ed6..68b74a47 100644 --- a/services/venus__ips/src/venus-ips.js +++ b/services/venus__ips/src/venus-ips.js @@ -199,9 +199,18 @@ const throwForHttpStatus = (status) => { throw errorWithCode('UNAVAILABLE', `upstream unavailable (HTTP ${status})`); }; +const cancelResponseBody = async (response) => { + try { + await response?.body?.cancel?.(); + } catch { + // Cancellation is best-effort and must not replace the mapped upstream error. + } +}; + const readBoundedText = async (response, maxBytes) => { const declared = Number(response.headers?.get?.('content-length')); if (Number.isFinite(declared) && declared > maxBytes) { + await cancelResponseBody(response); throw errorWithCode('RESOURCE_EXHAUSTED', 'upstream response exceeds configured limit'); } if (response.body?.getReader) { @@ -294,6 +303,7 @@ const runQueryIpsLog = async (req = {}, ctx = {}) => { const status = Number(response.status); if (!response.ok) { upstreamRequest.cleanup(); + await cancelResponseBody(response); throwForHttpStatus(status); } let text; @@ -325,7 +335,10 @@ const runProbeConnectivity = async (ctx = {}) => { request.cleanup(); } const status = Number(response.status); - if (!response.ok) throwForHttpStatus(status); + if (!response.ok) { + await cancelResponseBody(response); + throwForHttpStatus(status); + } response.body?.cancel?.().catch?.(() => {}); return { reachable: true, http_status: status }; }; @@ -346,6 +359,7 @@ export const handlers = { export const _test = { buildHeaders, buildRequestOptions, + cancelResponseBody, buildTlsOptions, decodeEntities, errorWithCode, diff --git a/services/venus__ips/test/venus-ips.test.js b/services/venus__ips/test/venus-ips.test.js index 4dedc025..d043d1a4 100644 --- a/services/venus__ips/test/venus-ips.test.js +++ b/services/venus__ips/test/venus-ips.test.js @@ -146,6 +146,32 @@ test('error mapping: network / http', async () => { await assert.rejects(() => invoke({}, ctx), (e) => e.legacyCode === 'FAILED_PRECONDITION' && !e.message.includes('secret')); }); +test('early HTTP and declared-size failures cancel the upstream body', async () => { + const ctx = buildCtx({ host: 'https://ips', cookie: 'c=1' }, { bindings: { maxResponseBytes: 1024 } }); + let cancellations = 0; + const body = () => ({ cancel: async () => { cancellations += 1; } }); + + withFetch(async () => ({ status: 500, ok: false, headers: createHeaders(), body: body() })); + await assert.rejects(() => invoke({}, ctx), (e) => e.legacyCode === 'UNAVAILABLE'); + + withFetch(async () => ({ status: 403, ok: false, headers: createHeaders(), body: body() })); + await assert.rejects( + () => handlers[METHOD_PROBE_CONNECTIVITY_FULL](ctx), + (e) => e.legacyCode === 'PERMISSION_DENIED', + ); + + withFetch(async () => ({ + status: 200, + ok: true, + headers: createHeaders({ 'content-length': '2048' }), + body: body(), + })); + await assert.rejects(() => invoke({}, ctx), (e) => e.legacyCode === 'RESOURCE_EXHAUSTED'); + assert.equal(cancellations, 3); + + await _test.cancelResponseBody({ body: { cancel: async () => { throw new Error('cancel failed'); } } }); +}); + test('fetch errors are redacted', async () => { const ctx = buildCtx({ host: 'https://ips', cookie: 'c=1' }); withFetch(async () => { throw new Error('https://ips/?cookie=secret'); }); From 0bcc45779aa85204669f8609a43cf589c4163608 Mon Sep 17 00:00:00 2001 From: kingfs Date: Mon, 17 Aug 2026 21:06:01 +0800 Subject: [PATCH 5/7] Harden Venus IPS log parsing --- services/venus__ips/src/venus-ips.js | 24 ++++++++++++++-------- services/venus__ips/test/venus-ips.test.js | 9 ++++++++ 2 files changed, 24 insertions(+), 9 deletions(-) diff --git a/services/venus__ips/src/venus-ips.js b/services/venus__ips/src/venus-ips.js index 68b74a47..748163c5 100644 --- a/services/venus__ips/src/venus-ips.js +++ b/services/venus__ips/src/venus-ips.js @@ -251,13 +251,19 @@ const requireBindings = (ctx = {}) => { return { ...callCtx, bindings, host, cookie }; }; -const decodeEntities = (s) => String(s) - .replace(/&/g, '&') - .replace(/</g, '<') - .replace(/>/g, '>') - .replace(/"/g, '"') - .replace(/�?39;/g, "'") - .replace(/ /g, ' '); +const HTML_ENTITIES = Object.freeze({ + '&': '&', + '<': '<', + '>': '>', + '"': '"', + ''': "'", + ''': "'", + ' ': ' ', +}); +const decodeEntities = (s) => String(s).replace( + /&(amp|lt|gt|quot|#0?39|nbsp);/g, + (entity) => HTML_ENTITIES[entity], +); // 从单个 中按顺序取出带 title 的 文本。 const rowTitles = (rowHtml) => { @@ -275,8 +281,8 @@ const parseIpsLog = (html, limit = 0) => { let m; while ((m = rowRe.exec(html)) !== null) { const titles = rowTitles(m[1]); - if (titles.length < 13) continue; - if (!titles.some((v) => DATETIME_RE.test(v))) continue; + if (titles.length !== ENTRY_FIELDS.length) continue; + if (!DATETIME_RE.test(titles[6] ?? '')) continue; const entry = {}; ENTRY_FIELDS.forEach((key, i) => { entry[key] = titles[i] ?? ''; }); entries.push(entry); diff --git a/services/venus__ips/test/venus-ips.test.js b/services/venus__ips/test/venus-ips.test.js index d043d1a4..5eaa4984 100644 --- a/services/venus__ips/test/venus-ips.test.js +++ b/services/venus__ips/test/venus-ips.test.js @@ -249,6 +249,7 @@ test('helper coverage', () => { assert.equal(h.resolveCookie({ session_cookie: 'c' }), 'c'); assert.equal(h.resolveCookie({ sessionCookie: 'c2' }), 'c2'); assert.equal(h.decodeEntities('a&b<c>"' d'), 'a&b"\' d'); + assert.equal(h.decodeEntities('&lt; &amp; &quot;'), '< & "'); assert.equal(h.pickBoolean(true), true); assert.equal(h.pickBoolean(0), false); assert.equal(h.pickBoolean(undefined), undefined); @@ -264,6 +265,14 @@ test('helper coverage', () => { assert.equal(parsed[0].time, '2026-01-02 03:04:05'); // a row without a datetime is skipped assert.equal(h.parseIpsLog('' + Array.from({ length: 14 }, (_, i) => `x`).join('') + '').length, 0); + // Extra/missing titled cells and a datetime in the wrong column must not silently shift fields. + const titledCells = (count, timeIndex) => '' + Array.from( + { length: count }, + (_, i) => `x`, + ).join('') + ''; + assert.equal(h.parseIpsLog(titledCells(15, 7)).length, 0); + assert.equal(h.parseIpsLog(titledCells(13, 6)).length, 0); + assert.equal(h.parseIpsLog(titledCells(14, 5)).length, 0); // limit const two = '' + Array.from({ length: 14 }, (_, i) => `a`).join('') + ''; assert.equal(h.parseIpsLog(two + two, 1).length, 1); From f34354895aab27fa34fc0a32349509c1f39879fb Mon Sep 17 00:00:00 2001 From: kingfs Date: Mon, 17 Aug 2026 21:12:59 +0800 Subject: [PATCH 6/7] Fail closed on malformed IPS logs --- services/venus__ips/src/venus-ips.js | 21 ++++++++++++--- services/venus__ips/test/venus-ips.test.js | 30 ++++++++++++++++------ 2 files changed, 39 insertions(+), 12 deletions(-) diff --git a/services/venus__ips/src/venus-ips.js b/services/venus__ips/src/venus-ips.js index 748163c5..e5310ba8 100644 --- a/services/venus__ips/src/venus-ips.js +++ b/services/venus__ips/src/venus-ips.js @@ -277,18 +277,27 @@ const rowTitles = (rowHtml) => { // 解析 HTML 日志页为结构化条目:数据行需含时间且至少 13 个 title 单元格。 const parseIpsLog = (html, limit = 0) => { const entries = []; + let structuralRows = 0; + let skipped = 0; const rowRe = /]*>([\s\S]*?)<\/tr>/gi; let m; while ((m = rowRe.exec(html)) !== null) { const titles = rowTitles(m[1]); - if (titles.length !== ENTRY_FIELDS.length) continue; - if (!DATETIME_RE.test(titles[6] ?? '')) continue; + // Header rows have no titled data cells. Any titled row is a candidate log row and + // must have exactly 14 columns with the timestamp at index 6; fail closed upstream + // if a candidate is malformed so callers never receive a silently incomplete list. + if (titles.length === 0) continue; + structuralRows += 1; + if (titles.length !== ENTRY_FIELDS.length || !DATETIME_RE.test(titles[6] ?? '')) { + skipped += 1; + continue; + } const entry = {}; ENTRY_FIELDS.forEach((key, i) => { entry[key] = titles[i] ?? ''; }); entries.push(entry); if (limit > 0 && entries.length >= limit) break; } - return entries; + return { entries, skipped, structuralRows }; }; const runQueryIpsLog = async (req = {}, ctx = {}) => { @@ -325,7 +334,11 @@ const runQueryIpsLog = async (req = {}, ctx = {}) => { if (!String(text || '').includes(LOG_PAGE_MARKER)) { throw errorWithCode('FAILED_PRECONDITION', 'unexpected response (session may be expired or not the IPS log page)'); } - const entries = parseIpsLog(text, limit); + const parsed = parseIpsLog(text, limit); + if (parsed.skipped > 0) { + throw errorWithCode('FAILED_PRECONDITION', 'unexpected IPS log table structure'); + } + const { entries } = parsed; return { http_status: status, total: entries.length, entries }; }; diff --git a/services/venus__ips/test/venus-ips.test.js b/services/venus__ips/test/venus-ips.test.js index 5eaa4984..d3de8b9e 100644 --- a/services/venus__ips/test/venus-ips.test.js +++ b/services/venus__ips/test/venus-ips.test.js @@ -9,6 +9,7 @@ import { METHOD_PROBE_CONNECTIVITY_FULL, METHOD_QUERY_IPS_LOG_FULL, IPS_LOG_URI, + LOG_PAGE_MARKER, _test, handlers, rpcdef, @@ -202,6 +203,18 @@ test('valid log page with zero data rows returns empty entries', async () => { assert.deepEqual(out.entries, []); }); +test('malformed candidate log rows fail closed instead of returning incomplete data', async () => { + const ctx = buildCtx({ host: 'https://ips', cookie: 'c=1' }); + const titledCells = (count, timeIndex) => '' + Array.from( + { length: count }, + (_, i) => `x`, + ).join('') + ''; + for (const row of [titledCells(15, 7), titledCells(13, 6), titledCells(14, 5)]) { + withFetch(async () => fakeResponse(200, `${LOG_PAGE_MARKER}${row}
header
`)); + await assert.rejects(() => invoke({}, ctx), (e) => e.legacyCode === 'FAILED_PRECONDITION'); + } +}); + test('response size and read failures are bounded and redacted', async () => { const ctx = buildCtx({ host: 'https://ips', cookie: 'c=1' }, { bindings: { maxResponseBytes: 1024 } }); withFetch(async () => ({ status: 200, ok: true, headers: createHeaders({ 'content-length': '2048' }), text: async () => 'not read' })); @@ -260,22 +273,23 @@ test('helper coverage', () => { const html = 'h' + '#' + Array.from({ length: 14 }, (_, i) => `v${i}`).join('').replace('v6', '2026-01-02 03:04:05') + ''; const parsed = h.parseIpsLog(html); - assert.equal(parsed.length, 1); - assert.equal(parsed[0].name, 'v0'); - assert.equal(parsed[0].time, '2026-01-02 03:04:05'); + assert.equal(parsed.entries.length, 1); + assert.equal(parsed.entries[0].name, 'v0'); + assert.equal(parsed.entries[0].time, '2026-01-02 03:04:05'); + assert.deepEqual({ skipped: parsed.skipped, structuralRows: parsed.structuralRows }, { skipped: 0, structuralRows: 1 }); // a row without a datetime is skipped - assert.equal(h.parseIpsLog('' + Array.from({ length: 14 }, (_, i) => `x`).join('') + '').length, 0); + assert.equal(h.parseIpsLog('' + Array.from({ length: 14 }, (_, i) => `x`).join('') + '').skipped, 1); // Extra/missing titled cells and a datetime in the wrong column must not silently shift fields. const titledCells = (count, timeIndex) => '' + Array.from( { length: count }, (_, i) => `x`, ).join('') + ''; - assert.equal(h.parseIpsLog(titledCells(15, 7)).length, 0); - assert.equal(h.parseIpsLog(titledCells(13, 6)).length, 0); - assert.equal(h.parseIpsLog(titledCells(14, 5)).length, 0); + assert.equal(h.parseIpsLog(titledCells(15, 7)).skipped, 1); + assert.equal(h.parseIpsLog(titledCells(13, 6)).skipped, 1); + assert.equal(h.parseIpsLog(titledCells(14, 5)).skipped, 1); // limit const two = '' + Array.from({ length: 14 }, (_, i) => `a`).join('') + ''; - assert.equal(h.parseIpsLog(two + two, 1).length, 1); + assert.equal(h.parseIpsLog(two + two, 1).entries.length, 1); assert.equal(h.pickInt({ a: '5' }, ['a'], 0), 5); assert.equal(h.pickInt({ a: '' }, ['a'], 9), 9); From 5450659fc31a7b2d8859e4460d3bf9279f086206 Mon Sep 17 00:00:00 2001 From: kingfs Date: Tue, 18 Aug 2026 11:50:16 +0800 Subject: [PATCH 7/7] fix venus IPS strict log parsing --- services/venus__ips/src/venus-ips.js | 14 +++++++++++--- services/venus__ips/test/venus-ips.test.js | 4 ++++ 2 files changed, 15 insertions(+), 3 deletions(-) diff --git a/services/venus__ips/src/venus-ips.js b/services/venus__ips/src/venus-ips.js index e5310ba8..3e9a5c91 100644 --- a/services/venus__ips/src/venus-ips.js +++ b/services/venus__ips/src/venus-ips.js @@ -274,7 +274,7 @@ const rowTitles = (rowHtml) => { return titles; }; -// 解析 HTML 日志页为结构化条目:数据行需含时间且至少 13 个 title 单元格。 +// 解析 HTML 日志页为结构化条目：数据行必须包含恰好 14 个 title 单元格，时间位于第 7 列。 const parseIpsLog = (html, limit = 0) => { const entries = []; let structuralRows = 0; @@ -283,6 +283,13 @@ const parseIpsLog = (html, limit = 0) => { let m; while ((m = rowRe.exec(html)) !== null) { const titles = rowTitles(m[1]); + // Header rows have no data cells. A row containing but no title is a + // malformed candidate and must fail closed rather than silently disappearing. + if (titles.length === 0 && / { } const entry = {}; ENTRY_FIELDS.forEach((key, i) => { entry[key] = titles[i] ?? ''; }); - entries.push(entry); - if (limit > 0 && entries.length >= limit) break; + // Once the requested limit is reached, stop collecting but continue scanning + // remaining rows so malformed rows cannot be hidden by pagination. + if (limit === 0 || entries.length < limit) entries.push(entry); } return { entries, skipped, structuralRows }; }; diff --git a/services/venus__ips/test/venus-ips.test.js b/services/venus__ips/test/venus-ips.test.js index d3de8b9e..ce4049f5 100644 --- a/services/venus__ips/test/venus-ips.test.js +++ b/services/venus__ips/test/venus-ips.test.js @@ -287,9 +287,13 @@ test('helper coverage', () => { assert.equal(h.parseIpsLog(titledCells(15, 7)).skipped, 1); assert.equal(h.parseIpsLog(titledCells(13, 6)).skipped, 1); assert.equal(h.parseIpsLog(titledCells(14, 5)).skipped, 1); + // A data row with no recognizable title cells is structural corruption. + const untitled = 'name2026-01-02 03:04:05'; + assert.equal(h.parseIpsLog(untitled).skipped, 1); // limit const two = '' + Array.from({ length: 14 }, (_, i) => `a`).join('') + ''; assert.equal(h.parseIpsLog(two + two, 1).entries.length, 1); + assert.equal(h.parseIpsLog(two + titledCells(13, 6), 1).skipped, 1); assert.equal(h.pickInt({ a: '5' }, ['a'], 0), 5); assert.equal(h.pickInt({ a: '' }, ['a'], 9), 9);