diff --git a/services/bin/octobus-tentacles.js b/services/bin/octobus-tentacles.js index ea8155f4..919414fb 100755 --- a/services/bin/octobus-tentacles.js +++ b/services/bin/octobus-tentacles.js @@ -461,6 +461,10 @@ const services = { entryFile: "../topsec__edr/bin/topsec-edr.js", serviceModule: "../topsec__edr/src/service.js", }, + "venus-ips": { + entryFile: "../venus__ips/bin/venus-ips.js", + serviceModule: "../venus__ips/src/service.js", + }, "venus-ads-v3-6": { entryFile: "../venus__ads_v3-6/bin/venus-ads-v3-6.js", serviceModule: "../venus__ads_v3-6/src/service.js", diff --git a/services/bin/venus-ips.js b/services/bin/venus-ips.js new file mode 100755 index 00000000..42608733 --- /dev/null +++ b/services/bin/venus-ips.js @@ -0,0 +1,10 @@ +#!/usr/bin/env node + +import { fileURLToPath } from "node:url"; +import { runServiceMain } from "@chaitin-ai/octobus-sdk"; + +import { service } from "../venus__ips/src/service.js"; + +runServiceMain(service, { + entryFile: fileURLToPath(new URL("../venus__ips/bin/venus-ips.js", import.meta.url)), +}); diff --git a/services/package.json b/services/package.json index 6764a88d..882725f6 100644 --- a/services/package.json +++ b/services/package.json @@ -127,6 +127,7 @@ "leadsec-waf": "bin/leadsec-waf.js", "venus-ips-v6079": "bin/venus-ips-v6079.js", "venus-tar": "bin/venus-tar.js", + "venus-ips": "bin/venus-ips.js", "wangsu-label-ip": "bin/wangsu-label-ip.js", "wd-k01": "bin/wd-k01.js", "threatbook-hfish": "bin/threatbook-hfish.js", @@ -289,6 +290,7 @@ "bin/leadsec-waf.js", "bin/venus-ips-v6079.js", "bin/venus-tar.js", + "bin/venus-ips.js", "bin/wd-k01.js", "bin/threatbook-hfish.js", "bin/dongtai-iast.js", @@ -408,6 +410,7 @@ "leadsec__waf", "venus__ips_v6079", "venus__tar", + "venus__ips", "wd__k01", "huoxian__dongtai-iast", "filigran__opencti", diff --git a/services/venus__ips/README.md b/services/venus__ips/README.md new file mode 100644 index 00000000..1325f2de --- /dev/null +++ b/services/venus__ips/README.md @@ -0,0 +1,74 @@ +# Venustech IPS + +启明星辰 IPS(入侵防御系统)攻击日志查询的 OctoBus service package。 +属于「流量检测 / NIPS」类。设备日志页返回 HTML,本包解析其中的日志表行为结构化条目。 + +## 支持版本 + +启明星辰 IPS(web 控制台,日志页 `/log/memorylog/ipslog.php`)。请求/响应按真机抓包对齐。 + +## 认证方式(web 会话 Cookie) + +控制台以 **会话 Cookie** 鉴权(浏览器 `credentials: include`)。请求头 `Cookie: <会话cookie>`, +cookie 经 `secret.cookie` 外部传入。 + +> ⚠️ cookie 有时效,过期需更换。会话失效时设备会以 200 返回登录页;本包用日志页标记 +> (`ips_log_filter`)识别,识别失败时报 `FAILED_PRECONDITION`,避免把登录页当成空结果。 + +## 配置 + +```json +// config +{ "host": "https://192.168.1.10", "timeoutMs": 5000, "maxResponseBytes": 2097152, "skipTlsVerify": true } +// secret +{ "cookie": "PHPSESSID=<会话id>" } +``` + +## 方法 + +| RPC | 上游接口 | +| --- | --- | +| `ProbeConnectivity` | `GET /log/memorylog/ipslog.php`(仅报告 HTTP 可达性,不读取正文) | +| `QueryIpsLog` | `GET /log/memorylog/ipslog.php` | + +### 请求 / 响应 + +- 请求:`limit`(返回条目上限,客户端侧截断;`0` 表示全部,负值返回 `INVALID_ARGUMENT`)。 +- 响应:`http_status`、`total`(解析到的条目数)、`entries[]`。每条 `entries` 含: + `name`(名称)、`src_ip`/`src_port`、`dst_ip`/`dst_port`、`protocol`、`time`、`type`(类型)、 + `severity`(事件级别)、`priority`(优先级)、`action`(动作)、`policy_id`(策略ID)、`count`(发生次数)、`content`(内容)。 + +> 注:当前抓包为不带过滤的全量 GET;按源/目的 IP/时间过滤(设备 `ips_log_filter` 表单)未实现, +> 待补对应抓包后扩展。响应原始 HTML(较大且含内网地址)不回传,仅返回结构化条目。 + +## 风险边界 + +- 本方法为**只读查询**,无写操作,风险面低。 +- 会话 cookie 等同登录态,泄露即会话失陷;仅放 `secret`,勿写入 `config`、日志或截图。 +- 默认校验 TLS;私有自签部署需 `skipTlsVerify: true`。 +- 禁止跨站重定向,响应体默认最多 2 MiB;上游正文、网络异常详情和 cookie 不写入错误信息。 + +## 错误映射 + +| 场景 | gRPC code | +| --- | --- | +| 缺 host/cookie | `INVALID_ARGUMENT` | +| 上游 401/403 | `PERMISSION_DENIED` | +| 其它 4xx / 会话失效(返回登录页) | `FAILED_PRECONDITION` | +| 网络错误/超时/5xx | `UNAVAILABLE` | + +## 建议 capset + +`query-ips-log`(只读),可直接授权给 AI SOC / 工作流做告警拉取与研判。 + +## 验证方式 + +```bash +cd services +npm run validate -- --service-dir venus__ips +npm test -- --service-dir venus__ips --coverage +npm run pack:check +``` + +真机验证:用一个有效会话 cookie 调 `query-ips-log`,确认返回 IPS 告警条目(名称/源IP/目的IP/时间/级别/动作 等)。 +原作者说明实现按设备抓包对齐,但本 PR 当前没有可独立审计的脱敏真机截图;合并前仍需维护者核验实际设备兼容性。**代码/测试/截图里不得出现真实 cookie、内网地址或业务数据。** diff --git a/services/venus__ips/bin/venus-ips.js b/services/venus__ips/bin/venus-ips.js new file mode 100755 index 00000000..8e0066ef --- /dev/null +++ b/services/venus__ips/bin/venus-ips.js @@ -0,0 +1,6 @@ +#!/usr/bin/env node +import { runServiceMain } from '@chaitin-ai/octobus-sdk'; + +import { service } from '../src/service.js'; + +runServiceMain(service); diff --git a/services/venus__ips/config.schema.json b/services/venus__ips/config.schema.json new file mode 100644 index 00000000..a911f21e --- /dev/null +++ b/services/venus__ips/config.schema.json @@ -0,0 +1,16 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "type": "object", + "additionalProperties": true, + "properties": { + "host": { "type": "string", "description": "Venus IPS console base URL with scheme, e.g. https://192.168.1.10." }, + "restBaseUrl": { "type": "string", "description": "Alias for host." }, + "baseUrl": { "type": "string", "description": "Alias for host." }, + "timeoutMs": { "type": "integer", "minimum": 1, "default": 5000, "description": "HTTP timeout in milliseconds." }, + "maxResponseBytes": { "type": "integer", "minimum": 1024, "maximum": 8388608, "default": 2097152, "description": "Maximum upstream response size in bytes." }, + "skipTlsVerify": { "type": "boolean", "default": false, "description": "Skip TLS certificate verification for private deployments." }, + "tlsInsecureSkipVerify": { "type": "boolean", "default": false, "description": "Legacy alias for skipTlsVerify." }, + "insecureSkipVerify": { "type": "boolean", "default": false, "description": "Alias for skipTlsVerify." }, + "headers": { "type": "object", "additionalProperties": { "type": "string" }, "description": "Optional non-sensitive HTTP headers. Authentication and hop-by-hop headers are ignored." } + } +} diff --git a/services/venus__ips/package.json b/services/venus__ips/package.json new file mode 100644 index 00000000..e77c3b2d --- /dev/null +++ b/services/venus__ips/package.json @@ -0,0 +1,12 @@ +{ + "name": "venus-ips", + "version": "0.0.0", + "private": true, + "type": "module", + "bin": { + "venus-ips": "bin/venus-ips.js" + }, + "dependencies": { + "@chaitin-ai/octobus-sdk": "^0.6.0" + } +} diff --git a/services/venus__ips/proto/venus_ips.proto b/services/venus__ips/proto/venus_ips.proto new file mode 100644 index 00000000..d046b2d9 --- /dev/null +++ b/services/venus__ips/proto/venus_ips.proto @@ -0,0 +1,50 @@ +syntax = "proto3"; + +package VENUS_IPS; + +option go_package = "miner/grpc-service/VENUS_IPS"; + +// 启明星辰 IPS(入侵防御系统)攻击日志查询。 +// 认证为 web 会话 Cookie:GET /log/memorylog/ipslog.php,浏览器 credentials=include。 +// 设备返回 HTML 日志页,本 service 解析其中的日志表行,映射为结构化条目。 +service VENUS_IPS { + // 探测控制台 HTTP 可达性;不读取或返回响应正文,也不代表会话一定有效。 + rpc ProbeConnectivity(ProbeConnectivityRequest) returns (ProbeConnectivityResponse) {} + // 查询 IPS 攻击日志(内存日志): GET /log/memorylog/ipslog.php + rpc QueryIpsLog(QueryIpsLogRequest) returns (QueryIpsLogResponse) {} +} + +message ProbeConnectivityRequest {} + +message ProbeConnectivityResponse { + bool reachable = 1; + int32 http_status = 2; +} + +message QueryIpsLogRequest { + int32 limit = 1; // 返回条目上限(客户端侧截断),0 表示全部;负值返回 INVALID_ARGUMENT +} + +// 对应日志表列:名称/源IP/源端口/目的IP/目的端口/协议/时间/类型/级别/优先级/动作/策略ID/次数/内容。 +message IpsLogEntry { + string name = 1; // 名称 + string src_ip = 2; // 源IP + string src_port = 3; // 源端口 + string dst_ip = 4; // 目的IP + string dst_port = 5; // 目的端口 + string protocol = 6; // 协议类型 + string time = 7; // 时间 + string type = 8; // 类型 + string severity = 9; // 事件级别 + string priority = 10; // 优先级 + string action = 11; // 动作 + string policy_id = 12;// 入侵防御策略ID + string count = 13; // 发生次数 + string content = 14; // 内容 +} + +message QueryIpsLogResponse { + int32 http_status = 1; // 上游 HTTP 状态码 + int32 total = 2; // 解析到的条目数 + repeated IpsLogEntry entries = 3; // 结构化日志条目 +} diff --git a/services/venus__ips/secret.schema.json b/services/venus__ips/secret.schema.json new file mode 100644 index 00000000..c344c96b --- /dev/null +++ b/services/venus__ips/secret.schema.json @@ -0,0 +1,11 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "type": "object", + "additionalProperties": true, + "required": ["cookie"], + "properties": { + "cookie": { "type": "string", "description": "Venus IPS web session cookie header value (e.g. 'PHPSESSID=...'). Time-limited; refresh when expired." }, + "sessionCookie": { "type": "string", "description": "Alias for cookie." }, + "session_cookie": { "type": "string", "description": "Alias for cookie." } + } +} diff --git a/services/venus__ips/service.json b/services/venus__ips/service.json new file mode 100644 index 00000000..fd56a072 --- /dev/null +++ b/services/venus__ips/service.json @@ -0,0 +1,24 @@ +{ + "schema": "chaitin.octobus.service.v1", + "name": "venus-ips", + "displayName": "Venustech IPS", + "description": "OctoBus package for Venustech IPS attack-log query (GET /log/memorylog/ipslog.php, web session cookie, HTML log page parsed into structured entries).", + "runtime": { "mode": "long-running" }, + "proto": { "roots": ["proto"], "files": ["proto/venus_ips.proto"] }, + "configSchema": "config.schema.json", + "secretSchema": "secret.schema.json", + "sdk": { + "cli": { + "commands": { + "VENUS_IPS.VENUS_IPS/ProbeConnectivity": { + "name": "probe-connectivity", + "description": "Probe Venus IPS console HTTP connectivity without returning response content." + }, + "VENUS_IPS.VENUS_IPS/QueryIpsLog": { + "name": "query-ips-log", + "description": "Query Venus IPS attack logs (memory log)." + } + } + } + } +} diff --git a/services/venus__ips/src/service.js b/services/venus__ips/src/service.js new file mode 100644 index 00000000..0fe48e87 --- /dev/null +++ b/services/venus__ips/src/service.js @@ -0,0 +1,7 @@ +import { defineService } from '@chaitin-ai/octobus-sdk'; + +import { handlers } from './venus-ips.js'; + +export { handlers } from './venus-ips.js'; + +export const service = defineService({ handlers }); diff --git a/services/venus__ips/src/venus-ips.js b/services/venus__ips/src/venus-ips.js new file mode 100644 index 00000000..3e9a5c91 --- /dev/null +++ b/services/venus__ips/src/venus-ips.js @@ -0,0 +1,413 @@ +import { Buffer } from 'node:buffer'; +import { Agent } from 'undici'; + +// 启明星辰 IPS 攻击日志查询适配。 +// 认证:web 会话 Cookie。GET /log/memorylog/ipslog.php 返回 HTML 日志页,解析表行为结构化条目。 +import { GrpcError, grpcStatus } from '@chaitin-ai/octobus-sdk'; + +const SVC = 'VENUS_IPS.VENUS_IPS'; +export const QUERY_IPS_LOG_PATH = `/${SVC}/QueryIpsLog`; +export const METHOD_QUERY_IPS_LOG_FULL = `${SVC}/QueryIpsLog`; +export const PROBE_CONNECTIVITY_PATH = `/${SVC}/ProbeConnectivity`; +export const METHOD_PROBE_CONNECTIVITY_FULL = `${SVC}/ProbeConnectivity`; + +export const IPS_LOG_URI = '/log/memorylog/ipslog.php'; +export const LOG_PAGE_MARKER = 'ips_log_filter'; +export const DEFAULT_TIMEOUT_MS = 5000; +export const DEFAULT_MAX_RESPONSE_BYTES = 2 * 1024 * 1024; +export const MAX_LIMIT = 10_000; + +const BLOCKED_HEADERS = new Set([ + 'authorization', 'cookie', 'host', 'connection', 'content-length', 'proxy-authorization', + 'transfer-encoding', 'upgrade', 'x-engine-instance', 'x-request-id', +]); + +// 日志表 14 个有 title 的数据单元格,按列顺序映射。 +const ENTRY_FIELDS = [ + 'name', 'src_ip', 'src_port', 'dst_ip', 'dst_port', 'protocol', + 'time', 'type', 'severity', 'priority', 'action', 'policy_id', 'count', 'content', +]; +const DATETIME_RE = /\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}/; +const insecureTlsDispatcher = new Agent({ connect: { rejectUnauthorized: false } }); + +const grpcCodeFor = (code) => ({ + FAILED_PRECONDITION: grpcStatus.FAILED_PRECONDITION, + INVALID_ARGUMENT: grpcStatus.INVALID_ARGUMENT, + PERMISSION_DENIED: grpcStatus.PERMISSION_DENIED, + RESOURCE_EXHAUSTED: grpcStatus.RESOURCE_EXHAUSTED, + UNAVAILABLE: grpcStatus.UNAVAILABLE, + UNKNOWN: grpcStatus.UNKNOWN, +})[code] ?? grpcStatus.UNKNOWN; + +const errorWithCode = (code, message) => { + const err = new GrpcError(grpcCodeFor(code), `${code}: ${message}`); + err.legacyCode = code; + return err; +}; + +const hasOwn = (obj, key) => Object.prototype.hasOwnProperty.call(obj ?? {}, key); + +const unwrapScalar = (value) => { + if (value === undefined || value === null) return undefined; + if (typeof value === 'object' && hasOwn(value, 'value')) return unwrapScalar(value.value); + return value; +}; + +const pickFirstString = (values = []) => { + for (const value of values) { + const raw = unwrapScalar(value); + if (raw === undefined || raw === null) continue; + const str = String(raw).trim(); + if (str) return str; + } + return ''; +}; + +const pickStringFrom = (source = {}, keys = []) => { + for (const key of keys) { + if (!hasOwn(source, key)) continue; + const raw = unwrapScalar(source[key]); + if (raw === undefined || raw === null) continue; + const value = String(raw).trim(); + if (value) return value; + } + return ''; +}; + +const pickInt = (source = {}, keys = [], fallback = 0) => { + for (const key of keys) { + if (!hasOwn(source, key)) continue; + const raw = unwrapScalar(source[key]); + if (raw === undefined || raw === null || raw === '') continue; + const num = Number(raw); + if (Number.isFinite(num)) return Math.trunc(num); + } + return fallback; +}; + +const pickBoolean = (value) => { + const raw = unwrapScalar(value); + if (raw === undefined || raw === null) return undefined; + if (typeof raw === 'boolean') return raw; + if (typeof raw === 'number') return Number.isNaN(raw) ? undefined : raw !== 0; + if (typeof raw === 'string') { + const normalized = raw.trim().toLowerCase(); + if (['true', '1', 'yes', 'y', 'on'].includes(normalized)) return true; + if (['false', '0', 'no', 'n', 'off', ''].includes(normalized)) return false; + } + return undefined; +}; + +const pickFirstBoolean = (values = []) => { + for (const value of values) { + const bool = pickBoolean(value); + if (bool !== undefined) return bool; + } + return undefined; +}; + +const normalizeBaseUrl = (value) => { + const raw = String(unwrapScalar(value) || '').trim(); + try { + const url = new URL(raw); + if (!['http:', 'https:'].includes(url.protocol) + || url.username || url.password || url.search || url.hash + || (url.pathname && url.pathname !== '/')) return ''; + return url.origin; + } catch { + return ''; + } +}; + +const resolveCallContext = (ctx = {}) => ({ + ...ctx, + bindings: { + ...(ctx.config ?? {}), + ...(ctx.secret ?? {}), + ...(ctx.bindings ?? {}), + }, + limits: ctx.limits ?? {}, + meta: ctx.meta ?? {}, + req: ctx.req ?? ctx.request ?? {}, +}); + +const resolveHost = (bindings = {}) => { + for (const candidate of [bindings.host, bindings.restBaseUrl, bindings.baseUrl]) { + const normalized = normalizeBaseUrl(candidate); + if (normalized) return normalized; + } + return ''; +}; +const resolveCookie = (bindings = {}) => pickStringFrom(bindings, ['cookie', 'sessionCookie', 'session_cookie']); + +const resolveTimeoutMs = (ctx = {}) => { + const raw = Number(unwrapScalar(ctx.limits?.timeoutMs ?? ctx.bindings?.timeoutMs ?? DEFAULT_TIMEOUT_MS)); + return Number.isFinite(raw) && raw > 0 ? raw : DEFAULT_TIMEOUT_MS; +}; + +const resolveMaxResponseBytes = (ctx = {}) => { + const raw = Number(unwrapScalar(ctx.bindings?.maxResponseBytes ?? DEFAULT_MAX_RESPONSE_BYTES)); + return Number.isFinite(raw) && raw >= 1024 && raw <= 8 * 1024 * 1024 + ? Math.trunc(raw) : DEFAULT_MAX_RESPONSE_BYTES; +}; + +const buildTlsOptions = (bindings = {}) => { + const enabled = pickFirstBoolean([bindings.skipTlsVerify, bindings.tlsInsecureSkipVerify, bindings.insecureSkipVerify]) || false; + return enabled ? { dispatcher: insecureTlsDispatcher } : {}; +}; + +const buildRequestOptions = (bound) => { + const controller = new AbortController(); + const timer = setTimeout(() => controller.abort(), resolveTimeoutMs(bound)); + return { + options: { + method: 'GET', + redirect: 'manual', + signal: controller.signal, + ...buildTlsOptions(bound.bindings), + headers: buildHeaders(bound.bindings, bound.meta, bound.cookie), + }, + cleanup: () => clearTimeout(timer), + }; +}; + +const sanitizeHeaders = (headers) => { + const raw = unwrapScalar(headers); + if (!raw || typeof raw !== 'object' || Array.isArray(raw)) return {}; + const result = {}; + for (const [key, value] of Object.entries(raw)) { + const normalized = key.trim().toLowerCase(); + const text = String(unwrapScalar(value) ?? ''); + if (!normalized || BLOCKED_HEADERS.has(normalized) || /[\r\n]/.test(key) || /[\r\n]/.test(text)) continue; + result[key] = text; + } + return result; +}; + +const buildHeaders = (bindings = {}, meta = {}, cookie = '') => ({ + ...sanitizeHeaders(bindings.headers), + accept: 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8', + cookie, + 'x-engine-instance': pickFirstString([meta.instance_id, meta.instanceId, 'unknown']), + 'x-request-id': pickFirstString([meta.request_id, meta.requestId, 'unknown']), +}); + +const throwForHttpStatus = (status) => { + if (status === 401 || status === 403) throw errorWithCode('PERMISSION_DENIED', `upstream rejected authentication (HTTP ${status})`); + if (status >= 300 && status < 400) throw errorWithCode('FAILED_PRECONDITION', 'upstream redirect refused (session may be expired)'); + if (status >= 400 && status < 500) throw errorWithCode('FAILED_PRECONDITION', `upstream rejected request (HTTP ${status})`); + throw errorWithCode('UNAVAILABLE', `upstream unavailable (HTTP ${status})`); +}; + +const cancelResponseBody = async (response) => { + try { + await response?.body?.cancel?.(); + } catch { + // Cancellation is best-effort and must not replace the mapped upstream error. + } +}; + +const readBoundedText = async (response, maxBytes) => { + const declared = Number(response.headers?.get?.('content-length')); + if (Number.isFinite(declared) && declared > maxBytes) { + await cancelResponseBody(response); + throw errorWithCode('RESOURCE_EXHAUSTED', 'upstream response exceeds configured limit'); + } + if (response.body?.getReader) { + const reader = response.body.getReader(); + const chunks = []; + let size = 0; + try { + while (true) { + const { done, value } = await reader.read(); + if (done) break; + size += value.byteLength; + if (size > maxBytes) { + await reader.cancel?.(); + throw errorWithCode('RESOURCE_EXHAUSTED', 'upstream response exceeds configured limit'); + } + chunks.push(Buffer.from(value)); + } + } finally { + reader.releaseLock?.(); + } + return Buffer.concat(chunks).toString('utf8'); + } + const text = await response.text(); + if (Buffer.byteLength(String(text), 'utf8') > maxBytes) { + throw errorWithCode('RESOURCE_EXHAUSTED', 'upstream response exceeds configured limit'); + } + return String(text); +}; + +const requireBindings = (ctx = {}) => { + const callCtx = resolveCallContext(ctx); + const bindings = callCtx.bindings || {}; + const host = resolveHost(bindings); + if (!host) throw errorWithCode('INVALID_ARGUMENT', 'bindings.host is required'); + const cookie = resolveCookie(bindings); + if (!cookie) throw errorWithCode('INVALID_ARGUMENT', 'bindings.cookie (web session cookie) is required'); + if (cookie.length > 8192 || /[\r\n]/.test(cookie)) throw errorWithCode('INVALID_ARGUMENT', 'bindings.cookie is invalid'); + return { ...callCtx, bindings, host, cookie }; +}; + +const HTML_ENTITIES = Object.freeze({ + '&': '&', + '<': '<', + '>': '>', + '"': '"', + ''': "'", + ''': "'", + ' ': ' ', +}); +const decodeEntities = (s) => String(s).replace( + /&(amp|lt|gt|quot|#0?39|nbsp);/g, + (entity) => HTML_ENTITIES[entity], +); + +// 从单个 中按顺序取出带 title 的 文本。 +const rowTitles = (rowHtml) => { + const titles = []; + const tdRe = /]*\btitle="([^"]*)"[^>]*>/gi; + let c; + while ((c = tdRe.exec(rowHtml)) !== null) titles.push(decodeEntities(c[1])); + return titles; +}; + +// 解析 HTML 日志页为结构化条目:数据行必须包含恰好 14 个 title 单元格,时间位于第 7 列。 +const parseIpsLog = (html, limit = 0) => { + const entries = []; + let structuralRows = 0; + let skipped = 0; + const rowRe = /]*>([\s\S]*?)<\/tr>/gi; + let m; + while ((m = rowRe.exec(html)) !== null) { + const titles = rowTitles(m[1]); + // Header rows have no data cells. A row containing but no title is a + // malformed candidate and must fail closed rather than silently disappearing. + if (titles.length === 0 && / { entry[key] = titles[i] ?? ''; }); + // Once the requested limit is reached, stop collecting but continue scanning + // remaining rows so malformed rows cannot be hidden by pagination. + if (limit === 0 || entries.length < limit) entries.push(entry); + } + return { entries, skipped, structuralRows }; +}; + +const runQueryIpsLog = async (req = {}, ctx = {}) => { + const bound = requireBindings(ctx); + const request = bound.req ? { ...bound.req, ...req } : req; + const rawLimit = pickInt(request, ['limit'], 0); + if (rawLimit < 0 || rawLimit > MAX_LIMIT) throw errorWithCode('INVALID_ARGUMENT', `limit must be between 0 and ${MAX_LIMIT}`); + const limit = rawLimit; + let response; + const upstreamRequest = buildRequestOptions(bound); + try { + response = await fetch(`${bound.host}${IPS_LOG_URI}`, upstreamRequest.options); + } catch (err) { + upstreamRequest.cleanup(); + if (err instanceof GrpcError) throw err; + throw errorWithCode('UNAVAILABLE', 'upstream request failed'); + } + const status = Number(response.status); + if (!response.ok) { + upstreamRequest.cleanup(); + await cancelResponseBody(response); + throwForHttpStatus(status); + } + let text; + try { + text = await readBoundedText(response, resolveMaxResponseBytes(bound)); + } catch (err) { + if (err instanceof GrpcError) throw err; + throw errorWithCode('UNAVAILABLE', 'failed to read upstream response'); + } finally { + upstreamRequest.cleanup(); + } + // 会话失效时设备会重定向到登录页(同样 200),用日志页标记区分。 + if (!String(text || '').includes(LOG_PAGE_MARKER)) { + throw errorWithCode('FAILED_PRECONDITION', 'unexpected response (session may be expired or not the IPS log page)'); + } + const parsed = parseIpsLog(text, limit); + if (parsed.skipped > 0) { + throw errorWithCode('FAILED_PRECONDITION', 'unexpected IPS log table structure'); + } + const { entries } = parsed; + return { http_status: status, total: entries.length, entries }; +}; + +const runProbeConnectivity = async (ctx = {}) => { + const bound = requireBindings(ctx); + let response; + const request = buildRequestOptions(bound); + try { + response = await fetch(`${bound.host}${IPS_LOG_URI}`, request.options); + } catch { + throw errorWithCode('UNAVAILABLE', 'upstream request failed'); + } finally { + request.cleanup(); + } + const status = Number(response.status); + if (!response.ok) { + await cancelResponseBody(response); + throwForHttpStatus(status); + } + response.body?.cancel?.().catch?.(() => {}); + return { reachable: true, http_status: status }; +}; + +export function rpcdef(ctx = {}) { + const callCtx = resolveCallContext(ctx); + return { + [PROBE_CONNECTIVITY_PATH]: async () => runProbeConnectivity(callCtx), + [QUERY_IPS_LOG_PATH]: async (req) => runQueryIpsLog(req ?? callCtx.req, callCtx), + }; +} + +export const handlers = { + [METHOD_PROBE_CONNECTIVITY_FULL]: (ctx = {}) => runProbeConnectivity(ctx), + [METHOD_QUERY_IPS_LOG_FULL]: (ctx = {}) => runQueryIpsLog(ctx.request ?? ctx.req ?? {}, ctx), +}; + +export const _test = { + buildHeaders, + buildRequestOptions, + cancelResponseBody, + buildTlsOptions, + decodeEntities, + errorWithCode, + grpcCodeFor, + hasOwn, + normalizeBaseUrl, + parseIpsLog, + pickBoolean, + pickFirstBoolean, + pickFirstString, + pickInt, + pickStringFrom, + requireBindings, + resolveCallContext, + resolveCookie, + resolveHost, + resolveMaxResponseBytes, + resolveTimeoutMs, + rowTitles, + readBoundedText, + sanitizeHeaders, + throwForHttpStatus, + unwrapScalar, +}; diff --git a/services/venus__ips/test/mock_upstream.js b/services/venus__ips/test/mock_upstream.js new file mode 100644 index 00000000..a90e2df8 --- /dev/null +++ b/services/venus__ips/test/mock_upstream.js @@ -0,0 +1,47 @@ +/* node:coverage disable */ +import http from 'node:http'; + +// 合成的 IPS 日志 HTML 页(结构与真机一致: 数据单元格 + ips_log_filter 标记), +// 使用文档保留地址段(198.51.100.x / 203.0.113.x),不含任何真实数据。 +const FIELDS_ROWS = [ + ['TCP_可疑行为_安全风险_MYSQL_查询系统变量', '198.51.100.10', '60782', '203.0.113.5', '3883', 'TCP', '2026-06-25 17:49:45', '可疑行为', '中', '警示', 'PASS', '1', '3', ''], + ['UDP_扫描_端口扫描', '198.51.100.11', '53', '203.0.113.6', '161', 'UDP', '2026-06-25 17:10:44', '扫描', '高', '严重', 'DROP', '2', '1', '备注X'], +]; + +const dataRow = (cells) => + `#${cells.map((v) => `${v}`).join('')}操作`; + +const buildLogHtml = () => ` +
+ + + ${FIELDS_ROWS.map(dataRow).join('\n ')} +
#名称源IP源端口目的IP目的端口协议类型时间类型事件级别优先级动作策略ID次数内容操作
`; + +// 登录页(无 ips_log_filter 标记),用于模拟会话失效后的 200 重定向。 +const LOGIN_HTML = '
'; + +export const createMockServer = async ({ cookie = 'PHPSESSID=abc123' } = {}) => { + const state = { requests: [] }; + + const server = http.createServer((req, res) => { + state.requests.push({ url: req.url, method: req.method, cookie: req.headers.cookie }); + if (req.url !== '/log/memorylog/ipslog.php' || req.method !== 'GET') { + res.writeHead(404, { 'content-type': 'text/html' }); res.end('not found'); return; + } + // 未带正确 cookie -> 返回登录页(会话失效) + const body = req.headers.cookie === cookie ? buildLogHtml() : LOGIN_HTML; + res.writeHead(200, { 'content-type': 'text/html; charset=utf-8' }); + res.end(body); + }); + + await new Promise((resolve) => server.listen(0, '127.0.0.1', resolve)); + const { port } = server.address(); + return { + state, + host: `http://127.0.0.1:${port}`, + cookie, + rowCount: FIELDS_ROWS.length, + async close() { await new Promise((resolve) => server.close(resolve)); }, + }; +}; diff --git a/services/venus__ips/test/smoke.json b/services/venus__ips/test/smoke.json new file mode 100644 index 00000000..a6f83fc7 --- /dev/null +++ b/services/venus__ips/test/smoke.json @@ -0,0 +1,12 @@ +{ + "method": "VENUS_IPS.VENUS_IPS/ProbeConnectivity", + "request": {}, + "expectUpstream": true, + "requireBusinessSuccess": true, + "requireUpstreamPerProtocol": true, + "protocols": ["connect", "grpc", "mcp"], + "upstream": { + "method": "GET", + "path": "/log/memorylog/ipslog.php" + } +} diff --git a/services/venus__ips/test/venus-ips.test.js b/services/venus__ips/test/venus-ips.test.js new file mode 100644 index 00000000..ce4049f5 --- /dev/null +++ b/services/venus__ips/test/venus-ips.test.js @@ -0,0 +1,333 @@ +import assert from 'node:assert/strict'; +import test from 'node:test'; + +import { GrpcError, grpcStatus } from '@chaitin-ai/octobus-sdk'; + +import { + QUERY_IPS_LOG_PATH, + PROBE_CONNECTIVITY_PATH, + METHOD_PROBE_CONNECTIVITY_FULL, + METHOD_QUERY_IPS_LOG_FULL, + IPS_LOG_URI, + LOG_PAGE_MARKER, + _test, + handlers, + rpcdef, +} from '../src/venus-ips.js'; +import { service } from '../src/service.js'; +import { createMockServer } from './mock_upstream.js'; + +const originalFetch = globalThis.fetch; +let seq = 0; +const nextId = () => `inst-${++seq}`; + +const buildCtx = (mock, overrides = {}) => ({ + bindings: { host: mock?.host, cookie: mock?.cookie, ...(overrides.bindings || {}) }, + config: overrides.config || {}, + secret: overrides.secret || {}, + limits: { timeoutMs: 10_000, ...(overrides.limits || {}) }, + meta: { instance_id: nextId(), request_id: 'req', ...(overrides.meta || {}) }, + req: overrides.req || {}, +}); + +const createHeaders = (entries = {}) => { + const map = new Map(); + for (const [k, v] of Object.entries(entries)) map.set(String(k).toLowerCase(), Array.isArray(v) ? v.map(String) : [String(v)]); + return { get(n) { const x = map.get(String(n).toLowerCase()); return x?.length ? x.join(', ') : null; } }; +}; +const fakeResponse = (status, body, ok = status >= 200 && status < 300) => ({ status, ok, headers: createHeaders(), text: async () => body }); +const withFetch = (impl) => { globalThis.fetch = impl; }; +const invoke = (request, ctx) => handlers[METHOD_QUERY_IPS_LOG_FULL]({ ...ctx, request }); + +test.afterEach(() => { globalThis.fetch = originalFetch; }); + +// ---------- end-to-end against mock ---------- + +test('parses IPS log HTML into structured entries', async () => { + const mock = await createMockServer(); + try { + const out = await rpcdef(buildCtx(mock))[QUERY_IPS_LOG_PATH]({}); + assert.equal(out.http_status, 200); + assert.equal(out.total, mock.rowCount); + const first = out.entries[0]; + assert.equal(first.name, 'TCP_可疑行为_安全风险_MYSQL_查询系统变量'); + assert.equal(first.src_ip, '198.51.100.10'); + assert.equal(first.src_port, '60782'); + assert.equal(first.dst_ip, '203.0.113.5'); + assert.equal(first.protocol, 'TCP'); + assert.equal(first.time, '2026-06-25 17:49:45'); + assert.equal(first.severity, '中'); + assert.equal(first.action, 'PASS'); + assert.equal(first.count, '3'); + assert.equal(out.entries[1].content, '备注X'); + // request shape + const r = mock.state.requests[0]; + assert.equal(r.method, 'GET'); + assert.equal(r.url, IPS_LOG_URI); + assert.equal(r.cookie, mock.cookie); + } finally { + await mock.close(); + } +}); + +test('connectivity probe uses the same hardened upstream request', async () => { + const mock = await createMockServer(); + try { + const ctx = buildCtx(mock); + const out = await handlers[METHOD_PROBE_CONNECTIVITY_FULL](ctx); + assert.deepEqual(out, { reachable: true, http_status: 200 }); + assert.equal(mock.state.requests[0].cookie, mock.cookie); + const viaRpcdef = await rpcdef(ctx)[PROBE_CONNECTIVITY_PATH](); + assert.equal(viaRpcdef.reachable, true); + } finally { + await mock.close(); + } +}); + +test('connectivity probe maps network and HTTP failures without leaking details', async () => { + const ctx = buildCtx({ host: 'https://ips', cookie: 'c=1' }); + withFetch(async () => { throw new Error('secret network detail'); }); + await assert.rejects( + () => handlers[METHOD_PROBE_CONNECTIVITY_FULL](ctx), + (e) => e.legacyCode === 'UNAVAILABLE' && !e.message.includes('secret'), + ); + withFetch(async () => fakeResponse(401, 'secret response', false)); + await assert.rejects( + () => handlers[METHOD_PROBE_CONNECTIVITY_FULL](ctx), + (e) => e.legacyCode === 'PERMISSION_DENIED' && !e.message.includes('secret'), + ); +}); + +test('limit caps the number of returned entries', async () => { + const mock = await createMockServer(); + try { + const out = await invoke({ limit: 1 }, buildCtx(mock)); + assert.equal(out.total, 1); + assert.equal(out.entries.length, 1); + } finally { + await mock.close(); + } +}); + +test('expired session (login page, no marker) -> FAILED_PRECONDITION', async () => { + const mock = await createMockServer(); + try { + await assert.rejects( + () => invoke({}, buildCtx(mock, { bindings: { cookie: 'PHPSESSID=wrong' } })), + (e) => e.legacyCode === 'FAILED_PRECONDITION', + ); + } finally { + await mock.close(); + } +}); + +// ---------- validation ---------- + +test('binding validation', async () => { + await assert.rejects(() => invoke({}, buildCtx({ host: '' })), (e) => e.legacyCode === 'INVALID_ARGUMENT'); + await assert.rejects(() => invoke({}, buildCtx({ host: 'https://h', cookie: '' })), (e) => e.legacyCode === 'INVALID_ARGUMENT'); + await assert.rejects(() => invoke({ limit: -1 }, buildCtx({ host: 'https://h', cookie: 'c' })), (e) => e.legacyCode === 'INVALID_ARGUMENT'); + await assert.rejects(() => invoke({ limit: 10_001 }, buildCtx({ host: 'https://h', cookie: 'c' })), (e) => e.legacyCode === 'INVALID_ARGUMENT'); + await assert.rejects(() => invoke({}, buildCtx({ host: 'https://h', cookie: 'bad\r\nx: y' })), (e) => e.legacyCode === 'INVALID_ARGUMENT'); +}); + +// ---------- error mapping ---------- + +test('error mapping: network / http', async () => { + const ctx = buildCtx({ host: 'https://ips', cookie: 'c=1' }); + withFetch(async () => { throw new Error('ECONNREFUSED'); }); + await assert.rejects(() => invoke({}, ctx), (e) => e.legacyCode === 'UNAVAILABLE' && !e.message.includes('ECONNREFUSED')); + withFetch(async () => fakeResponse(401, 'no', false)); + await assert.rejects(() => invoke({}, ctx), (e) => e.legacyCode === 'PERMISSION_DENIED' && !e.message.includes('no')); + withFetch(async () => fakeResponse(404, 'no', false)); + await assert.rejects(() => invoke({}, ctx), (e) => e.legacyCode === 'FAILED_PRECONDITION'); + withFetch(async () => fakeResponse(500, 'no', false)); + await assert.rejects(() => invoke({}, ctx), (e) => e.legacyCode === 'UNAVAILABLE'); + withFetch(async () => fakeResponse(302, 'location secret', false)); + await assert.rejects(() => invoke({}, ctx), (e) => e.legacyCode === 'FAILED_PRECONDITION' && !e.message.includes('secret')); +}); + +test('early HTTP and declared-size failures cancel the upstream body', async () => { + const ctx = buildCtx({ host: 'https://ips', cookie: 'c=1' }, { bindings: { maxResponseBytes: 1024 } }); + let cancellations = 0; + const body = () => ({ cancel: async () => { cancellations += 1; } }); + + withFetch(async () => ({ status: 500, ok: false, headers: createHeaders(), body: body() })); + await assert.rejects(() => invoke({}, ctx), (e) => e.legacyCode === 'UNAVAILABLE'); + + withFetch(async () => ({ status: 403, ok: false, headers: createHeaders(), body: body() })); + await assert.rejects( + () => handlers[METHOD_PROBE_CONNECTIVITY_FULL](ctx), + (e) => e.legacyCode === 'PERMISSION_DENIED', + ); + + withFetch(async () => ({ + status: 200, + ok: true, + headers: createHeaders({ 'content-length': '2048' }), + body: body(), + })); + await assert.rejects(() => invoke({}, ctx), (e) => e.legacyCode === 'RESOURCE_EXHAUSTED'); + assert.equal(cancellations, 3); + + await _test.cancelResponseBody({ body: { cancel: async () => { throw new Error('cancel failed'); } } }); +}); + +test('fetch errors are redacted', async () => { + const ctx = buildCtx({ host: 'https://ips', cookie: 'c=1' }); + withFetch(async () => { throw new Error('https://ips/?cookie=secret'); }); + await assert.rejects(() => invoke({}, ctx), (e) => e.legacyCode === 'UNAVAILABLE' && !e.message.includes('secret')); +}); + +test('timeout uses AbortSignal and insecure TLS uses an undici dispatcher', async () => { + const ctx = buildCtx( + { host: 'https://ips', cookie: 'c=1' }, + { bindings: { skipTlsVerify: true }, limits: { timeoutMs: 5 } }, + ); + withFetch(async (_url, options) => { + assert.ok(options.signal instanceof AbortSignal); + assert.ok(options.dispatcher); + assert.equal('timeoutMs' in options, false); + assert.equal('skipTlsVerify' in options, false); + await new Promise((resolve) => options.signal.addEventListener('abort', resolve, { once: true })); + throw new DOMException('aborted', 'AbortError'); + }); + await assert.rejects(() => invoke({}, ctx), (e) => e.legacyCode === 'UNAVAILABLE'); +}); + +test('valid log page with zero data rows returns empty entries', async () => { + const ctx = buildCtx({ host: 'https://ips', cookie: 'c=1' }); + withFetch(async () => fakeResponse(200, '
名称
')); + const out = await invoke({}, ctx); + assert.equal(out.total, 0); + assert.deepEqual(out.entries, []); +}); + +test('malformed candidate log rows fail closed instead of returning incomplete data', async () => { + const ctx = buildCtx({ host: 'https://ips', cookie: 'c=1' }); + const titledCells = (count, timeIndex) => '' + Array.from( + { length: count }, + (_, i) => `x`, + ).join('') + ''; + for (const row of [titledCells(15, 7), titledCells(13, 6), titledCells(14, 5)]) { + withFetch(async () => fakeResponse(200, `${LOG_PAGE_MARKER}${row}
header
`)); + await assert.rejects(() => invoke({}, ctx), (e) => e.legacyCode === 'FAILED_PRECONDITION'); + } +}); + +test('response size and read failures are bounded and redacted', async () => { + const ctx = buildCtx({ host: 'https://ips', cookie: 'c=1' }, { bindings: { maxResponseBytes: 1024 } }); + withFetch(async () => ({ status: 200, ok: true, headers: createHeaders({ 'content-length': '2048' }), text: async () => 'not read' })); + await assert.rejects(() => invoke({}, ctx), (e) => e.legacyCode === 'RESOURCE_EXHAUSTED'); + + withFetch(async (_url, options) => { + assert.equal(options.redirect, 'manual'); + return fakeResponse(200, 'x'.repeat(1025)); + }); + await assert.rejects(() => invoke({}, ctx), (e) => e.legacyCode === 'RESOURCE_EXHAUSTED'); + + withFetch(async () => ({ + status: 200, ok: true, headers: createHeaders(), + text: async () => { throw new Error('secret response failure'); }, + })); + await assert.rejects(() => invoke({}, ctx), (e) => e.legacyCode === 'UNAVAILABLE' && !e.message.includes('secret')); + + let cancelled = false; + const values = [new Uint8Array(700), new Uint8Array(700)]; + withFetch(async () => ({ + status: 200, ok: true, headers: createHeaders(), + body: { getReader: () => ({ + read: async () => (values.length ? { done: false, value: values.shift() } : { done: true }), + cancel: async () => { cancelled = true; }, + releaseLock: () => {}, + }) }, + })); + await assert.rejects(() => invoke({}, ctx), (e) => e.legacyCode === 'RESOURCE_EXHAUSTED'); + assert.equal(cancelled, true); +}); + +// ---------- service surface + helpers ---------- + +test('service exposes the QueryIpsLog handler', () => { + assert.equal(typeof service.handlers[METHOD_QUERY_IPS_LOG_FULL], 'function'); + assert.equal(typeof service.handlers[METHOD_PROBE_CONNECTIVITY_FULL], 'function'); +}); + +test('helper coverage', () => { + const h = _test; + assert.equal(h.normalizeBaseUrl('https://h/'), 'https://h'); + assert.equal(h.normalizeBaseUrl('ftp://x'), ''); + assert.equal(h.normalizeBaseUrl('https://user:pass@h'), ''); + assert.equal(h.normalizeBaseUrl('https://h/path'), ''); + assert.equal(h.resolveCookie({ session_cookie: 'c' }), 'c'); + assert.equal(h.resolveCookie({ sessionCookie: 'c2' }), 'c2'); + assert.equal(h.decodeEntities('a&b<c>"' d'), 'a&b"\' d'); + assert.equal(h.decodeEntities('&lt; &amp; &quot;'), '< & "'); + assert.equal(h.pickBoolean(true), true); + assert.equal(h.pickBoolean(0), false); + assert.equal(h.pickBoolean(undefined), undefined); + + // rowTitles + parseIpsLog + const row = '#名称X1.1.1.1'; + assert.deepEqual(h.rowTitles(row), ['名称X', '1.1.1.1']); + const html = 'h' + + '#' + Array.from({ length: 14 }, (_, i) => `v${i}`).join('').replace('v6', '2026-01-02 03:04:05') + ''; + const parsed = h.parseIpsLog(html); + assert.equal(parsed.entries.length, 1); + assert.equal(parsed.entries[0].name, 'v0'); + assert.equal(parsed.entries[0].time, '2026-01-02 03:04:05'); + assert.deepEqual({ skipped: parsed.skipped, structuralRows: parsed.structuralRows }, { skipped: 0, structuralRows: 1 }); + // a row without a datetime is skipped + assert.equal(h.parseIpsLog('' + Array.from({ length: 14 }, (_, i) => `x`).join('') + '').skipped, 1); + // Extra/missing titled cells and a datetime in the wrong column must not silently shift fields. + const titledCells = (count, timeIndex) => '' + Array.from( + { length: count }, + (_, i) => `x`, + ).join('') + ''; + assert.equal(h.parseIpsLog(titledCells(15, 7)).skipped, 1); + assert.equal(h.parseIpsLog(titledCells(13, 6)).skipped, 1); + assert.equal(h.parseIpsLog(titledCells(14, 5)).skipped, 1); + // A data row with no recognizable title cells is structural corruption. + const untitled = 'name2026-01-02 03:04:05'; + assert.equal(h.parseIpsLog(untitled).skipped, 1); + // limit + const two = '' + Array.from({ length: 14 }, (_, i) => `a`).join('') + ''; + assert.equal(h.parseIpsLog(two + two, 1).entries.length, 1); + assert.equal(h.parseIpsLog(two + titledCells(13, 6), 1).skipped, 1); + + assert.equal(h.pickInt({ a: '5' }, ['a'], 0), 5); + assert.equal(h.pickInt({ a: '' }, ['a'], 9), 9); + assert.equal(h.pickFirstString([null, '', 'y']), 'y'); + assert.equal(h.pickBoolean('off'), false); + assert.equal(h.pickBoolean('maybe'), undefined); + assert.equal(h.pickFirstBoolean(['x', 'true']), true); + assert.equal(h.unwrapScalar({ value: { value: 2 } }), 2); + assert.deepEqual(h.sanitizeHeaders({ A: 1, '': 2, Cookie: 'bad', 'X-B': 'bad\r\nx: y' }), { A: '1' }); + assert.deepEqual(h.sanitizeHeaders('x'), {}); + assert.ok(h.buildTlsOptions({ skipTlsVerify: true }).dispatcher); + assert.deepEqual(h.buildTlsOptions({}), {}); + assert.equal(h.resolveTimeoutMs({ limits: { timeoutMs: 0 } }), 5000); + assert.equal(h.resolveTimeoutMs({ limits: { timeoutMs: 321 } }), 321); + assert.equal(h.resolveMaxResponseBytes({ bindings: { maxResponseBytes: 2048 } }), 2048); + assert.equal(h.resolveMaxResponseBytes({ bindings: { maxResponseBytes: 1 } }), 2 * 1024 * 1024); + assert.equal(h.grpcCodeFor('NOPE'), grpcStatus.UNKNOWN); + assert.ok(h.errorWithCode('UNAVAILABLE', 'x') instanceof GrpcError); + assert.throws(() => h.throwForHttpStatus(403), (e) => e.legacyCode === 'PERMISSION_DENIED'); + assert.throws(() => h.throwForHttpStatus(400), (e) => e.legacyCode === 'FAILED_PRECONDITION'); + assert.throws(() => h.throwForHttpStatus(500), (e) => e.legacyCode === 'UNAVAILABLE'); + const hdr = h.buildHeaders({ headers: { 'X-A': '1' } }, { instance_id: 'i', request_id: 'r' }, 'c=1'); + assert.equal(hdr.cookie, 'c=1'); + assert.equal(hdr['X-A'], '1'); + assert.deepEqual(h.resolveCallContext({ request: { a: 1 } }).req, { a: 1 }); + assert.deepEqual(h.resolveCallContext({}).req, {}); +}); + +test('rpcdef falls back to ctx.req when called without an argument', async () => { + const mock = await createMockServer(); + try { + const out = await rpcdef(buildCtx(mock, { req: { limit: 1 } }))[QUERY_IPS_LOG_PATH](); + assert.equal(out.total, 1); + } finally { + await mock.close(); + } +});