feat(insights): correlate CSP vulnerabilities/misconfigurations with entity store

src/commands/misc/index.ts

@@ -34,7 +34,7 @@ export const miscCommands: CommandModule = {
       .option('--v2', 'generate v2 ML anomaly data with user.id, host.id, and event.module fields')
       .option(
         '--correlate-with-entity-store',
-        'correlate generated anomaly data with entities from the entity store',
+        'correlate generated anomaly, vulnerability and misconfiguration data with entities from the entity store',
       )
       .description(
         'Generate vulnerabilities, misconfigurations, ML jobs, and anomalous behavior for entities.',

src/commands/misc/insights.ts

@@ -10,6 +10,7 @@ import createMisconfigurations, {
 } from '../../generators/create_misconfigurations.ts';
 import { installPackage } from '../../utils/kibana_api.ts';
 import { generateAnomalousBehaviorDataWithMlJobs } from './anomalous_behavior/index.ts';
+import { fetchHostIdentitiesForDed, fetchUserIdentitiesForDed } from '../utils/entity_store.ts';
 
 const VULNERABILITY_INDEX_NAME = 'logs-cloud_security_posture.vulnerabilities_latest-default';
 
@@ -18,6 +19,16 @@ const MISCONFIGURATION_INDEX_NAME =
 
 const PACKAGE_TO_INSTALL = 'cloud_security_posture';
 
+// Mirror the cap the anomaly (DED) correlation uses so all correlated data is drawn from
+// the same bounded slice of the entity store.
+const ENTITY_STORE_CORRELATION_MAX = 10;
+
+interface EntityData {
+  username?: string;
+  hostname?: string;
+  hostId?: string;
+}
+
 interface GenerateAiInsightsOpts {
   users: number;
   hosts: number;
@@ -29,6 +40,42 @@ interface GenerateAiInsightsOpts {
   seed?: number;
   correlateWithEntityStore?: boolean;
 }
+
+// Faked host/user names — the default source for vulnerability + misconfiguration docs.
+const buildFakedInsightEntities = (
+  users: number,
+  hosts: number,
+): { usersData: EntityData[]; hostsData: EntityData[] } => ({
+  usersData: Array.from({ length: users }, () => ({ username: faker.internet.username() })),
+  hostsData: Array.from({ length: hosts }, () => ({ hostname: faker.internet.domainName() })),
+});
+
+// Reuse the same entity-store identities the anomaly (DED) correlation uses so the CSP docs
+// land on real entities. For hosts we carry `host.id` through so the vulnerability/misconfig
+// doc's computed EUID (`host:<host.id>`) equals the host entity's `entity.id` — that is how
+// the v2 entity flyout / AI summary matches vulnerabilities (host-only, EUID-based). Falls
+// back to `host.name` when the entity has no id. Returns empty arrays when the store has no
+// usable identities.
+const buildCorrelatedInsightEntities = async (
+  space: string,
+): Promise<{ usersData: EntityData[]; hostsData: EntityData[] }> => {
+  const [hostIdentities, userIdentities] = await Promise.all([
+    fetchHostIdentitiesForDed(space, ENTITY_STORE_CORRELATION_MAX),
+    fetchUserIdentitiesForDed(space, ENTITY_STORE_CORRELATION_MAX),
+  ]);
+
+  const hostsData = hostIdentities
+    .filter((host) => Boolean(host.name || host.id))
+    .map((host) => ({ hostname: host.name, hostId: host.id }));
+
+  const usersData = userIdentities
+    .map((user) => user.ecsArrays['user.name']?.[0] ?? user.displayLabel)
+    .filter((username): username is string => Boolean(username))
+    .map((username) => ({ username }));
+
+  return { usersData, hostsData };
+};
+
 export const generateAiInsights = async ({
   users,
   hosts,
@@ -41,13 +88,20 @@ export const generateAiInsights = async ({
   correlateWithEntityStore = false,
 }: GenerateAiInsightsOpts) => {
   faker.seed(seed);
-  const usersData = Array.from({ length: users }, () => ({
-    username: faker.internet.username(),
-  }));
 
-  const hostsData = Array.from({ length: hosts }, () => ({
-    hostname: faker.internet.domainName(),
-  }));
+  let { usersData, hostsData } = buildFakedInsightEntities(users, hosts);
+
+  if (correlateWithEntityStore) {
+    log.info('Correlating vulnerabilities and misconfigurations with entity-store entities');
+    const correlated = await buildCorrelatedInsightEntities(space);
+    if (correlated.hostsData.length === 0 && correlated.usersData.length === 0) {
+      log.warn(
+        'No entity-store host/user identities found to correlate against; falling back to generated names. Populate the entity store first (e.g. `risk-score-v2`).',
+      );
+    } else {
+      ({ usersData, hostsData } = correlated);
+    }
+  }
 
   log.info('Installing cloud posture package');
   await installPackage({ packageName: PACKAGE_TO_INSTALL, space });
@@ -78,11 +132,6 @@ export const generateAiInsights = async ({
   }
 };
 
-interface EntityData {
-  username?: string;
-  hostname?: string;
-}
-
 export const generateDocs = (
   entityData: EntityData[],
   space: string,

src/generators/create_misconfigurations.ts

@@ -4,12 +4,14 @@ import dayjs from 'dayjs';
 export interface CreateMisconfigurationsParams {
   username?: string;
   hostname?: string;
+  hostId?: string;
   space?: string;
 }
 
 export default function createMisconfigurations({
   username = 'user-1',
   hostname = 'host-1',
+  hostId,
   space = 'default',
 }: CreateMisconfigurationsParams) {
   const now = dayjs().format('YYYY-MM-DDTHH:mm:ss.SSSZ');
@@ -160,6 +162,9 @@ export default function createMisconfigurations({
     },
     host: {
       name: hostname,
+      // host.id drives the entity EUID (host:<id>) that the v2 entity flyout / AI summary
+      // matches findings on; only set it when correlating to a real entity.
+      ...(hostId ? { id: hostId } : {}),
     },
   };
 }

src/generators/create_vulnerability.ts

@@ -4,11 +4,13 @@ import { faker } from '@faker-js/faker';
 export interface CreateVulnerabilitiesParams {
   username?: string;
   hostname?: string;
+  hostId?: string;
   space?: string;
 }
 export default function createVulnerabilities({
   username = 'user-1',
   hostname = 'host-1',
+  hostId,
   space = 'default',
 }: CreateVulnerabilitiesParams) {
   const now = dayjs().format('YYYY-MM-DDTHH:mm:ss.SSSZ');
@@ -105,6 +107,9 @@ export default function createVulnerabilities({
     host: {
       architecture: 'x86_64',
       name: hostname,
+      // host.id drives the entity EUID (host:<id>) that the v2 entity flyout / AI summary
+      // matches vulnerabilities on; only set it when correlating to a real entity.
+      ...(hostId ? { id: hostId } : {}),
       os: {
         platform: 'Linux/UNIX',
       },