Skip to content

Commit 8bed69d

Browse files
authored
fix: remove idle-exit loop and add graceful worker swap (#6)
* fix: replace idle-exit with socket health watchdog and graceful worker swap * fix: address PR review for reconnect watchdog and worker swap --------- Co-authored-by: leecoder <leecoder@users.noreply.github.com>
1 parent 0fb86b3 commit 8bed69d

4 files changed

Lines changed: 354 additions & 80 deletions

File tree

dist/plugin.js

Lines changed: 104 additions & 18 deletions
Original file line numberDiff line numberDiff line change
@@ -12431,6 +12431,7 @@ import { spawn } from "child_process";
1243112431
import { appendFileSync, readFileSync, writeFileSync, existsSync } from "fs";
1243212432
import { dirname, join } from "path";
1243312433
import { fileURLToPath } from "url";
12434+
import { homedir } from "os";
1243412435
var LOG_FILE = "/tmp/slack-agent-plugin.log";
1243512436
var SLACK_MSG_LIMIT = 3900;
1243612437
var RESPONSE_STALL_MS = 45e3;
@@ -12441,6 +12442,7 @@ var BG_OUTPUT_CANDIDATE_PATHS = ["/background/output", "/background_output", "/a
1244112442
var BG_TASK_ID_PATTERN = /^bg_[A-Za-z0-9][A-Za-z0-9_-]*$/;
1244212443
var AUTO_ATTACH_MAX_TASKS_PER_MESSAGE = 3;
1244312444
var AUTO_ATTACH_MAX_MONITOR_BUDGET_MS = 12e4;
12445+
var INBOUND_EVENT_DEDUPE_TTL_MS = 10 * 60 * 1e3;
1244412446
var initialized = false;
1244512447
var worker = null;
1244612448
var pluginClient = null;
@@ -12453,8 +12455,14 @@ var allowedUsers = null;
1245312455
var allowlistReady = true;
1245412456
var attachBgTimeoutMs = DEFAULT_ATTACH_TIMEOUT_SEC * 1e3;
1245512457
var cachedBackgroundOutputPath = null;
12458+
var seenInboundEventKeys = /* @__PURE__ */ new Map();
1245612459
var pendingPermissions = /* @__PURE__ */ new Map();
1245712460
var pendingQuestions = /* @__PURE__ */ new Map();
12461+
function expandTilde(p) {
12462+
if (p === "~") return homedir();
12463+
if (p.startsWith("~/")) return join(homedir(), p.slice(2));
12464+
return p;
12465+
}
1245812466
function log(m) {
1245912467
try {
1246012468
appendFileSync(LOG_FILE, `[${(/* @__PURE__ */ new Date()).toISOString()}] plugin: ${m}
@@ -12519,6 +12527,42 @@ function markdownToSlackMrkdwn(text) {
1251912527
function sendIPC(msg) {
1252012528
if (worker && worker.connected) worker.send(msg);
1252112529
}
12530+
function buildInboundEventKey(msg) {
12531+
const eventId = typeof msg?.eventId === "string" ? msg.eventId : "";
12532+
if (eventId) return `event:${eventId}`;
12533+
const channel = typeof msg?.channel === "string" ? msg.channel : "";
12534+
const messageTs = typeof msg?.messageTs === "string" ? msg.messageTs : "";
12535+
if (channel && messageTs) return `msg:${channel}:${messageTs}`;
12536+
return "";
12537+
}
12538+
function inboundMeta(msg) {
12539+
const eventId = typeof msg?.eventId === "string" ? msg.eventId : "-";
12540+
const channel = typeof msg?.channel === "string" ? msg.channel : "-";
12541+
const threadTs = typeof msg?.threadTs === "string" ? msg.threadTs : "-";
12542+
const messageTs = typeof msg?.messageTs === "string" ? msg.messageTs : "-";
12543+
const user = typeof msg?.user === "string" ? msg.user : "-";
12544+
return `event_id=${eventId} channel=${channel} thread_ts=${threadTs} message_ts=${messageTs} user=${user}`;
12545+
}
12546+
function shouldProcessInboundEvent(msg) {
12547+
const key = buildInboundEventKey(msg);
12548+
if (!key) {
12549+
log(`inbound key_missing ${inboundMeta(msg)}`);
12550+
return true;
12551+
}
12552+
if (seenInboundEventKeys.has(key)) {
12553+
log(`inbound duplicate skipped key=${key} ${inboundMeta(msg)}`);
12554+
return false;
12555+
}
12556+
seenInboundEventKeys.set(key, Date.now() + INBOUND_EVENT_DEDUPE_TTL_MS);
12557+
log(`inbound accepted key=${key} ${inboundMeta(msg)}`);
12558+
return true;
12559+
}
12560+
setInterval(() => {
12561+
const now = Date.now();
12562+
for (const [key, expiresAt] of seenInboundEventKeys) {
12563+
if (expiresAt <= now) seenInboundEventKeys.delete(key);
12564+
}
12565+
}, 6e4);
1252212566
function slackSend(channel, text, threadTs) {
1252312567
sendIPC({ type: "slack_send", channel, text: markdownToSlackMrkdwn(text), threadTs });
1252412568
}
@@ -13287,13 +13331,14 @@ async function handleCommand(channel, text, ts) {
1328713331
slackSend(channel, `*\uD604\uC7AC \uC6CC\uD06C\uC2A4\uD398\uC774\uC2A4:* \`${currentDir}\``, ts);
1328813332
return true;
1328913333
}
13334+
const resolvedDir = expandTilde(arg);
1329013335
if (!sessions[ts]) {
13291-
sessions[ts] = { sessionId: "", channel, lastUsed: Date.now(), directory: arg };
13336+
sessions[ts] = { sessionId: "", channel, lastUsed: Date.now(), directory: resolvedDir };
1329213337
} else {
13293-
sessions[ts].directory = arg;
13338+
sessions[ts].directory = resolvedDir;
1329413339
}
1329513340
saveSessions();
13296-
slackSend(channel, `\u2705 \uC6CC\uD06C\uC2A4\uD398\uC774\uC2A4 \uBCC0\uACBD: \`${arg}\`
13341+
slackSend(channel, `\u2705 \uC6CC\uD06C\uC2A4\uD398\uC774\uC2A4 \uBCC0\uACBD: \`${resolvedDir}\`
1329713342
\uB2E4\uC74C \uBA54\uC2DC\uC9C0\uBD80\uD130 \uC774 \uB514\uB809\uD1A0\uB9AC\uC5D0\uC11C \uC138\uC158 \uC0DD\uC131.`, ts);
1329813343
return true;
1329913344
}
@@ -13404,22 +13449,20 @@ ${textParts}`, ts);
1340413449
}
1340513450
return false;
1340613451
}
13407-
function startWorker(env) {
13408-
const workerPath = join(dirname(fileURLToPath(import.meta.url)), "socket-worker.js");
13409-
log(`starting worker: ${workerPath}`);
13410-
worker = spawn("node", [workerPath], {
13411-
env: { ...process.env, ...env },
13412-
stdio: ["ignore", "ignore", "ignore", "ipc"],
13413-
detached: false
13414-
});
13415-
worker.on("message", (msg) => {
13452+
function attachWorkerHandlers(w) {
13453+
w.on("message", (msg) => {
1341613454
if (msg?.type === "slack_event") {
13455+
log(`inbound received ${inboundMeta(msg)}`);
13456+
if (!shouldProcessInboundEvent(msg)) {
13457+
return;
13458+
}
1341713459
const isAllowed = !allowedUsers || !allowlistReady || allowedUsers.has(msg.user);
1341813460
const isThreadReply = msg.threadTs !== msg.messageTs;
1341913461
if (!isAllowed && !isThreadReply) {
13420-
log(`blocked user: ${msg.user} (new thread)`);
13462+
log(`inbound blocked_user ${inboundMeta(msg)} reason=new_thread_not_allowlisted`);
1342113463
return;
1342213464
}
13465+
log(`inbound dispatch_handleMessage ${inboundMeta(msg)}`);
1342313466
handleMessage(msg.channel, msg.text, msg.threadTs, msg.messageTs, isAllowed);
1342413467
} else if (msg?.type === "resolved_emails") {
1342513468
if (allowedUsers && msg.users) {
@@ -13429,18 +13472,61 @@ function startWorker(env) {
1342913472
log(`resolved email users: ${msg.users.join(", ")}`);
1343013473
}
1343113474
allowlistReady = true;
13475+
} else if (msg?.type === "worker_connected") {
1343213476
}
1343313477
});
13434-
worker.on("exit", (code) => {
13478+
}
13479+
var GRACEFUL_RESTART_TIMEOUT_MS = 3e4;
13480+
var workerEnvCache = null;
13481+
function startWorker(env, dyingWorker) {
13482+
workerEnvCache = env;
13483+
const workerPath = join(dirname(fileURLToPath(import.meta.url)), "socket-worker.js");
13484+
log(`starting worker: ${workerPath}`);
13485+
const newWorker = spawn("node", [workerPath], {
13486+
env: { ...process.env, ...env },
13487+
stdio: ["ignore", "ignore", "ignore", "ipc"],
13488+
detached: false
13489+
});
13490+
attachWorkerHandlers(newWorker);
13491+
const oldWorker = dyingWorker ?? null;
13492+
let swapped = false;
13493+
const doSwap = (reason) => {
13494+
if (swapped) return;
13495+
swapped = true;
13496+
worker = newWorker;
13497+
if (oldWorker && !oldWorker.killed) {
13498+
log(`graceful swap: ${reason} \u2014 terminating old worker`);
13499+
oldWorker.removeAllListeners();
13500+
oldWorker.kill("SIGTERM");
13501+
} else {
13502+
log(`worker ready: ${reason} (no old worker to terminate)`);
13503+
}
13504+
};
13505+
const onNewWorkerMessage = (msg) => {
13506+
if (msg?.type === "worker_connected") {
13507+
doSwap("new worker connected");
13508+
}
13509+
};
13510+
newWorker.on("message", onNewWorkerMessage);
13511+
const swapTimer = setTimeout(() => {
13512+
doSwap("timeout \u2014 forcing swap");
13513+
}, GRACEFUL_RESTART_TIMEOUT_MS);
13514+
newWorker.on("exit", (code) => {
13515+
clearTimeout(swapTimer);
1343513516
log(`worker exited: code=${code}`);
13436-
worker = null;
13437-
if (initialized) {
13517+
if (worker === newWorker) {
13518+
worker = null;
13519+
}
13520+
if (initialized && workerEnvCache) {
1343813521
setTimeout(() => {
1343913522
log("restarting worker...");
13440-
startWorker(env);
13523+
startWorker(workerEnvCache, newWorker);
1344113524
}, 5e3);
1344213525
}
1344313526
});
13527+
if (!oldWorker) {
13528+
doSwap("initial start");
13529+
}
1344413530
log("worker spawned");
1344513531
}
1344613532
function stopWorker() {
@@ -13479,7 +13565,7 @@ var pluginModule = {
1347913565
return { tool: { slack_status: slackStatusTool } };
1348013566
}
1348113567
pluginClient = input.client;
13482-
defaultDirectory = options?.DEFAULT_DIRECTORY || process.env.SLACK_DEFAULT_DIRECTORY || input.directory;
13568+
defaultDirectory = expandTilde(options?.DEFAULT_DIRECTORY || process.env.SLACK_DEFAULT_DIRECTORY || input.directory);
1348313569
attachBgTimeoutMs = resolveAttachTimeoutMs(options?.ATTACH_TIMEOUT_SEC);
1348413570
log(`attach timeout set to ${attachBgTimeoutMs}ms`);
1348513571
sessionsPath = join(input.directory, "slack-sessions.json");

dist/socket-worker.js

Lines changed: 101 additions & 19 deletions
Original file line numberDiff line numberDiff line change
@@ -28,6 +28,38 @@ const slack = new WebClient(botToken, agent ? { agent, tls: { ca: agent.options.
2828
const socketClient = new SocketModeClient({ appToken });
2929

3030
let botUserId;
31+
const EVENT_DEDUPE_TTL_MS = 10 * 60 * 1000;
32+
const seenEventKeys = new Map();
33+
34+
function formatInboundMeta(body, ev) {
35+
const eventId = typeof body?.event_id === "string" ? body.event_id : "-";
36+
const eventType = typeof ev?.type === "string" ? ev.type : "-";
37+
const channel = typeof ev?.channel === "string" ? ev.channel : "-";
38+
const ts = typeof ev?.ts === "string" ? ev.ts : "-";
39+
const threadTs = typeof ev?.thread_ts === "string" ? ev.thread_ts : "-";
40+
const user = typeof ev?.user === "string" ? ev.user : "-";
41+
return `event_id=${eventId} type=${eventType} channel=${channel} ts=${ts} thread_ts=${threadTs} user=${user}`;
42+
}
43+
44+
function buildEventKey(body, ev) {
45+
const channel = typeof ev?.channel === "string" ? ev.channel : "";
46+
const ts = typeof ev?.ts === "string" ? ev.ts : "";
47+
if (channel && ts) return `msg:${channel}:${ts}`;
48+
const eventId = typeof body?.event_id === "string" ? body.event_id : "";
49+
if (eventId) return `event:${eventId}`;
50+
return "";
51+
}
52+
53+
function shouldProcessEvent(eventKey) {
54+
if (!eventKey) return true;
55+
const now = Date.now();
56+
for (const [key, expiresAt] of seenEventKeys) {
57+
if (expiresAt <= now) seenEventKeys.delete(key);
58+
}
59+
if (seenEventKeys.has(eventKey)) return false;
60+
seenEventKeys.set(eventKey, now + EVENT_DEDUPE_TTL_MS);
61+
return true;
62+
}
3163

3264
async function init() {
3365
const auth = await slack.auth.test();
@@ -106,42 +138,92 @@ process.on("message", async (msg) => {
106138
});
107139

108140
socketClient.on("slack_event", async ({ body, ack }) => {
109-
if (ack) await ack();
110141
const ev = body?.event;
142+
log(`ingress received ${formatInboundMeta(body, ev)}`);
143+
144+
if (ack) {
145+
try {
146+
await ack();
147+
log(`ingress ack sent ${formatInboundMeta(body, ev)}`);
148+
} catch (e) {
149+
log(`ingress ack failed ${formatInboundMeta(body, ev)} err=${e?.message || String(e)}`);
150+
}
151+
}
152+
111153
if (!ev) return;
112154

155+
const eventKey = buildEventKey(body, ev);
156+
if (!shouldProcessEvent(eventKey)) {
157+
log(`ingress duplicate skipped key=${eventKey} ${formatInboundMeta(body, ev)}`);
158+
return;
159+
}
160+
113161
if (ev.type === "message" || ev.type === "app_mention") {
114-
if (ev.bot_id || ev.user === botUserId) return;
115-
if (ev.subtype && ev.subtype !== "file_share") return;
116-
if (!ev.channel || !ev.text || !ev.ts) return;
162+
if (ev.bot_id || ev.user === botUserId) {
163+
log(`ingress dropped reason=self_or_bot ${formatInboundMeta(body, ev)}`);
164+
return;
165+
}
166+
if (ev.subtype && ev.subtype !== "file_share") {
167+
log(`ingress dropped reason=subtype_${ev.subtype} ${formatInboundMeta(body, ev)}`);
168+
return;
169+
}
170+
if (!ev.channel || !ev.text || !ev.ts) {
171+
log(`ingress dropped reason=missing_required_fields ${formatInboundMeta(body, ev)}`);
172+
return;
173+
}
117174

118175
const text = ev.type === "app_mention"
119176
? ev.text.replace(/<@[A-Z0-9]+>/g, "").trim()
120177
: ev.text;
121178

122179
if (text && process.send) {
123-
process.send({ type: "slack_event", channel: ev.channel, text, ts: ev.ts, threadTs: ev.thread_ts || ev.ts, messageTs: ev.ts, user: ev.user });
124-
log(`event sent via IPC: ${text.slice(0, 50)}`);
180+
process.send({
181+
type: "slack_event",
182+
channel: ev.channel,
183+
text,
184+
ts: ev.ts,
185+
threadTs: ev.thread_ts || ev.ts,
186+
messageTs: ev.ts,
187+
eventId: typeof body?.event_id === "string" ? body.event_id : undefined,
188+
user: ev.user,
189+
});
190+
log(`ingress forwarded_to_plugin key=${eventKey || "-"} ${formatInboundMeta(body, ev)} text_preview=${text.slice(0, 50)}`);
125191
}
126192
}
127193
});
128194

129-
socketClient.on("connected", () => log("Socket Mode connected"));
130-
socketClient.on("disconnected", () => log("Socket Mode disconnected"));
195+
// --- Connection health monitoring ---
196+
// SDK handles ping/pong internally (serverPingTimeout=30s, clientPingTimeout=5s).
197+
// We only exit if SDK fires 'disconnected' AND fails to reconnect within RECONNECT_GRACE_MS.
198+
const RECONNECT_GRACE_MS = 60000;
199+
let outageStartedAt = null;
131200

132-
await init();
133-
await socketClient.start();
134-
log("worker running");
201+
socketClient.on("connected", () => {
202+
outageStartedAt = null;
203+
log("Socket Mode connected");
204+
if (process.send) process.send({ type: "worker_connected" });
205+
});
135206

136-
const STALE_TIMEOUT = 120000;
137-
let lastEventTime = Date.now();
207+
socketClient.on("disconnected", () => {
208+
if (!outageStartedAt) outageStartedAt = Date.now();
209+
log("Socket Mode disconnected — waiting for SDK auto-reconnect");
210+
});
138211

139-
socketClient.on("slack_event", () => { lastEventTime = Date.now(); });
212+
socketClient.on("reconnecting", () => {
213+
if (!outageStartedAt) outageStartedAt = Date.now();
214+
log("Socket Mode reconnecting...");
215+
});
216+
217+
await init();
218+
await socketClient.start();
219+
log("worker running (idle-exit disabled, relying on SDK ping/pong health check)");
140220

141221
setInterval(() => {
142-
const elapsed = Date.now() - lastEventTime;
143-
if (elapsed > STALE_TIMEOUT) {
144-
log(`no events for ${Math.round(elapsed / 1000)}s, exiting for restart`);
145-
process.exit(1);
222+
if (outageStartedAt) {
223+
const elapsed = Date.now() - outageStartedAt;
224+
if (elapsed > RECONNECT_GRACE_MS) {
225+
log(`connection unhealthy for ${Math.round(elapsed / 1000)}s with no reconnect — exiting for restart`);
226+
process.exit(1);
227+
}
146228
}
147-
}, 30000);
229+
}, 15000);

0 commit comments

Comments
 (0)