🤖 fix: line-buffered filtering for bash_output filter_exclude (#1101)

When using `filter_exclude` with `bash_output`, the filter was
incorrectly matching line fragments instead of complete lines. This
caused:

1. Partial line fragments (like `)' from '`⏳ Still waiting... (1
pending)`) to not match the filter pattern and trigger early return
2. Agents polling repeatedly because each call returned tiny fragments

## Root Cause

From the `runtime-16ps` chat history analysis:
- The script outputs `⏳ Still waiting... (X pending, Y failed)` every 15
seconds
- Each `bash_output` call reads incrementally from where the last call
stopped
- Previous calls consumed most of the line, leaving only `)\n`
- The filter `"Still waiting"` couldn't match the fragment `")"`
- So `)\n` passed through as "meaningful output" and triggered early
return

## Changes

- Track `incompleteLineBuffer` per process to buffer partial lines
across calls
- Only apply filter to complete lines (ending with newline)
- On process exit, flush any remaining buffered content
- Add 'better pattern' note when `filter_exclude` is used but still
polling frequently

## Tests

Added 3 new tests for line-buffered filtering:
- `should only filter complete lines, not fragments`
- `should buffer incomplete lines across calls`
- `should include incomplete line on process exit`

Updated existing test to verify new polling detection for filter_exclude
users.

---

_Generated with `mux`_

Author: ammar-agent

src/node/services/backgroundProcessManager.test.ts

@@ -871,6 +871,75 @@ describe("BackgroundProcessManager", () => {
     });
   });
 
+  describe("line-buffered filtering", () => {
+    it("should only filter complete lines, not fragments", async () => {
+      // Process that outputs lines that should be filtered and one that shouldn't
+      const result = await manager.spawn(
+        runtime,
+        testWorkspaceId,
+        // Output lines: some with 'progress', one without
+        "echo 'progress 1'; echo 'progress 2'; echo 'FINAL RESULT'",
+        { cwd: process.cwd(), displayName: "test" }
+      );
+
+      expect(result.success).toBe(true);
+      if (!result.success) return;
+
+      // Wait for process to complete
+      await new Promise((resolve) => setTimeout(resolve, 300));
+
+      // Filter out lines containing 'progress', should only get 'FINAL RESULT'
+      const output = await manager.getOutput(result.processId, "progress", true, 0.5);
+      expect(output.success).toBe(true);
+      if (!output.success) return;
+      expect(output.output).toContain("FINAL RESULT");
+      expect(output.output).not.toContain("progress");
+    });
+
+    it("should buffer incomplete lines across calls", async () => {
+      // Process that outputs progress lines
+      const result = await manager.spawn(
+        runtime,
+        testWorkspaceId,
+        "echo 'progress: 50%'; sleep 0.1; echo 'progress: 100%'; echo 'DONE'",
+        { cwd: process.cwd(), displayName: "test" }
+      );
+
+      expect(result.success).toBe(true);
+      if (!result.success) return;
+
+      // Wait for process to complete
+      await new Promise((resolve) => setTimeout(resolve, 500));
+
+      // Filter out progress lines, should only get 'DONE'
+      const output = await manager.getOutput(result.processId, "progress", true, 0.5);
+      expect(output.success).toBe(true);
+      if (!output.success) return;
+      expect(output.output).toContain("DONE");
+      expect(output.output).not.toContain("progress");
+    });
+
+    it("should include incomplete line on process exit", async () => {
+      // Process that exits without final newline
+      const result = await manager.spawn(runtime, testWorkspaceId, "printf 'no newline at end'", {
+        cwd: process.cwd(),
+        displayName: "test",
+      });
+
+      expect(result.success).toBe(true);
+      if (!result.success) return;
+
+      // Wait for process to exit
+      await new Promise((resolve) => setTimeout(resolve, 300));
+
+      const output = await manager.getOutput(result.processId, undefined, undefined, 0.5);
+      expect(output.success).toBe(true);
+      if (!output.success) return;
+      expect(output.output).toContain("no newline at end");
+      expect(output.status).not.toBe("running");
+    });
+  });
+
   describe("polling detection", () => {
     it("should return note after 3+ calls without filter_exclude on running process", async () => {
       // Long-running process
@@ -903,7 +972,7 @@ describe("BackgroundProcessManager", () => {
       expect(output3.note).toContain("3+ times");
     });
 
-    it("should NOT return note when filter_exclude is already used", async () => {
+    it("should return better pattern note when filter_exclude is used but still polling", async () => {
       const result = await manager.spawn(
         runtime,
         testWorkspaceId,
@@ -914,14 +983,17 @@ describe("BackgroundProcessManager", () => {
       expect(result.success).toBe(true);
       if (!result.success) return;
 
-      // Make 3+ calls with filter_exclude
+      // Make 3+ calls with filter_exclude - should get "better pattern" note
+      let lastNote: string | undefined;
       for (let i = 0; i < 4; i++) {
         const output = await manager.getOutput(result.processId, "nomatch", true, 0.1);
         expect(output.success).toBe(true);
         if (!output.success) return;
-        // Should never get the note since we're using filter_exclude
-        expect(output.note).toBeUndefined();
+        lastNote = output.note;
       }
+      // Should get the "better pattern" note since we're using filter_exclude but still polling
+      expect(lastNote).toContain("filter_exclude but still polling");
+      expect(lastNote).toContain("broader pattern");
     });
 
     it("should NOT return note when process has exited", async () => {

src/node/services/backgroundProcessManager.ts

@@ -46,6 +46,8 @@ export interface BackgroundProcess {
   outputLock: AsyncMutex;
   /** Tracks how many times getOutput() has been called (for polling detection) */
   getOutputCallCount: number;
+  /** Buffer for incomplete lines (no trailing newline) from previous read */
+  incompleteLineBuffer: string;
 }
 
 /**
@@ -222,6 +224,7 @@ export class BackgroundProcessManager extends EventEmitter<BackgroundProcessMana
       outputBytesRead: 0,
       outputLock: new AsyncMutex(),
       getOutputCallCount: 0,
+      incompleteLineBuffer: "",
     };
 
     // Store process in map
@@ -331,6 +334,7 @@ export class BackgroundProcessManager extends EventEmitter<BackgroundProcessMana
       outputBytesRead: 0,
       outputLock: new AsyncMutex(),
       getOutputCallCount: 0,
+      incompleteLineBuffer: "",
     };
 
     // Store process in map
@@ -504,10 +508,10 @@ export class BackgroundProcessManager extends EventEmitter<BackgroundProcessMana
       }
     }
 
-    // Apply filtering to output, returns filtered result
-    const applyFilter = (raw: string): string => {
-      if (!filterRegex) return raw;
-      const lines = raw.split("\n");
+    // Apply filtering to complete lines only
+    // Incomplete line fragments (no trailing newline) are kept in buffer for next read
+    const applyFilter = (lines: string[]): string => {
+      if (!filterRegex) return lines.join("\n");
       const filtered = filterExclude
         ? lines.filter((line) => !filterRegex.test(line))
         : lines.filter((line) => filterRegex.test(line));
@@ -521,6 +525,9 @@ export class BackgroundProcessManager extends EventEmitter<BackgroundProcessMana
     let accumulatedRaw = "";
     let currentStatus = proc.status;
 
+    // Track the previous buffer to prepend to accumulated output
+    const previousBuffer = proc.incompleteLineBuffer;
+
     while (true) {
       // Read new content via the handle (works for both local and SSH runtimes)
       // Output is already unified in output.log (stdout + stderr via 2>&1)
@@ -534,16 +541,25 @@ export class BackgroundProcessManager extends EventEmitter<BackgroundProcessMana
       const refreshedProc = await this.getProcess(processId);
       currentStatus = refreshedProc?.status ?? proc.status;
 
+      // Line-buffered filtering: prepend incomplete line from previous call
+      const rawWithBuffer = previousBuffer + accumulatedRaw;
+      const allLines = rawWithBuffer.split("\n");
+
+      // Last element is incomplete if content doesn't end with newline
+      const hasTrailingNewline = rawWithBuffer.endsWith("\n");
+      const completeLines = hasTrailingNewline ? allLines.slice(0, -1) : allLines.slice(0, -1);
+      const incompleteLine = hasTrailingNewline ? "" : allLines[allLines.length - 1];
+
       // When using filter_exclude, check if we have meaningful (non-excluded) output
-      // If all new output matches the exclusion pattern, keep waiting
-      const filteredOutput = applyFilter(accumulatedRaw);
+      // Only consider complete lines for filtering - fragments can't match patterns
+      const filteredOutput = applyFilter(completeLines);
       const hasMeaningfulOutput = filterExclude
         ? filteredOutput.trim().length > 0
-        : accumulatedRaw.length > 0;
+        : completeLines.length > 0 || incompleteLine.length > 0;
 
       // Return immediately if:
       // 1. We have meaningful output (after filtering if filter_exclude is set)
-      // 2. Process is no longer running (exited/killed/failed)
+      // 2. Process is no longer running (exited/killed/failed) - flush buffer
       // 3. Timeout elapsed
       // 4. Abort signal received (user sent a new message)
       if (hasMeaningfulOutput || currentStatus !== "running") {
@@ -569,14 +585,52 @@ export class BackgroundProcessManager extends EventEmitter<BackgroundProcessMana
       await new Promise((resolve) => setTimeout(resolve, pollIntervalMs));
     }
 
-    log.debug(`BackgroundProcessManager.getOutput: read rawLen=${accumulatedRaw.length}`);
+    // Final line processing with buffer from previous call
+    const rawWithBuffer = previousBuffer + accumulatedRaw;
+    const allLines = rawWithBuffer.split("\n");
+    const hasTrailingNewline = rawWithBuffer.endsWith("\n");
+
+    // On process exit, include incomplete line; otherwise keep it buffered
+    const linesToReturn =
+      currentStatus !== "running"
+        ? allLines.filter((l) => l.length > 0) // Include all non-empty lines on exit
+        : hasTrailingNewline
+          ? allLines.slice(0, -1)
+          : allLines.slice(0, -1);
+
+    // Update buffer for next call (clear on exit, keep incomplete line otherwise)
+    proc.incompleteLineBuffer =
+      currentStatus === "running" && !hasTrailingNewline ? allLines[allLines.length - 1] : "";
 
-    const filteredOutput = applyFilter(accumulatedRaw);
+    log.debug(
+      `BackgroundProcessManager.getOutput: read rawLen=${accumulatedRaw.length}, completeLines=${linesToReturn.length}`
+    );
+
+    const filteredOutput = applyFilter(linesToReturn);
 
     // Suggest filter_exclude if polling too frequently on a running process
     const shouldSuggestFilterExclude =
       callCount >= 3 && !filterExclude && currentStatus === "running";
 
+    // Suggest better pattern if using filter_exclude but still polling frequently
+    const shouldSuggestBetterPattern =
+      callCount >= 3 && filterExclude && currentStatus === "running";
+
+    let note: string | undefined;
+    if (shouldSuggestFilterExclude) {
+      note =
+        "STOP POLLING. You've called bash_output 3+ times on this process. " +
+        "This wastes tokens and clutters the conversation. " +
+        "Instead, make ONE call with: filter='⏳|progress|waiting|\\\\\\.\\\\\\.\\\\\\.', " +
+        "filter_exclude=true, timeout_secs=120. This blocks until meaningful output arrives.";
+    } else if (shouldSuggestBetterPattern) {
+      note =
+        "You're using filter_exclude but still polling frequently. " +
+        "Your filter pattern may not be matching the actual output. " +
+        "Try a broader pattern like: filter='\\\\.|\\\\d+%|running|progress|pending|⏳|waiting'. " +
+        "Wait for the FULL timeout before checking again.";
+    }
+
     return {
       success: true,
       status: currentStatus,
@@ -586,12 +640,7 @@ export class BackgroundProcessManager extends EventEmitter<BackgroundProcessMana
           ? ((await this.getProcess(processId))?.exitCode ?? undefined)
           : undefined,
       elapsed_ms: Date.now() - startTime,
-      note: shouldSuggestFilterExclude
-        ? "STOP POLLING. You've called bash_output 3+ times on this process. " +
-          "This wastes tokens and clutters the conversation. " +
-          "Instead, make ONE call with: filter='⏳|progress|waiting|\\\\.\\\\.\\\\.', " +
-          "filter_exclude=true, timeout_secs=120. This blocks until meaningful output arrives."
-        : undefined,
+      note,
     };
   }