fix: redact gh auth output in code queue
This commit is contained in:
File diff suppressed because one or more lines are too long
@@ -0,0 +1,121 @@
|
||||
import { mkdtempSync, readFileSync, rmSync } from "node:fs";
|
||||
import { tmpdir } from "node:os";
|
||||
import { join } from "node:path";
|
||||
import { appendOutput, configureTaskOutput, taskFullOutput } from "../src/components/microservices/code-queue/src/task-output";
|
||||
import { sanitizeTaskOutputText } from "../src/components/microservices/code-queue/src/output-redaction";
|
||||
import type { JsonValue, QueueTask } from "../src/components/microservices/code-queue/src/types";
|
||||
|
||||
type JsonRecord = Record<string, unknown>;
|
||||
|
||||
function assertCondition(condition: unknown, message: string, detail: unknown = {}): void {
|
||||
if (!condition) throw new Error(`${message}: ${JSON.stringify(detail)}`);
|
||||
}
|
||||
|
||||
function fixtureTask(): QueueTask {
|
||||
const at = "2026-05-23T00:00:00.000Z";
|
||||
return {
|
||||
id: "codex_gh_auth_redaction_contract",
|
||||
queueId: "default",
|
||||
queueEnteredAt: at,
|
||||
prompt: "redaction fixture",
|
||||
basePrompt: "redaction fixture",
|
||||
referenceTaskIds: [],
|
||||
referenceInjection: null,
|
||||
providerId: "D601",
|
||||
cwd: "/workspace",
|
||||
model: "gpt-5.5",
|
||||
reasoningEffort: null,
|
||||
executionMode: "default",
|
||||
maxAttempts: 1,
|
||||
status: "running",
|
||||
createdAt: at,
|
||||
updatedAt: at,
|
||||
startedAt: at,
|
||||
finishedAt: null,
|
||||
readAt: null,
|
||||
currentAttempt: 1,
|
||||
currentMode: "initial",
|
||||
codexThreadId: null,
|
||||
activeTurnId: null,
|
||||
finalResponse: "",
|
||||
lastError: null,
|
||||
lastJudge: null,
|
||||
judgeFailCount: 0,
|
||||
promptHistory: [],
|
||||
output: [],
|
||||
events: [],
|
||||
attempts: [],
|
||||
cancelRequested: false,
|
||||
nextPrompt: null,
|
||||
nextMode: null,
|
||||
};
|
||||
}
|
||||
|
||||
function assertNoTokenFragments(value: string, label: string): void {
|
||||
assertCondition(!/\bgh[pousr]_[A-Za-z0-9_]{6,}\b/u.test(value), `${label} must redact gh token-like values`, value);
|
||||
assertCondition(!/\bgithub_pat_[A-Za-z0-9_]{6,}\b/u.test(value), `${label} must redact GitHub PAT-like values`, value);
|
||||
assertCondition(!/Token:\s*\S+/iu.test(value), `${label} must not expose raw gh auth token lines`, value);
|
||||
assertCondition(!/Token scopes?:\s*\S+/iu.test(value), `${label} must not expose raw gh auth scope lines`, value);
|
||||
}
|
||||
|
||||
export function runCodeQueueGhAuthRedactionContract(): JsonRecord {
|
||||
const tmp = mkdtempSync(join(tmpdir(), "code-queue-gh-auth-redaction-"));
|
||||
const task = fixtureTask();
|
||||
let seq = 0;
|
||||
try {
|
||||
configureTaskOutput({
|
||||
config: { maxInMemoryOutputRecords: 1000, outputArchiveDir: tmp },
|
||||
allocateSeq: () => {
|
||||
seq += 1;
|
||||
return seq;
|
||||
},
|
||||
errorToJson: (error: unknown): JsonValue => error instanceof Error ? { message: error.message } : String(error),
|
||||
logger: () => undefined,
|
||||
markTaskDirty: () => undefined,
|
||||
nowIso: () => "2026-05-23T00:00:01.000Z",
|
||||
schedulePersistState: () => undefined,
|
||||
});
|
||||
|
||||
const rawGhAuth = [
|
||||
"github.com",
|
||||
" \u2713 Logged in to github.com account example (keyring)",
|
||||
" - Active account: true",
|
||||
" - Git operations protocol: ssh",
|
||||
" - Token: ghp_abcdef1234567890abcdef1234567890",
|
||||
" - Token scopes: 'repo', 'read:org'",
|
||||
"generic token=github_pat_abcdef1234567890abcdef1234567890",
|
||||
].join("\n");
|
||||
const sanitized = sanitizeTaskOutputText(rawGhAuth);
|
||||
assertNoTokenFragments(sanitized, "direct sanitizer output");
|
||||
assertCondition(sanitized.includes("[redacted gh auth status line]"), "sanitizer must redact gh auth status lines", sanitized);
|
||||
assertCondition(sanitized.includes("bun scripts/cli.ts gh auth status"), "sanitizer must include UniDesk gh wrapper hint", sanitized);
|
||||
|
||||
appendOutput(task, "command", `${rawGhAuth}\n`, "item/commandExecution/outputDelta", "call-gh-auth", true);
|
||||
const retained = JSON.stringify(task.output);
|
||||
const archived = readFileSync(join(tmp, `${task.id}.jsonl`), "utf8");
|
||||
const full = JSON.stringify(taskFullOutput(task));
|
||||
assertNoTokenFragments(retained, "retained output");
|
||||
assertNoTokenFragments(archived, "archived output");
|
||||
assertNoTokenFragments(full, "full output replay");
|
||||
assertCondition(full.includes("bun scripts/cli.ts gh auth status"), "full output replay must keep wrapper hint", full);
|
||||
|
||||
return {
|
||||
ok: true,
|
||||
checks: [
|
||||
"raw gh auth status token and scope lines are redacted",
|
||||
"token-like GitHub values are redacted before retained output persistence",
|
||||
"output archive replay remains redacted",
|
||||
"redacted output nudges runner toward bun scripts/cli.ts gh auth status",
|
||||
],
|
||||
};
|
||||
} finally {
|
||||
rmSync(tmp, { recursive: true, force: true });
|
||||
}
|
||||
}
|
||||
|
||||
try {
|
||||
process.stdout.write(`${JSON.stringify(runCodeQueueGhAuthRedactionContract(), null, 2)}\n`);
|
||||
} catch (error) {
|
||||
process.stderr.write(`${error instanceof Error ? error.stack ?? error.message : String(error)}\n`);
|
||||
process.exit(1);
|
||||
}
|
||||
@@ -33,6 +33,7 @@ const syntaxFiles = [
|
||||
"scripts/code-queue-cli-disclosure-contract-test.ts",
|
||||
"scripts/code-queue-cli-steer-test.ts",
|
||||
"scripts/code-queue-cli-submit-prompt-contract-test.ts",
|
||||
"scripts/code-queue-gh-auth-redaction-contract-test.ts",
|
||||
"scripts/code-queue-supervisor-disclosure-contract-test.ts",
|
||||
"src/components/frontend/src/index.ts",
|
||||
"src/components/frontend/src/app.tsx",
|
||||
@@ -310,6 +311,7 @@ export function runChecks(config: UniDeskConfig, options: CheckOptions = default
|
||||
fileItem("scripts/code-queue-cli-disclosure-contract-test.ts"),
|
||||
fileItem("scripts/code-queue-cli-steer-test.ts"),
|
||||
fileItem("scripts/code-queue-submit-routing-contract-test.ts"),
|
||||
fileItem("scripts/code-queue-gh-auth-redaction-contract-test.ts"),
|
||||
fileItem("scripts/code-queue-supervisor-disclosure-contract-test.ts"),
|
||||
fileItem("scripts/host-codex-commander-skeleton-contract-test.ts"),
|
||||
fileItem("scripts/host-codex-commander-no-daemon-smoke-contract-test.ts"),
|
||||
@@ -350,6 +352,7 @@ export function runChecks(config: UniDeskConfig, options: CheckOptions = default
|
||||
items.push(commandItem("code-queue:cli-steer-contract", ["bun", "scripts/code-queue-cli-steer-test.ts"], 30_000));
|
||||
items.push(commandItem("code-queue:submit-prompt-contract", ["bun", "scripts/code-queue-cli-submit-prompt-contract-test.ts"], 30_000));
|
||||
items.push(commandItem("code-queue:submit-routing-contract", ["bun", "scripts/code-queue-submit-routing-contract-test.ts"], 30_000));
|
||||
items.push(commandItem("code-queue:gh-auth-redaction-contract", ["bun", "scripts/code-queue-gh-auth-redaction-contract-test.ts"], 30_000));
|
||||
items.push(commandItem("code-queue:supervisor-disclosure-contract", ["bun", "scripts/code-queue-supervisor-disclosure-contract-test.ts"], 30_000));
|
||||
items.push(commandItem("host-codex-commander:skeleton-contract", ["bun", "scripts/host-codex-commander-skeleton-contract-test.ts"], 30_000));
|
||||
items.push(commandItem("host-codex-commander:no-daemon-smoke-contract", ["bun", "scripts/host-codex-commander-no-daemon-smoke-contract-test.ts"], 30_000));
|
||||
@@ -379,6 +382,7 @@ export function runChecks(config: UniDeskConfig, options: CheckOptions = default
|
||||
items.push(skippedItem("code-queue:cli-steer-contract", "Code Queue steer CLI contract is opt-in with script checks", "--scripts-typecheck or --full"));
|
||||
items.push(skippedItem("code-queue:submit-prompt-contract", "Code Queue submit prompt contract is opt-in with script checks", "--scripts-typecheck or --full"));
|
||||
items.push(skippedItem("code-queue:submit-routing-contract", "Code Queue submit routing contract is opt-in with script checks", "--scripts-typecheck or --full"));
|
||||
items.push(skippedItem("code-queue:gh-auth-redaction-contract", "Code Queue GitHub auth output redaction contract is opt-in with script checks", "--scripts-typecheck or --full"));
|
||||
items.push(skippedItem("code-queue:supervisor-disclosure-contract", "Code Queue supervisor disclosure contract is opt-in with script checks", "--scripts-typecheck or --full"));
|
||||
items.push(skippedItem("host-codex-commander:skeleton-contract", "host Codex commander skeleton contract is opt-in with script checks", "--scripts-typecheck or --full"));
|
||||
items.push(skippedItem("host-codex-commander:no-daemon-smoke-contract", "host Codex commander no-daemon smoke contract is opt-in with script checks", "--scripts-typecheck or --full"));
|
||||
|
||||
@@ -0,0 +1,40 @@
|
||||
const ghAuthStatusLinePatterns = [
|
||||
/^\s*(?:[\u2713\u2714]\s*)?Logged in to\s+\S+\s+account\s+\S+\s+\([^)]+\)\s*$/iu,
|
||||
/^\s*(?:[\u2713\u2714]\s*)?Token:\s+\S+\s*$/iu,
|
||||
/^\s*(?:[\u2713\u2714]\s*)?Token scopes?:\s+.*$/iu,
|
||||
/^\s*(?:-\s*)?Token(?:\s+(?:scopes?|source|preview|value))?\s*[:=]\s*\S+.*$/iu,
|
||||
] as const;
|
||||
|
||||
const tokenLikePatterns = [
|
||||
/\bgh[pousr]_[A-Za-z0-9_]{6,}\b/gu,
|
||||
/\bgithub_pat_[A-Za-z0-9_]{6,}\b/gu,
|
||||
/\b(?:sk|xoxb|xoxp|AKIA)[A-Za-z0-9_=-]{8,}\b/gu,
|
||||
/\b(?:token|secret|password|passwd|authorization|cookie|api[_-]?key)\s*[:=]\s*[^,\s]+/giu,
|
||||
/\bBearer\s+[A-Za-z0-9._~+/-]+=*\b/giu,
|
||||
/https?:\/\/[^/\s]+:[^@\s]+@[^/\s]+/giu,
|
||||
] as const;
|
||||
|
||||
export const ghAuthStatusWrapperHint = "Use bun scripts/cli.ts gh auth status --repo pikasTech/unidesk for structured redacted GitHub auth diagnostics.";
|
||||
|
||||
export function sanitizeTaskOutputText(text: string): string {
|
||||
let redactionsApplied = 0;
|
||||
let ghAuthStatusRedactions = 0;
|
||||
const lines = String(text || "").split(/\r?\n/u).map((line) => {
|
||||
if (ghAuthStatusLinePatterns.some((pattern) => pattern.test(line))) {
|
||||
redactionsApplied += 1;
|
||||
ghAuthStatusRedactions += 1;
|
||||
return "[redacted gh auth status line]";
|
||||
}
|
||||
let next = line;
|
||||
for (const pattern of tokenLikePatterns) {
|
||||
next = next.replace(pattern, () => {
|
||||
redactionsApplied += 1;
|
||||
return "<redacted>";
|
||||
});
|
||||
}
|
||||
return next;
|
||||
});
|
||||
const sanitized = lines.join("\n");
|
||||
if (redactionsApplied === 0 || ghAuthStatusRedactions === 0 || sanitized.includes(ghAuthStatusWrapperHint)) return sanitized;
|
||||
return `${sanitized}\n${ghAuthStatusWrapperHint}`;
|
||||
}
|
||||
@@ -5,6 +5,7 @@ import { codeAgentPortForModel, codeAgentPortInfo, codeExecutionModeInfo, codeEx
|
||||
import { claudeQqNotificationOutboxStats, notificationTargetConfigured, notificationTargetLabel } from "./notifications";
|
||||
import { executionModeOptions, executionProviderOptions } from "./provider-runtime";
|
||||
import { taskFullOutput } from "./task-output";
|
||||
import { sanitizeTaskOutputText } from "./output-redaction";
|
||||
import { applyOaTraceStatsToTaskJson, taskScopeId, type OaTraceStats, type TraceStatsFallback } from "./oa-events";
|
||||
import { buildExecutionDiagnostics, schedulerHeartbeatStaleMs } from "./execution-diagnostics";
|
||||
import { buildCompactTaskTranscript, buildTaskTranscript, cachedPreviewTranscript, fullTranscript, prefixPreview, safePreview, statsDaysFromUrl, taskForCompactMetaResponse, taskForMetaResponse, taskListStepCount, taskStatisticsSummary, taskTiming, timestampMs } from "./task-view";
|
||||
@@ -167,13 +168,14 @@ function outputChunkResponse(task: QueueTask, url: URL): Response {
|
||||
const fullOutput = taskFullOutput(task);
|
||||
const page = ctx().pageBySeq(fullOutput, url, limit);
|
||||
const output = page.chunk.map((item) => {
|
||||
const truncated = !fullText && item.text.length > maxTextChars;
|
||||
const text = sanitizeTaskOutputText(item.text);
|
||||
const truncated = !fullText && text.length > maxTextChars;
|
||||
return {
|
||||
...item,
|
||||
text: truncated ? item.text.slice(0, maxTextChars) : item.text,
|
||||
textChars: item.text.length,
|
||||
text: truncated ? text.slice(0, maxTextChars) : text,
|
||||
textChars: text.length,
|
||||
textTruncated: truncated,
|
||||
omittedChars: truncated ? item.text.length - maxTextChars : 0,
|
||||
omittedChars: truncated ? text.length - maxTextChars : 0,
|
||||
};
|
||||
});
|
||||
return ctx().jsonResponse({
|
||||
|
||||
@@ -3,6 +3,7 @@
|
||||
import { appendFileSync, existsSync, mkdirSync, readFileSync, statSync } from "node:fs";
|
||||
import { resolve } from "node:path";
|
||||
import type { ArchivedLiveOutput, JsonValue, LiveOutput, OutputChannel, QueueTask, RuntimeConfig } from "./types";
|
||||
import { sanitizeTaskOutputText } from "./output-redaction";
|
||||
|
||||
export interface TaskOutputContext {
|
||||
config: Pick<RuntimeConfig, "maxInMemoryOutputRecords" | "outputArchiveDir">;
|
||||
@@ -187,7 +188,8 @@ function outputArchiveSignature(task: QueueTask): string {
|
||||
}
|
||||
|
||||
function appendOutput(task: QueueTask, channel: OutputChannel, text: string, method?: string, itemId?: string, append = false): LiveOutput | null {
|
||||
if (text.length === 0) return null;
|
||||
const safeText = sanitizeTaskOutputText(text);
|
||||
if (safeText.length === 0) return null;
|
||||
try {
|
||||
ensureTaskOutputArchiveSeeded(task);
|
||||
} catch (error) {
|
||||
@@ -196,14 +198,14 @@ function appendOutput(task: QueueTask, channel: OutputChannel, text: string, met
|
||||
const last = task.output[task.output.length - 1];
|
||||
let output: LiveOutput;
|
||||
let archiveOp: ArchivedLiveOutput["op"] = "set";
|
||||
let archiveText = text;
|
||||
let archiveText = safeText;
|
||||
if (append && last !== undefined && last.channel === channel && last.itemId === itemId && last.method === method && last.text.length < 24_000) {
|
||||
last.text += text;
|
||||
last.text += safeText;
|
||||
last.at = ctx().nowIso();
|
||||
output = last;
|
||||
archiveOp = "append";
|
||||
} else {
|
||||
output = { seq: ctx().allocateSeq(), at: ctx().nowIso(), channel, text, method, itemId };
|
||||
output = { seq: ctx().allocateSeq(), at: ctx().nowIso(), channel, text: safeText, method, itemId };
|
||||
task.output.push(output);
|
||||
}
|
||||
appendOutputArchive(task, output, archiveOp, archiveText);
|
||||
|
||||
@@ -20,6 +20,7 @@ import type {
|
||||
} from "./types";
|
||||
import { codeAgentPortForModel, codeAgentPortInfo, codeExecutionModeInfo, extractRecord } from "./code-agent/common";
|
||||
import { currentTaskPromptMarker, resolvedReferenceContextTitle, stripCodeQueueEnvironmentHint, userPromptForDisplay } from "./prompts";
|
||||
import { sanitizeTaskOutputText } from "./output-redaction";
|
||||
import { outputArchiveSignature, taskFullOutput } from "./task-output";
|
||||
import { retryPrompt } from "./judge";
|
||||
import { readOaTraceStepsForTask, type OaTraceStepSummary } from "./oa-events";
|
||||
@@ -118,7 +119,7 @@ function prefixPreview(value: string, max = 900): string {
|
||||
}
|
||||
|
||||
function linePreview(text: string, maxLines: number, maxChars: number): { text: string; omittedLines: number } {
|
||||
const clean = text.replace(/\u001b\[[0-9;]*m/gu, "").trimEnd();
|
||||
const clean = sanitizeTaskOutputText(text).replace(/\u001b\[[0-9;]*m/gu, "").trimEnd();
|
||||
if (clean.length === 0) return { text: "", omittedLines: 0 };
|
||||
const lines = clean.split(/\r?\n/u);
|
||||
const kept: string[] = [];
|
||||
@@ -132,7 +133,7 @@ function linePreview(text: string, maxLines: number, maxChars: number): { text:
|
||||
}
|
||||
|
||||
function completeTraceText(text: string): { text: string; omittedLines: number } {
|
||||
return { text: text.replace(/\u001b\[[0-9;]*m/gu, "").trimEnd(), omittedLines: 0 };
|
||||
return { text: sanitizeTaskOutputText(text).replace(/\u001b\[[0-9;]*m/gu, "").trimEnd(), omittedLines: 0 };
|
||||
}
|
||||
|
||||
function editedOutputPreview(text: string): { text: string; omittedLines: number } {
|
||||
@@ -1324,8 +1325,11 @@ function cachedPreviewTranscript(task: QueueTask): TranscriptLine[] {
|
||||
}
|
||||
|
||||
function outputForResponse(task: QueueTask, includeRaw: boolean): LiveOutput[] {
|
||||
if (includeRaw) return taskFullOutput(task);
|
||||
return task.output.slice(-80).map((item) => ({ ...item, text: safePreview(item.text, 4000) }));
|
||||
const output = includeRaw ? taskFullOutput(task) : task.output.slice(-80);
|
||||
return output.map((item) => ({
|
||||
...item,
|
||||
text: includeRaw ? sanitizeTaskOutputText(item.text) : safePreview(sanitizeTaskOutputText(item.text), 4000),
|
||||
}));
|
||||
}
|
||||
|
||||
function attemptForResponse(attempt: AttemptSummary, full = false): JsonValue {
|
||||
|
||||
Reference in New Issue
Block a user