fix: improve hwlab v02 cli observability
This commit is contained in:
Codex
+3
-3
@@ -80,8 +80,8 @@ export function rootHelp(): unknown {
|
||||
{ command: "codex steer-confirm <taskId> --steer-id <id> [--raw]", description: "Read-only lookup for a steerId in task trace so deliveryUnconfirmed can be resolved without resending the corrective prompt." },
|
||||
{ command: "codex interrupt|cancel <taskId>", description: "Request interrupt for a running Code Queue task, or cancel a queued/retry_wait task, through the same private proxy." },
|
||||
{ command: "codex (queues [--full|--all] | queue create <queueId> | queue merge <sourceQueueId> --into <targetQueueId> | move <taskId> --queue <queueId>)", description: "List low-noise queue summaries by default, including effective activity counts that distinguish scheduler-local queues, DB running tasks, and heartbeat-fresh runners; full queue rows require --full/--all." },
|
||||
{ command: "job list [--limit N] [--include-command]", description: "List async jobs from .state/jobs with a bounded default page." },
|
||||
{ command: "job status <jobId|latest> [--tail-bytes N]", description: "Show job state with bounded stdout/stderr tails." },
|
||||
{ command: "job list [--limit N] [--include-command]", description: "List async jobs from .state/jobs with a bounded default page and progress summaries." },
|
||||
{ command: "job status <jobId|latest> [--tail-bytes N]", description: "Show job state with a structured progress summary and bounded stdout/stderr tails." },
|
||||
{ command: "debug health", description: "Probe internal core, nodes, system/Docker status, frontend, provider ingress, and public boundary." },
|
||||
{ command: "debug dispatch [providerId] [docker.ps|provider.upgrade|host.ssh|microservice.http|echo] [--wait-ms N]", description: "Submit a real internal-core dispatch request for CLI debugging." },
|
||||
{ command: "debug task <taskId|latest>", description: "Read a dispatched task record from internal core for CLI debugging." },
|
||||
@@ -481,7 +481,7 @@ function jobHelp(): unknown {
|
||||
"bun scripts/cli.ts job list [--limit N] [--include-command]",
|
||||
"bun scripts/cli.ts job status <jobId|latest> [--tail-bytes N]",
|
||||
],
|
||||
description: "Inspect fire-and-forget job state from .state/jobs without streaming unbounded logs.",
|
||||
description: "Inspect fire-and-forget job state from .state/jobs with structured progress summaries and bounded log tails.",
|
||||
};
|
||||
}
|
||||
|
||||
|
||||
@@ -43,6 +43,8 @@ const V02_SERVICE_IDS = [
|
||||
"hwlab-edge-proxy",
|
||||
"hwlab-agent-skills",
|
||||
];
|
||||
const V02_CLOUD_WEB_URL = "http://74.48.78.17:19666";
|
||||
const V02_CLOUD_API_URL = "http://74.48.78.17:19667";
|
||||
|
||||
export function v02PipelineServiceIds(): string[] {
|
||||
return [...V02_SERVICE_IDS];
|
||||
@@ -712,11 +714,152 @@ function v02ControlPlaneStatusBundle(sourceCommit: string | null | undefined): C
|
||||
`section obsoleteCronJobs kubectl get cronjob -n ${shellQuote(CI_NAMESPACE)} ${shellQuote(V02_POLLER)} ${shellQuote(V02_RECONCILER)} --ignore-not-found -o name`,
|
||||
`section argo kubectl get application -n ${shellQuote(ARGO_NAMESPACE)} ${shellQuote(V02_APP)} -o 'jsonpath={.spec.source.targetRevision}{"\\n"}{.spec.source.path}{"\\n"}{.status.sync.revision}{"\\n"}{.status.sync.status}{"\\n"}{.status.health.status}{"\\n"}'`,
|
||||
`if [ -n "$pipeline_run" ]; then section pipelineRun kubectl get pipelinerun -n ${shellQuote(CI_NAMESPACE)} "$pipeline_run" -o 'jsonpath={.status.conditions[0].status}{"\\n"}{.status.conditions[0].reason}{"\\n"}{.status.conditions[0].message}{"\\n"}'; else section pipelineRun sh -c 'true'; fi`,
|
||||
`if [ -n "$pipeline_run" ]; then section taskRuns kubectl get taskrun -n ${shellQuote(CI_NAMESPACE)} -l "tekton.dev/pipelineRun=$pipeline_run" -o 'jsonpath={range .items[*]}{.metadata.name}{"\\t"}{.status.conditions[0].status}{"\\t"}{.status.conditions[0].reason}{"\\t"}{.status.startTime}{"\\t"}{.status.completionTime}{"\\n"}{end}'; else section taskRuns sh -c 'true'; fi`,
|
||||
`section recentPipelineRuns kubectl get pipelinerun -n ${shellQuote(CI_NAMESPACE)} -l hwlab.pikastech.local/gitops-target=v02 -o ${shellQuote(pipelineRunRowsJsonPath())}`,
|
||||
`section webAssets sh -c ${shellQuote(v02WebAssetsProbeScript())}`,
|
||||
].join("\n");
|
||||
return g14K3s(["script", "--", script], 60_000);
|
||||
}
|
||||
|
||||
function v02WebAssetsProbeScript(): string {
|
||||
return [
|
||||
"set +e",
|
||||
`base=${shellQuote(V02_CLOUD_WEB_URL)}`,
|
||||
`api=${shellQuote(V02_CLOUD_API_URL)}`,
|
||||
"fetch_url() {",
|
||||
" if command -v curl >/dev/null 2>&1; then",
|
||||
" curl -fsS --connect-timeout 2 --max-time 5 \"$1\"",
|
||||
" elif command -v wget >/dev/null 2>&1; then",
|
||||
" wget -q -T 5 -O - \"$1\"",
|
||||
" else",
|
||||
" return 127",
|
||||
" fi",
|
||||
"}",
|
||||
"printf 'baseUrl\\t%s\\n' \"$base\"",
|
||||
"printf 'apiUrl\\t%s\\n' \"$api\"",
|
||||
"html=$(fetch_url \"$base/\" 2>/dev/null)",
|
||||
"html_code=$?",
|
||||
"printf 'htmlOk\\t%s\\n' \"$html_code\"",
|
||||
"printf 'readonlyNote\\t%s\\n' \"$(printf '%s' \"$html\" | grep -Eiq 'readonly-rpc|复核入口'; printf '%s' \"$?\")\"",
|
||||
"css=$(fetch_url \"$base/styles.css\" 2>/dev/null)",
|
||||
"css_code=$?",
|
||||
"printf 'cssOk\\t%s\\n' \"$css_code\"",
|
||||
"printf 'sidebarFitCss\\t%s\\n' \"$(printf '%s' \"$css\" | grep -Eq 'grid-template-rows:[[:space:]]*auto[[:space:]]+auto[[:space:]]+auto[[:space:]]+auto[[:space:]]+minmax\\(0,[[:space:]]*1fr\\)'; printf '%s' \"$?\")\"",
|
||||
"printf 'workspaceFitCss\\t%s\\n' \"$(printf '%s' \"$css\" | grep -Eq 'grid-template-rows:[[:space:]]*auto[[:space:]]+minmax\\(0,[[:space:]]*1fr\\)'; printf '%s' \"$?\")\"",
|
||||
"printf 'eventPanelFitCss\\t%s\\n' \"$(printf '%s' \"$css\" | grep -Eq 'grid-template-rows:[[:space:]]*auto[[:space:]]+minmax\\(132px,[[:space:]]*1fr\\)'; printf '%s' \"$?\")\"",
|
||||
"health=$(fetch_url \"$api/health/live\" 2>/dev/null)",
|
||||
"health_code=$?",
|
||||
"printf 'apiHealthOk\\t%s\\n' \"$health_code\"",
|
||||
"printf 'apiRevision\\t%s\\n' \"$(printf '%s' \"$health\" | sed -n 's/.*\"revision\"[[:space:]]*:[[:space:]]*\"\\([0-9A-Za-z._-]*\\)\".*/\\1/p' | head -1)\"",
|
||||
].join("\n");
|
||||
}
|
||||
|
||||
function taskRunsCompactFromText(text: string, commandOk: boolean, pipelineRun: string | null, exitCode: number | null, stderr: string): Record<string, unknown> {
|
||||
if (!commandOk) {
|
||||
return {
|
||||
ok: false,
|
||||
pipelineRun,
|
||||
exitCode,
|
||||
stderr: stderr.trim().slice(0, 2000),
|
||||
counts: { succeeded: 0, failed: 0, running: 0, unknown: 0 },
|
||||
items: [],
|
||||
};
|
||||
}
|
||||
const items = text
|
||||
.split(/\r?\n/u)
|
||||
.map((line) => line.trim())
|
||||
.filter(Boolean)
|
||||
.map((line) => {
|
||||
const [name = "", status = "", reason = "", startTime = "", completionTime = ""] = line.split("\t");
|
||||
return {
|
||||
name,
|
||||
status: status || null,
|
||||
reason: reason || null,
|
||||
startTime: startTime || null,
|
||||
completionTime: completionTime || null,
|
||||
durationSeconds: secondsBetween(startTime, completionTime),
|
||||
};
|
||||
});
|
||||
const counts = {
|
||||
succeeded: items.filter((item) => item.status === "True").length,
|
||||
failed: items.filter((item) => item.status === "False").length,
|
||||
running: items.filter((item) => item.status === "Unknown").length,
|
||||
unknown: items.filter((item) => item.status !== "True" && item.status !== "False" && item.status !== "Unknown").length,
|
||||
};
|
||||
return {
|
||||
ok: true,
|
||||
pipelineRun,
|
||||
counts,
|
||||
items,
|
||||
summary: `taskruns succeeded=${counts.succeeded} failed=${counts.failed} running=${counts.running} unknown=${counts.unknown}`,
|
||||
disclosure: items.length > 0 ? "complete taskrun condition summary" : "no taskruns observed yet",
|
||||
};
|
||||
}
|
||||
|
||||
function v02WebAssetsFromText(text: string, commandOk: boolean, sourceCommit: string | null, argoSyncRevision: string | null, exitCode: number | null, stderr: string): Record<string, unknown> {
|
||||
const fields: Record<string, string> = {};
|
||||
for (const line of text.split(/\r?\n/u)) {
|
||||
const [key = "", ...rest] = line.split("\t");
|
||||
if (key.length > 0) fields[key] = rest.join("\t");
|
||||
}
|
||||
const htmlOk = fields.htmlOk === "0";
|
||||
const cssOk = fields.cssOk === "0";
|
||||
const apiHealthOk = fields.apiHealthOk === "0";
|
||||
const readonlyNoteAbsent = fields.readonlyNote === "1";
|
||||
const sidebarFitCss = fields.sidebarFitCss === "0";
|
||||
const workspaceFitCss = fields.workspaceFitCss === "0";
|
||||
const eventPanelFitCss = fields.eventPanelFitCss === "0";
|
||||
const apiRevision = fields.apiRevision || null;
|
||||
const webChecksPass = htmlOk && cssOk && readonlyNoteAbsent && sidebarFitCss && workspaceFitCss && eventPanelFitCss && apiHealthOk;
|
||||
const failedChecks = Object.entries({
|
||||
htmlOk,
|
||||
cssOk,
|
||||
readonlyNoteAbsent,
|
||||
sidebarFitCss,
|
||||
workspaceFitCss,
|
||||
eventPanelFitCss,
|
||||
apiHealthOk,
|
||||
}).filter(([, ok]) => !ok).map(([name]) => name);
|
||||
return {
|
||||
ok: commandOk && webChecksPass,
|
||||
summary: commandOk && webChecksPass ? "19666/19667 probes passed" : `19666/19667 probe issues: ${failedChecks.join(", ") || "command failed"}`,
|
||||
baseUrl: fields.baseUrl || V02_CLOUD_WEB_URL,
|
||||
apiUrl: fields.apiUrl || V02_CLOUD_API_URL,
|
||||
sourceCommit,
|
||||
argoSyncRevision: argoSyncRevision || null,
|
||||
checks: {
|
||||
htmlOk,
|
||||
cssOk,
|
||||
readonlyNoteAbsent,
|
||||
sidebarFitCss,
|
||||
workspaceFitCss,
|
||||
eventPanelFitCss,
|
||||
apiHealthOk,
|
||||
},
|
||||
probeExitCodes: {
|
||||
html: numericField(fields.htmlOk),
|
||||
css: numericField(fields.cssOk),
|
||||
readonlyNoteGrep: numericField(fields.readonlyNote),
|
||||
sidebarFitCssGrep: numericField(fields.sidebarFitCss),
|
||||
workspaceFitCssGrep: numericField(fields.workspaceFitCss),
|
||||
eventPanelFitCssGrep: numericField(fields.eventPanelFitCss),
|
||||
apiHealth: numericField(fields.apiHealthOk),
|
||||
},
|
||||
apiRevision,
|
||||
note: apiRevision && sourceCommit && apiRevision !== sourceCommit
|
||||
? "cloud-api image revision can differ when a change only republishes Cloud Web static assets; use webAssets.checks for 19666 frontend asset readiness."
|
||||
: null,
|
||||
exitCode,
|
||||
stderr: commandOk ? "" : stderr.trim().slice(0, 2000),
|
||||
};
|
||||
}
|
||||
|
||||
function numericField(value: string | undefined): number | null {
|
||||
if (value === undefined || value.trim().length === 0) return null;
|
||||
const parsed = Number(value);
|
||||
return Number.isFinite(parsed) ? parsed : null;
|
||||
}
|
||||
|
||||
function listV02PipelineRunsCompactFromText(text: string, commandOk: boolean, command: string[] | string, exitCode: number | null, stderr: string, limit = 8, nowMs = Date.now()): Record<string, unknown> {
|
||||
if (!commandOk) {
|
||||
return {
|
||||
@@ -1184,6 +1327,8 @@ function v02ControlPlaneStatus(sourceCommitInput?: string | null): Record<string
|
||||
const obsoleteCronJobs = sections.obsoleteCronJobs;
|
||||
const argo = sections.argo;
|
||||
const pipelineRunSection = sections.pipelineRun;
|
||||
const taskRunsSection = sections.taskRuns;
|
||||
const webAssetsSection = sections.webAssets;
|
||||
const recentPipelineRuns = listV02PipelineRunsCompactFromText(
|
||||
sections.recentPipelineRuns?.stdout ?? "",
|
||||
shellSectionOk(sections.recentPipelineRuns),
|
||||
@@ -1237,6 +1382,21 @@ function v02ControlPlaneStatus(sourceCommitInput?: string | null): Record<string
|
||||
pipelineRunSection?.exitCode ?? null,
|
||||
bundle.stderr,
|
||||
),
|
||||
taskRuns: taskRunsCompactFromText(
|
||||
taskRunsSection?.stdout ?? "",
|
||||
shellSectionOk(taskRunsSection),
|
||||
pipelineRun,
|
||||
taskRunsSection?.exitCode ?? null,
|
||||
bundle.stderr,
|
||||
),
|
||||
webAssets: v02WebAssetsFromText(
|
||||
webAssetsSection?.stdout ?? "",
|
||||
shellSectionOk(webAssetsSection),
|
||||
sourceCommit,
|
||||
syncRevision,
|
||||
webAssetsSection?.exitCode ?? null,
|
||||
bundle.stderr,
|
||||
),
|
||||
activePipelineRuns,
|
||||
recentPipelineRuns,
|
||||
query: {
|
||||
|
||||
+117
-2
@@ -24,6 +24,19 @@ export interface JobRecord {
|
||||
note: string;
|
||||
}
|
||||
|
||||
export interface JobProgressSummary {
|
||||
kind: "hwlab-v02-trigger" | "generic";
|
||||
stage: string | null;
|
||||
stageStatus: string | null;
|
||||
sourceCommit: string | null;
|
||||
pipelineRun: string | null;
|
||||
pipelineCreated: boolean | null;
|
||||
lastEventAt: string | null;
|
||||
eventsObserved: number;
|
||||
summary: string;
|
||||
nextCommand: string | null;
|
||||
}
|
||||
|
||||
export interface StartJobOptions {
|
||||
runner?: "local" | "docker";
|
||||
dockerImage?: string;
|
||||
@@ -142,6 +155,7 @@ export async function runJob(id: string): Promise<JobRecord> {
|
||||
}
|
||||
|
||||
export function jobWithTail(job: JobRecord, maxBytes = 12000): JobRecord & {
|
||||
progress: JobProgressSummary;
|
||||
tailPolicy: {
|
||||
requestedTailBytes: number;
|
||||
stdoutBytes: number;
|
||||
@@ -155,8 +169,12 @@ export function jobWithTail(job: JobRecord, maxBytes = 12000): JobRecord & {
|
||||
} {
|
||||
const stdoutBytes = existsSync(job.stdoutFile) ? statSync(job.stdoutFile).size : 0;
|
||||
const stderrBytes = existsSync(job.stderrFile) ? statSync(job.stderrFile).size : 0;
|
||||
const progressTailBytes = Math.max(maxBytes, 96_000);
|
||||
const stdoutProgressTail = tailFile(job.stdoutFile, progressTailBytes);
|
||||
const stderrProgressTail = tailFile(job.stderrFile, progressTailBytes);
|
||||
return {
|
||||
...job,
|
||||
progress: summarizeJobProgress(job, progressTailBytes, { stdoutTail: stdoutProgressTail, stderrTail: stderrProgressTail }),
|
||||
tailPolicy: {
|
||||
requestedTailBytes: maxBytes,
|
||||
stdoutBytes,
|
||||
@@ -165,11 +183,107 @@ export function jobWithTail(job: JobRecord, maxBytes = 12000): JobRecord & {
|
||||
stderrTruncated: stderrBytes > maxBytes,
|
||||
fullLogPaths: { stdoutFile: job.stdoutFile, stderrFile: job.stderrFile },
|
||||
},
|
||||
stdoutTail: tailFile(job.stdoutFile, maxBytes),
|
||||
stderrTail: tailFile(job.stderrFile, maxBytes),
|
||||
stdoutTail: tailTextByBytes(stdoutProgressTail, maxBytes),
|
||||
stderrTail: tailTextByBytes(stderrProgressTail, maxBytes),
|
||||
};
|
||||
}
|
||||
|
||||
function summarizeJobProgress(job: JobRecord, maxBytes = 96_000, tails?: { stdoutTail: string; stderrTail: string }): JobProgressSummary {
|
||||
const knownWorkflow = job.name === "hwlab_g14_v02_trigger_current";
|
||||
if (!knownWorkflow && tails === undefined) return genericJobProgress(job);
|
||||
const progressTailBytes = Math.max(4096, Math.floor(maxBytes));
|
||||
const stderrTail = tails?.stderrTail ?? tailFile(job.stderrFile, progressTailBytes);
|
||||
const stdoutTail = tails?.stdoutTail ?? tailFile(job.stdoutFile, progressTailBytes);
|
||||
const events = parseJsonLineEvents(stderrTail, "hwlab.v02.trigger.progress");
|
||||
const lastEvent = events.at(-1) ?? {};
|
||||
const stage = stringField(lastEvent.stage);
|
||||
const stageStatus = stringField(lastEvent.status);
|
||||
const sourceCommit = stringField(lastEvent.sourceCommit) ?? firstMatch(stdoutTail, /"sourceCommit"\s*:\s*"([0-9a-f]{40})"/iu);
|
||||
const pipelineRun = stringField(lastEvent.pipelineRun) ?? firstMatch(stdoutTail, /"pipelineRun"\s*:\s*"([^"]+)"/u);
|
||||
const pipelineCreated = /pipelinerun\.tekton\.dev\/[^ \n]+ created/u.test(stdoutTail)
|
||||
? true
|
||||
: stage === "create-pipelinerun" && stageStatus === "failed"
|
||||
? false
|
||||
: null;
|
||||
const lastEventAt = stringField(lastEvent.at);
|
||||
const kind = events.length > 0 || knownWorkflow ? "hwlab-v02-trigger" : "generic";
|
||||
const nextCommand = pipelineRun
|
||||
? `bun scripts/cli.ts hwlab g14 control-plane status --lane v02`
|
||||
: job.status === "running"
|
||||
? `bun scripts/cli.ts job status ${job.id} --tail-bytes 12000`
|
||||
: null;
|
||||
const summary = kind === "hwlab-v02-trigger"
|
||||
? [
|
||||
job.status,
|
||||
stage ? `${stage}${stageStatus ? `:${stageStatus}` : ""}` : "stage:unknown",
|
||||
sourceCommit ? `source=${sourceCommit.slice(0, 12)}` : null,
|
||||
pipelineRun ? `pipelineRun=${pipelineRun}` : null,
|
||||
pipelineCreated === true ? "created" : pipelineCreated === false ? "create-failed" : null,
|
||||
].filter(Boolean).join(" ")
|
||||
: `${job.status}${job.exitCode === null ? "" : ` exit=${job.exitCode}`}`;
|
||||
return {
|
||||
kind,
|
||||
stage,
|
||||
stageStatus,
|
||||
sourceCommit,
|
||||
pipelineRun,
|
||||
pipelineCreated,
|
||||
lastEventAt,
|
||||
eventsObserved: events.length,
|
||||
summary,
|
||||
nextCommand,
|
||||
};
|
||||
}
|
||||
|
||||
function genericJobProgress(job: JobRecord): JobProgressSummary {
|
||||
return {
|
||||
kind: "generic",
|
||||
stage: null,
|
||||
stageStatus: null,
|
||||
sourceCommit: null,
|
||||
pipelineRun: null,
|
||||
pipelineCreated: null,
|
||||
lastEventAt: null,
|
||||
eventsObserved: 0,
|
||||
summary: `${job.status}${job.exitCode === null ? "" : ` exit=${job.exitCode}`}`,
|
||||
nextCommand: job.status === "running" ? `bun scripts/cli.ts job status ${job.id} --tail-bytes 12000` : null,
|
||||
};
|
||||
}
|
||||
|
||||
function tailTextByBytes(text: string, maxBytes: number): string {
|
||||
const safeMaxBytes = Math.max(0, Math.floor(maxBytes));
|
||||
if (safeMaxBytes === 0) return "";
|
||||
const buffer = Buffer.from(text, "utf8");
|
||||
if (buffer.length <= safeMaxBytes) return text;
|
||||
return buffer.subarray(buffer.length - safeMaxBytes).toString("utf8");
|
||||
}
|
||||
|
||||
function parseJsonLineEvents(text: string, eventName: string): Record<string, unknown>[] {
|
||||
const events: Record<string, unknown>[] = [];
|
||||
for (const line of text.split(/\r?\n/u)) {
|
||||
const trimmed = line.trim();
|
||||
if (!trimmed.startsWith("{")) continue;
|
||||
try {
|
||||
const parsed = JSON.parse(trimmed) as unknown;
|
||||
if (typeof parsed === "object" && parsed !== null && !Array.isArray(parsed) && (parsed as Record<string, unknown>).event === eventName) {
|
||||
events.push(parsed as Record<string, unknown>);
|
||||
}
|
||||
} catch {
|
||||
// Ignore non-JSON stderr lines; the raw tail remains available in stderrTail.
|
||||
}
|
||||
}
|
||||
return events;
|
||||
}
|
||||
|
||||
function stringField(value: unknown): string | null {
|
||||
return typeof value === "string" && value.trim().length > 0 ? value.trim() : null;
|
||||
}
|
||||
|
||||
function firstMatch(text: string, pattern: RegExp): string | null {
|
||||
const match = pattern.exec(text);
|
||||
return typeof match?.[1] === "string" && match[1].length > 0 ? match[1] : null;
|
||||
}
|
||||
|
||||
export interface JobListOptions {
|
||||
limit?: number;
|
||||
includeCommand?: boolean;
|
||||
@@ -182,6 +296,7 @@ export function listJobsSummary(options: JobListOptions = {}): unknown {
|
||||
id: job.id,
|
||||
name: job.name,
|
||||
status: job.status,
|
||||
progress: summarizeJobProgress(job, 32_000),
|
||||
runner: job.runner,
|
||||
runnerPid: job.runnerPid ?? null,
|
||||
runnerContainer: job.runnerContainer ?? null,
|
||||
|
||||
Reference in New Issue
Block a user