diff --git a/config/hwlab-web-probe-sentinel/profiles.yaml b/config/hwlab-web-probe-sentinel/profiles.yaml index 1f394b5c..a128f412 100644 --- a/config/hwlab-web-probe-sentinel/profiles.yaml +++ b/config/hwlab-web-probe-sentinel/profiles.yaml @@ -106,7 +106,7 @@ baselines: jd01BootstrapSource: &jd01-bootstrap-source purpose: bootstrap-admin sourceRef: .env/HWLAB_admin.txt - sourceKey: HWLAB_BOOTSTRAP_ADMIN_PASSWORD + sourceKey: HWLAB_ADMIN_PASSWORD sourceLine: 2 dsflashPromptSource: &dsflash-prompt-source purpose: prompt-set diff --git a/scripts/src/hwlab-node-web-sentinel-p5-observe.ts b/scripts/src/hwlab-node-web-sentinel-p5-observe.ts index 48d63057..4bdd475f 100644 --- a/scripts/src/hwlab-node-web-sentinel-p5-observe.ts +++ b/scripts/src/hwlab-node-web-sentinel-p5-observe.ts @@ -124,7 +124,8 @@ export function runSentinelQuickVerify(state: SentinelCicdState, reason: string, const observerId = observerIdFromText(String(record(started.result).stdoutPreview ?? "")); printQuickVerifyProgress(state, runId, "observe-start", started.ok && observerId !== null ? "succeeded" : "failed", { observerId, exitCode: record(started.result).exitCode ?? null, timedOut: record(started.result).timedOut === true, elapsedMs: elapsedMs() }); if (!started.ok || observerId === null) { - const findings = quickVerifyControlFindings("observe-start-failed", 0, null, null); + const findings = quickVerifyControlFindings("observe-start-failed", 0, null, null) + .map((finding) => enrichObserveStartFailureFinding(finding, record(started.result))); return recordQuickVerify(state, { ok: false, runId, @@ -789,6 +790,8 @@ function compactQuickVerifyRecordStepResult(value: Record): Rec stderrBytes: numberAtNullable(value, "stderrBytes"), stdoutPreview: boundQuickVerifyRecordText(value.stdoutPreview, 240), stderrPreview: boundQuickVerifyRecordText(value.stderrPreview, 240), + stdoutTail: boundQuickVerifyRecordText(value.stdoutTail, 1200), + stderrTail: boundQuickVerifyRecordText(value.stderrTail, 1200), valuesRedacted: true, }; } @@ -1428,6 +1431,32 @@ function quickVerifyControlFindings(failure: string | null, promptIndex: number, }]; } +function enrichObserveStartFailureFinding(finding: Record, result: Record): Record { + const stdout = stringAtNullable(result, "stdoutPreview") ?? stringAtNullable(result, "stdoutTail"); + const payload = stdout === null ? null : parseJsonObject(stdout); + const data = record(record(payload).data); + const degradedReason = stringAtNullable(data, "degradedReason"); + const command = stringAtNullable(data, "command"); + if (degradedReason === null) { + return { + ...finding, + evidenceSummary: `observe start exited ${String(result.exitCode ?? "-")}; timedOut=${result.timedOut === true}`, + valuesRedacted: true, + }; + } + return { + ...finding, + rootCause: `observe-start failed: ${degradedReason}`, + rootCauseStatus: "confirmed", + rootCauseConfidence: "high", + nextAction: degradedReason === "web_login_secret_missing" + ? "Align YAML-declared bootstrap secret env injection with web-probe observe credential materialization." + : "Inspect observe-start failure payload and fix the reported degradedReason.", + evidenceSummary: `${command ?? "web-probe observe start"} degradedReason=${degradedReason}`, + valuesRedacted: true, + }; +} + function quickVerifyCompletedTurnSummaryRow(promptIndex: number, turnSummary: Record | null): Record | null { const rows = Array.isArray(record(turnSummary?.collect).rows) ? record(turnSummary?.collect).rows.map(record) : []; const scopedRows = promptIndex > 0 ? rows.filter((row) => numberAtNullable(row, "round") === promptIndex) : rows; diff --git a/scripts/src/hwlab-node/web-probe.ts b/scripts/src/hwlab-node/web-probe.ts index 538177e0..e4fc4794 100644 --- a/scripts/src/hwlab-node/web-probe.ts +++ b/scripts/src/hwlab-node/web-probe.ts @@ -1336,7 +1336,7 @@ export function readBootstrapAdminSecretMaterial(spec: RuntimeSecretSpec): Boots sourceKey, sourceLine, sourcePath: password.sourcePath, - sourcePresent: true, + sourcePresent: password.sourcePresent, sourceFingerprint: shortSecretFingerprint(password.value), passwordHash: hwlabPasswordHash(password.value), error: null, @@ -1365,7 +1365,7 @@ export function readBootstrapAdminPasswordMaterial(spec: RuntimeSecretSpec): Boo sourceKey, sourceLine, sourcePath: password.sourcePath, - sourcePresent: true, + sourcePresent: password.sourcePresent, sourceFingerprint: shortSecretFingerprint(password.value), password: password.value, error: null, @@ -1377,22 +1377,27 @@ function readBootstrapAdminUsername(spec: RuntimeSecretSpec): { ok: true; value: if (ref === undefined) return { ok: true, value: spec.bootstrapAdminUsername }; const key = spec.bootstrapAdminUsernameSourceKey ?? "username"; const material = readSecretSourceScalar(ref, key, spec.bootstrapAdminUsernameSourceLine ?? null); + if (!material.ok && material.error === "secret-source-missing" && spec.bootstrapAdminUsername.length > 0) return { ok: true, value: spec.bootstrapAdminUsername }; if (!material.ok) return { ok: false, error: `username-${material.error}` }; return { ok: true, value: material.value }; } -function readSecretSourceScalar(sourceRef: string, sourceKey: string, sourceLine: number | null): { ok: true; value: string; sourcePath: string; sourcePresent: true } | { ok: false; sourcePath: string; sourcePresent: boolean; error: string } { +function readSecretSourceScalar(sourceRef: string, sourceKey: string, sourceLine: number | null): { ok: true; value: string; sourcePath: string; sourcePresent: boolean } | { ok: false; sourcePath: string; sourcePresent: boolean; error: string } { const paths = secretSourcePaths(sourceRef); const sourcePath = paths.find((candidate) => existsSync(candidate)) ?? paths[0] ?? join(repoRoot, ".state", "secrets", sourceRef); - if (!existsSync(sourcePath)) return { ok: false, sourcePath, sourcePresent: false, error: "secret-source-missing" }; + const runtimeValue = process.env[sourceKey]; + if (!existsSync(sourcePath)) { + if (runtimeValue !== undefined && runtimeValue.length > 0) return { ok: true, value: runtimeValue, sourcePath, sourcePresent: false }; + return { ok: false, sourcePath, sourcePresent: false, error: "secret-source-missing" }; + } const text = readFileSync(sourcePath, "utf8"); if (sourceLine !== null) { const line = text.split(/\r?\n/u)[sourceLine - 1]?.replace(/\r$/u, "") ?? ""; + if (line.length === 0 && runtimeValue !== undefined && runtimeValue.length > 0) return { ok: true, value: runtimeValue, sourcePath, sourcePresent: true }; if (line.length === 0) return { ok: false, sourcePath, sourcePresent: true, error: "secret-line-missing" }; return { ok: true, value: line, sourcePath, sourcePresent: true }; } const values = parseEnvFile(text); - const runtimeValue = process.env[sourceKey]; const value = values[sourceKey] ?? runtimeValue; if (value === undefined || value.length === 0) return { ok: false, sourcePath, sourcePresent: true, error: "secret-key-missing" }; return { ok: true, value, sourcePath, sourcePresent: true };