// SPEC: PJ2026-01060307 控制面模块化 draft-2026-06-25-p0. yaml-lane module for scripts/src/agentrun.ts. // Moved mechanically from scripts/src/agentrun.ts:3311-4219 for #903. // SPEC: PJ2026-01020108 cancel lifecycle + PJ2026-01020205 AipodSpec binding + PJ2026-01020302 session policy + PJ2026-01020305 cancel control + PJ2026-01060305/06 YAML execution policy and bounded output draft-2026-06-25-p0. // Exposes AgentRun lane-scoped policy, AipodSpec SecretRef binding, cancel lifecycle, and bounded default output in the UniDesk CLI. import { chmodSync, copyFileSync, existsSync, readFileSync, statSync, writeFileSync } from "node:fs"; import { join } from "node:path"; import { spawnSync } from "node:child_process"; import { rootPath, type UniDeskConfig } from "../config"; import type { RenderedCliResult } from "../output"; import { applyLocalCaddyManagedSite } from "../pk01-caddy"; import { runSshCommandCapture, type SshCaptureResult } from "../ssh"; import { runRemoteSshCommandCapture } from "../remote"; import { startJob } from "../jobs"; import { AGENTRUN_CONFIG_PATH, agentRunLaneSummary, agentRunPipelineRunName, agentRunProviderCredentialRefs, resolveAgentRunLaneTarget, type AgentRunCancelLifecycleSpec, type AgentRunLaneSpec, } from "../agentrun-lanes"; import { agentRunImageArtifact, placeholderAgentRunImage, renderedFilesDigest, renderedObjectsDigest, renderAgentRunControlPlaneManifests, renderAgentRunGitopsFiles, type AgentRunArtifactService, } from "../agentrun-manifests"; import { sha256Fingerprint } from "../platform-infra-ops-library"; import type { CleanupReleasedPvOptions, CleanupRunnersOptions, CleanupRunsOptions, RefreshOptions } from "./options"; import { cleanupReleasedPvsFinalizeNodeScript, cleanupReleasedPvsPlanNodeScript, cleanupRunnersFinalizeNodeScript, cleanupRunsFinalizeNodeScript, cleanupRunsPlanNodeScript, refreshYamlLaneScript } from "./git-mirror"; import { cleanupRunnersFactsNodeScript, cleanupRunnersPlanNodeScript, collectLaneSecretSources, createYamlLaneJobScript, yamlLaneGitopsPublishJobManifest, yamlLaneGitopsPublishPayloadFromProbe, yamlLaneJobProbeScript } from "./secrets"; import { capture, captureJsonPayload, compactCapture, progressEvent, shQuote, sleep, stringOrNull } from "./utils"; export async function refresh(config: UniDeskConfig, options: RefreshOptions): Promise> { return await refreshYamlLane(config, options); } export async function refreshYamlLane(config: UniDeskConfig, options: RefreshOptions): Promise> { const { configPath, spec } = resolveAgentRunLaneTarget(options); const plan = { node: spec.nodeId, lane: spec.lane, version: spec.version, argoNamespace: spec.gitops.argoNamespace, argoApplication: spec.gitops.argoApplication, gitopsBranch: spec.gitops.branch, mutation: "argocd-hard-refresh", valuesPrinted: false, }; if (options.dryRun || !options.confirm) { return { ok: true, command: "agentrun control-plane refresh", mode: "dry-run", mutation: false, configPath, target: agentRunLaneSummary(spec), plan, next: { confirm: `bun scripts/cli.ts agentrun control-plane refresh --node ${spec.nodeId} --lane ${spec.lane} --confirm`, status: `bun scripts/cli.ts agentrun control-plane status --node ${spec.nodeId} --lane ${spec.lane}`, }, valuesPrinted: false, }; } const refreshed = await capture(config, spec.nodeKubeRoute, ["sh", "--", refreshYamlLaneScript(spec)]); const payload = captureJsonPayload(refreshed); return { ok: refreshed.exitCode === 0 && payload.ok !== false, command: "agentrun control-plane refresh", mode: "confirmed-yaml-lane", mutation: true, configPath, target: agentRunLaneSummary(spec), plan, result: payload, refreshed: compactCapture(refreshed, { full: refreshed.exitCode !== 0, stdoutTailChars: 3000, stderrTailChars: 3000 }), next: { status: `bun scripts/cli.ts agentrun control-plane status --node ${spec.nodeId} --lane ${spec.lane}`, }, valuesPrinted: false, }; } export async function cleanupRunners(config: UniDeskConfig, options: CleanupRunnersOptions): Promise> { const { configPath, spec } = resolveAgentRunLaneTarget(options); const result = await capture(config, spec.nodeKubeRoute, ["sh", "--", cleanupRunnersScript(options, spec)]); const payload = captureJsonPayload(result); const ok = result.exitCode === 0 && payload.ok !== false; const base = { ...payload, ok, command: "agentrun control-plane cleanup-runners", configPath, target: agentRunLaneSummary(spec), mode: options.dryRun || !options.confirm ? "dry-run" : "confirmed-cleanup", namespace: spec.runtime.namespace, retention: spec.deployment.runner.retention, probe: compactCapture(result, { full: result.exitCode !== 0, stdoutTailChars: 3000, stderrTailChars: 3000 }), }; if (options.dryRun || !options.confirm) { return { ...base, dryRun: true, mutation: false, next: { confirm: `bun scripts/cli.ts agentrun control-plane cleanup-runners --node ${spec.nodeId} --lane ${spec.lane}${options.forceActive ? " --force-active" : ""} --confirm`, }, }; } return { ...base, dryRun: false, mutation: true, followUp: { dryRun: `bun scripts/cli.ts agentrun control-plane cleanup-runners --node ${spec.nodeId} --lane ${spec.lane}${options.forceActive ? " --force-active" : ""} --dry-run`, status: `bun scripts/cli.ts agentrun control-plane status --node ${spec.nodeId} --lane ${spec.lane}`, }, }; } export async function cleanupRuns(config: UniDeskConfig, options: CleanupRunsOptions): Promise> { const { configPath, spec } = resolveAgentRunLaneTarget(options); const result = await capture(config, spec.nodeKubeRoute, ["sh", "--", cleanupRunsScript(options, spec.ci.namespace, spec.ci.pipelineRunPrefix)]); const payload = captureJsonPayload(result); const ok = result.exitCode === 0 && payload.ok !== false; const base = { ...payload, ok, command: "agentrun control-plane cleanup-runs", configPath, target: agentRunLaneSummary(spec), mode: options.dryRun || !options.confirm ? "dry-run" : "confirmed-cleanup", namespace: spec.ci.namespace, minAgeMinutes: options.minAgeMinutes, limit: options.limit, probe: compactCapture(result, { full: result.exitCode !== 0, stdoutTailChars: 3000, stderrTailChars: 3000 }), }; if (options.dryRun || !options.confirm) { return { ...base, dryRun: true, mutation: false, next: { confirm: `bun scripts/cli.ts agentrun control-plane cleanup-runs --node ${spec.nodeId} --lane ${spec.lane} --min-age-minutes ${options.minAgeMinutes} --limit ${options.limit} --confirm`, }, }; } return { ...base, dryRun: false, mutation: true, followUp: { status: `bun scripts/cli.ts agentrun control-plane status --node ${spec.nodeId} --lane ${spec.lane}`, releasedPvs: `bun scripts/cli.ts agentrun control-plane cleanup-released-pvs --node ${spec.nodeId} --lane ${spec.lane} --limit ${options.limit} --dry-run`, diskPressure: `trans ${spec.nodeKubeRoute} kubectl get node -o jsonpath='{range .items[*]}{.metadata.name}{\"\\t\"}{range .status.conditions[*]}{.type}{\"=\"}{.status}{\" \"}{.reason}{\";\"}{end}{\"\\n\"}{end}'`, }, }; } export async function cleanupReleasedPvs(config: UniDeskConfig, options: CleanupReleasedPvOptions): Promise> { const { configPath, spec } = resolveAgentRunLaneTarget(options); const result = await capture(config, spec.nodeKubeRoute, ["sh", "--", cleanupReleasedPvsScript(options, spec.ci.namespace)]); const payload = captureJsonPayload(result); const ok = result.exitCode === 0 && payload.ok !== false; const base = { ...payload, ok, command: "agentrun control-plane cleanup-released-pvs", configPath, target: agentRunLaneSummary(spec), mode: options.dryRun || !options.confirm ? "dry-run" : "confirmed-cleanup", namespace: spec.ci.namespace, limit: options.limit, probe: compactCapture(result, { full: result.exitCode !== 0, stdoutTailChars: 3000, stderrTailChars: 3000 }), }; if (options.dryRun || !options.confirm) { return { ...base, dryRun: true, mutation: false, next: { confirm: `bun scripts/cli.ts agentrun control-plane cleanup-released-pvs --node ${spec.nodeId} --lane ${spec.lane} --limit ${options.limit} --confirm`, }, }; } return { ...base, dryRun: false, mutation: true, followUp: { cleanupRuns: `bun scripts/cli.ts agentrun control-plane cleanup-runs --node ${spec.nodeId} --lane ${spec.lane} --min-age-minutes 30 --limit ${options.limit} --dry-run`, diskPressure: `trans ${spec.nodeKubeRoute} kubectl get node -o jsonpath='{range .items[*]}{.metadata.name}{\"\\t\"}{range .status.conditions[*]}{.type}{\"=\"}{.status}{\" \"}{.reason}{\";\"}{end}{\"\\n\"}{end}'`, }, }; } export function cleanupRunnersScript(options: CleanupRunnersOptions, spec: AgentRunLaneSpec): string { const retention = spec.deployment.runner.retention; const matchLabelsB64 = Buffer.from(JSON.stringify(retention.selectors.matchLabels), "utf8").toString("base64"); const jobNamePrefixesB64 = Buffer.from(JSON.stringify(retention.selectors.jobNamePrefixes), "utf8").toString("base64"); return [ "set -eu", `namespace=${shQuote(spec.runtime.namespace)}`, `manager_deployment=${shQuote(spec.runtime.managerDeployment)}`, `max_runners=${String(retention.maxRunners)}`, `cleanup_order=${shQuote(retention.cleanupOrder)}`, `active_heartbeat_max_age_ms=${String(retention.activeHeartbeatMaxAgeMs)}`, `age_based_cleanup_enabled=${retention.ageBasedCleanup.enabled ? "true" : "false"}`, `age_based_max_age_hours=${retention.ageBasedCleanup.maxAgeHours === null ? "" : String(retention.ageBasedCleanup.maxAgeHours)}`, `timeout_seconds=${String(options.timeoutSeconds)}`, `force_active=${options.forceActive ? "true" : "false"}`, `match_labels_json_b64=${shQuote(matchLabelsB64)}`, `job_name_prefixes_json_b64=${shQuote(jobNamePrefixesB64)}`, "match_labels_json=$(printf '%s' \"$match_labels_json_b64\" | base64 -d)", "job_name_prefixes_json=$(printf '%s' \"$job_name_prefixes_json_b64\" | base64 -d)", "tmp_dir=$(mktemp -d)", "trap 'rm -rf \"$tmp_dir\"' EXIT", "kubectl -n \"$namespace\" get job -o json > \"$tmp_dir/jobs.json\"", "kubectl -n \"$namespace\" get pod -o json > \"$tmp_dir/pods.json\"", "facts_exit=0", "set +e", "kubectl -n \"$namespace\" exec -i deploy/\"$manager_deployment\" -- env RETENTION_NAMESPACE=\"$namespace\" sh -lc 'cat >/tmp/agentrun-runner-retention.mjs && bun /tmp/agentrun-runner-retention.mjs' > \"$tmp_dir/runner-facts.json\" 2> \"$tmp_dir/runner-facts.err\" <<'NODE'", cleanupRunnersFactsNodeScript(), "NODE", "facts_exit=$?", "set -e", "if [ \"$facts_exit\" -ne 0 ]; then", " printf '%s\\n' '{\"ok\":false,\"items\":[],\"failureKind\":\"manager-facts-unavailable\",\"valuesPrinted\":false}' > \"$tmp_dir/runner-facts.json\"", "fi", "MATCH_LABELS_JSON=\"$match_labels_json\" JOB_NAME_PREFIXES_JSON=\"$job_name_prefixes_json\" MAX_RUNNERS=\"$max_runners\" CLEANUP_ORDER=\"$cleanup_order\" ACTIVE_HEARTBEAT_MAX_AGE_MS=\"$active_heartbeat_max_age_ms\" AGE_BASED_CLEANUP_ENABLED=\"$age_based_cleanup_enabled\" AGE_BASED_MAX_AGE_HOURS=\"$age_based_max_age_hours\" FORCE_ACTIVE=\"$force_active\" FACTS_EXIT=\"$facts_exit\" TMP_DIR=\"$tmp_dir\" NAMESPACE=\"$namespace\" node <<'NODE' > \"$tmp_dir/plan.json\"", cleanupRunnersPlanNodeScript(), "NODE", "if [ " + shQuote(options.confirm && !options.dryRun ? "true" : "false") + " != true ]; then", " cat \"$tmp_dir/plan.json\"", " exit 0", "fi", "node -e 'const fs=require(\"node:fs\"); const plan=JSON.parse(fs.readFileSync(process.argv[1],\"utf8\")); const names=Array.isArray(plan.selectedRunnerJobs)?plan.selectedRunnerJobs:[]; fs.writeFileSync(process.argv[2], names.join(\"\\n\") + (names.length>0?\"\\n\":\"\"));' \"$tmp_dir/plan.json\" \"$tmp_dir/selected-jobs.txt\"", "delete_exit=0", "if [ -s \"$tmp_dir/selected-jobs.txt\" ]; then", " xargs -r kubectl -n \"$namespace\" delete job --ignore-not-found=true --wait=true --timeout=\"${timeout_seconds}s\" < \"$tmp_dir/selected-jobs.txt\" > \"$tmp_dir/delete.out\" 2> \"$tmp_dir/delete.err\" || delete_exit=$?", "else", " : > \"$tmp_dir/delete.out\"", " : > \"$tmp_dir/delete.err\"", "fi", "kubectl -n \"$namespace\" get job -o json > \"$tmp_dir/jobs-after.json\"", "kubectl -n \"$namespace\" get pod -o json > \"$tmp_dir/pods-after.json\"", "DELETE_EXIT=\"$delete_exit\" TMP_DIR=\"$tmp_dir\" node <<'NODE'", cleanupRunnersFinalizeNodeScript(), "NODE", ].join("\n"); } export function cleanupRunsScript(options: CleanupRunsOptions, namespace: string, pipelineRunPrefix: string): string { return [ "set -eu", `namespace=${shQuote(namespace)}`, `pipeline_run_prefix=${shQuote(pipelineRunPrefix)}`, `min_age_minutes=${String(options.minAgeMinutes)}`, `limit=${String(options.limit)}`, `timeout_seconds=${String(options.timeoutSeconds)}`, "tmp_dir=$(mktemp -d)", "trap 'rm -rf \"$tmp_dir\"' EXIT", "kubectl -n \"$namespace\" get pipelinerun -o json > \"$tmp_dir/pipelineruns.json\"", "kubectl -n \"$namespace\" get pvc -o json > \"$tmp_dir/pvcs.json\"", "kubectl get pv -o json > \"$tmp_dir/pvs.json\"", "kubectl -n \"$namespace\" get pod -o json > \"$tmp_dir/pods.json\"", "NAMESPACE=\"$namespace\" PIPELINE_RUN_PREFIX=\"$pipeline_run_prefix\" MIN_AGE_MINUTES=\"$min_age_minutes\" LIMIT=\"$limit\" TMP_DIR=\"$tmp_dir\" node <<'NODE' > \"$tmp_dir/plan.json\"", cleanupRunsPlanNodeScript(), "NODE", "if [ " + shQuote(options.confirm && !options.dryRun ? "true" : "false") + " != true ]; then", " cat \"$tmp_dir/plan.json\"", " exit 0", "fi", "node -e 'const fs=require(\"node:fs\"); const plan=JSON.parse(fs.readFileSync(process.argv[1],\"utf8\")); const names=Array.isArray(plan.selectedPipelineRuns)?plan.selectedPipelineRuns:[]; fs.writeFileSync(process.argv[2], names.join(\"\\n\") + (names.length>0?\"\\n\":\"\"));' \"$tmp_dir/plan.json\" \"$tmp_dir/selected-names.txt\"", "delete_exit=0", "if [ -s \"$tmp_dir/selected-names.txt\" ]; then", " xargs -r kubectl -n \"$namespace\" delete pipelinerun --ignore-not-found=true --wait=true --timeout=\"${timeout_seconds}s\" < \"$tmp_dir/selected-names.txt\" > \"$tmp_dir/delete.out\" 2> \"$tmp_dir/delete.err\" || delete_exit=$?", "else", " : > \"$tmp_dir/delete.out\"", " : > \"$tmp_dir/delete.err\"", "fi", "kubectl -n \"$namespace\" get pvc -o json > \"$tmp_dir/pvcs-after.json\"", "DELETE_EXIT=\"$delete_exit\" TMP_DIR=\"$tmp_dir\" node <<'NODE'", cleanupRunsFinalizeNodeScript(), "NODE", ].join("\n"); } export function cleanupReleasedPvsScript(options: CleanupReleasedPvOptions, namespace: string): string { return [ "set -eu", `namespace=${shQuote(namespace)}`, `limit=${String(options.limit)}`, `timeout_seconds=${String(options.timeoutSeconds)}`, "tmp_dir=$(mktemp -d)", "trap 'rm -rf \"$tmp_dir\"' EXIT", "kubectl get pv -o json > \"$tmp_dir/pvs.json\"", "NAMESPACE=\"$namespace\" LIMIT=\"$limit\" TMP_DIR=\"$tmp_dir\" node <<'NODE' > \"$tmp_dir/plan.json\"", cleanupReleasedPvsPlanNodeScript(), "NODE", "if [ " + shQuote(options.confirm && !options.dryRun ? "true" : "false") + " != true ]; then", " cat \"$tmp_dir/plan.json\"", " exit 0", "fi", "node -e 'const fs=require(\"node:fs\"); const plan=JSON.parse(fs.readFileSync(process.argv[1],\"utf8\")); const names=Array.isArray(plan.selectedPersistentVolumes)?plan.selectedPersistentVolumes:[]; fs.writeFileSync(process.argv[2], names.join(\"\\n\") + (names.length>0?\"\\n\":\"\"));' \"$tmp_dir/plan.json\" \"$tmp_dir/selected-pvs.txt\"", "delete_exit=0", "if [ -s \"$tmp_dir/selected-pvs.txt\" ]; then", " xargs -r kubectl delete pv --ignore-not-found=true --wait=true --timeout=\"${timeout_seconds}s\" < \"$tmp_dir/selected-pvs.txt\" > \"$tmp_dir/delete.out\" 2> \"$tmp_dir/delete.err\" || delete_exit=$?", "else", " : > \"$tmp_dir/delete.out\"", " : > \"$tmp_dir/delete.err\"", "fi", "kubectl get pv -o json > \"$tmp_dir/pvs-after.json\"", "DELETE_EXIT=\"$delete_exit\" TMP_DIR=\"$tmp_dir\" node <<'NODE'", cleanupReleasedPvsFinalizeNodeScript(), "NODE", ].join("\n"); } export function yamlLaneSourceStatusScript(spec: AgentRunLaneSpec): string { return [ "set +e", `expected_workspace=${shQuote(spec.source.workspace)}`, `source_branch=${shQuote(spec.source.branch)}`, "workspace_exists=false", "workspace_clean=null", "local_head=null", "branch=null", "remote_url=null", "remote_branch_exists=false", "remote_branch_commit=null", "status_short=''", "if [ -d .git ] || git rev-parse --show-toplevel >/dev/null 2>&1; then", " actual_workspace=$(pwd)", " workspace_exists=true", " branch=$(git rev-parse --abbrev-ref HEAD 2>/dev/null || true)", " remote_url=$(git remote get-url origin 2>/dev/null || true)", " local_head=$(git rev-parse HEAD 2>/dev/null || true)", " status_short=$(git status --short 2>/dev/null || true)", " if [ -z \"$status_short\" ]; then workspace_clean=true; else workspace_clean=false; fi", " git fetch origin \"$source_branch\" >/dev/null 2>&1 || true", " remote_branch_commit=$(git rev-parse \"refs/remotes/origin/$source_branch\" 2>/dev/null || true)", " if [ -n \"$remote_branch_commit\" ]; then remote_branch_exists=true; fi", "else", " actual_workspace=$(pwd)", "fi", "export expected_workspace source_branch workspace_exists workspace_clean local_head branch remote_url remote_branch_exists remote_branch_commit status_short actual_workspace", "node <<'NODE'", "function nullable(value) { return value && value !== 'null' ? value : null; }", "function booleanValue(value) { if (value === 'true') return true; if (value === 'false') return false; return null; }", "const env = process.env;", "console.log(JSON.stringify({", " ok: env.workspace_exists === 'true',", " expectedWorkspace: env.expected_workspace,", " actualWorkspace: env.actual_workspace || null,", " workspaceExists: env.workspace_exists === 'true',", " workspaceClean: booleanValue(env.workspace_clean),", " branch: nullable(env.branch),", " remoteUrl: nullable(env.remote_url),", " localHead: nullable(env.local_head),", " remoteBranch: env.source_branch,", " remoteBranchExists: env.remote_branch_exists === 'true',", " remoteBranchCommit: nullable(env.remote_branch_commit),", " statusShort: nullable(env.status_short),", " valuesPrinted: false", "}));", "NODE", ].join("\n"); } export function yamlLaneRuntimeStatusScript(spec: AgentRunLaneSpec, pipelineRun: string | null): string { return [ "set +e", `runtime_namespace=${shQuote(spec.runtime.namespace)}`, `ci_namespace=${shQuote(spec.ci.namespace)}`, `pipeline_name=${shQuote(spec.ci.pipeline)}`, `pipeline_run=${pipelineRun === null ? "''" : shQuote(pipelineRun)}`, `service_account=${shQuote(spec.ci.serviceAccountName)}`, `argo_namespace=${shQuote(spec.gitops.argoNamespace)}`, `argo_application=${shQuote(spec.gitops.argoApplication)}`, `manager_deployment=${shQuote(spec.runtime.managerDeployment)}`, `manager_service=${shQuote(spec.runtime.managerService)}`, `database_secret=${shQuote(spec.database.secretRef.name)}`, `database_key=${shQuote(spec.database.secretRef.key)}`, `secrets_json=${shQuote(JSON.stringify(collectLaneSecretSources(spec).map((item) => item.targetRef)))}`, "export runtime_namespace ci_namespace pipeline_name pipeline_run service_account argo_namespace argo_application manager_deployment manager_service database_secret database_key secrets_json", "tmp_dir=$(mktemp -d)", "trap 'rm -rf \"$tmp_dir\"' EXIT", "kubectl get ns \"$runtime_namespace\" -o json > \"$tmp_dir/runtime-ns.json\" 2>/dev/null", "runtime_ns_exit=$?", "kubectl get ns \"$ci_namespace\" -o json > \"$tmp_dir/ci-ns.json\" 2>/dev/null", "ci_ns_exit=$?", "kubectl -n \"$ci_namespace\" get pipeline \"$pipeline_name\" -o json > \"$tmp_dir/pipeline.json\" 2>/dev/null", "pipeline_exit=$?", "kubectl -n \"$ci_namespace\" get serviceaccount \"$service_account\" -o json > \"$tmp_dir/serviceaccount.json\" 2>/dev/null", "sa_exit=$?", "if [ -n \"$pipeline_run\" ]; then kubectl -n \"$ci_namespace\" get pipelinerun \"$pipeline_run\" -o json > \"$tmp_dir/pipelinerun.json\" 2>/dev/null; pr_exit=$?; else pr_exit=2; fi", "kubectl -n \"$argo_namespace\" get application \"$argo_application\" -o json > \"$tmp_dir/argo.json\" 2>/dev/null", "argo_exit=$?", "kubectl -n \"$runtime_namespace\" get deploy \"$manager_deployment\" -o json > \"$tmp_dir/manager-deploy.json\" 2>/dev/null", "manager_deploy_exit=$?", "kubectl -n \"$runtime_namespace\" get svc \"$manager_service\" -o json > \"$tmp_dir/manager-svc.json\" 2>/dev/null", "manager_svc_exit=$?", "kubectl -n \"$runtime_namespace\" get secret \"$database_secret\" -o json > \"$tmp_dir/db-secret.json\" 2>/dev/null", "db_secret_exit=$?", "SECRET_REFS_JSON=\"$secrets_json\" NODE_TMP=\"$tmp_dir\" node <<'NODE' > \"$tmp_dir/secrets.json\"", "const fs = require('node:fs');", "const cp = require('node:child_process');", "const refs = JSON.parse(process.env.SECRET_REFS_JSON || '[]');", "const result = [];", "for (const ref of refs) {", " const out = cp.spawnSync('kubectl', ['-n', ref.namespace, 'get', 'secret', ref.name, '-o', 'json'], { encoding: 'utf8' });", " let keyPresent = false;", " if (out.status === 0) {", " try { keyPresent = Object.prototype.hasOwnProperty.call((JSON.parse(out.stdout).data || {}), ref.key); } catch {}", " }", " result.push({ namespace: ref.namespace, name: ref.name, key: ref.key, present: out.status === 0, keyPresent, valuesPrinted: false });", "}", "console.log(JSON.stringify({ ready: result.every((item) => item.present && item.keyPresent), count: result.length, items: result, valuesPrinted: false }));", "NODE", "kubectl -n \"$runtime_namespace\" get deploy,sts,svc,secret -o name > \"$tmp_dir/runtime-names.txt\" 2>/dev/null", "NODE_TMP=\"$tmp_dir\" RUNTIME_NS_EXIT=\"$runtime_ns_exit\" CI_NS_EXIT=\"$ci_ns_exit\" PIPELINE_EXIT=\"$pipeline_exit\" SERVICE_ACCOUNT_EXIT=\"$sa_exit\" PIPELINERUN_EXIT=\"$pr_exit\" ARGO_EXIT=\"$argo_exit\" MANAGER_DEPLOY_EXIT=\"$manager_deploy_exit\" MANAGER_SVC_EXIT=\"$manager_svc_exit\" DB_SECRET_EXIT=\"$db_secret_exit\" node <<'NODE'", "const fs = require('node:fs');", "const path = require('node:path');", "const dir = process.env.NODE_TMP;", "function readJson(name) { try { return JSON.parse(fs.readFileSync(path.join(dir, name), 'utf8')); } catch { return null; } }", "function exists(exitName) { return process.env[exitName] === '0'; }", "function condition(obj) { return obj?.status?.conditions?.[0] || null; }", "function envValue(deploy, name) { return deploy?.spec?.template?.spec?.containers?.[0]?.env?.find((entry) => entry.name === name)?.value || null; }", "function pipelineRunParam(obj, name) { return (obj?.spec?.params || []).find((entry) => entry.name === name)?.value || null; }", "const pipelineRun = readJson('pipelinerun.json');", "const argo = readJson('argo.json');", "const managerDeploy = readJson('manager-deploy.json');", "const managerSvc = readJson('manager-svc.json');", "const dbSecret = readJson('db-secret.json');", "const secrets = readJson('secrets.json') || { ready: true, count: 0, items: [], valuesPrinted: false };", "let names = ''; try { names = fs.readFileSync(path.join(dir, 'runtime-names.txt'), 'utf8'); } catch {}", "const c = condition(pipelineRun);", "console.log(JSON.stringify({", " ok: exists('RUNTIME_NS_EXIT') && exists('CI_NS_EXIT'),", " runtimeNamespaceExists: exists('RUNTIME_NS_EXIT'),", " ciNamespaceExists: exists('CI_NS_EXIT'),", " serviceAccountExists: exists('SERVICE_ACCOUNT_EXIT'),", " pipeline: { exists: exists('PIPELINE_EXIT'), name: process.env.pipeline_name },", " pipelineRun: { exists: exists('PIPELINERUN_EXIT'), name: process.env.pipeline_run || null, status: c?.status || null, reason: c?.reason || null, sourceCommit: pipelineRunParam(pipelineRun, 'revision'), startTime: pipelineRun?.status?.startTime || null, completionTime: pipelineRun?.status?.completionTime || null },", " argo: { exists: exists('ARGO_EXIT'), namespace: process.env.argo_namespace, application: process.env.argo_application, revision: argo?.status?.sync?.revision || null, syncStatus: argo?.status?.sync?.status || null, healthStatus: argo?.status?.health?.status || null },", " manager: { deploymentExists: exists('MANAGER_DEPLOY_EXIT'), serviceExists: exists('MANAGER_SVC_EXIT'), deployment: process.env.manager_deployment, service: process.env.manager_service, image: managerDeploy?.spec?.template?.spec?.containers?.[0]?.image || null, sourceCommit: envValue(managerDeploy, 'AGENTRUN_SOURCE_COMMIT'), servicePorts: Array.isArray(managerSvc?.spec?.ports) ? managerSvc.spec.ports.map((port) => ({ name: port.name || null, port: port.port || null, targetPort: port.targetPort || null })) : [] },", " database: { secretPresent: exists('DB_SECRET_EXIT'), secretName: process.env.database_secret, key: process.env.database_key, keyPresent: Boolean(dbSecret?.data && Object.prototype.hasOwnProperty.call(dbSecret.data, process.env.database_key || '')) , valuesPrinted: false },", " secrets,", " localPostgres: { absent: !/postgres/i.test(names), matchingObjects: names.split(/\\r?\\n/).filter((line) => /postgres/i.test(line)).slice(0, 20) },", " valuesPrinted: false", "}));", "NODE", ].join("\n"); } export function yamlLaneSourceBootstrapProbeScript(spec: AgentRunLaneSpec): string { return [ "set +e", `workspace=${shQuote(spec.source.workspace)}`, `remote=${shQuote(spec.source.remote)}`, `branch=${shQuote(spec.source.branch)}`, "workspace_exists=false", "remote_branch_exists=false", "source_commit=null", "if [ -d \"$workspace/.git\" ]; then", " workspace_exists=true", " cd \"$workspace\"", " remote_branch=$(git ls-remote --heads \"$remote\" \"$branch\" 2>/dev/null | awk '{print $1}' | head -n 1)", " if [ -n \"$remote_branch\" ]; then remote_branch_exists=true; source_commit=\"$remote_branch\"; else source_commit=$(git rev-parse HEAD 2>/dev/null || printf null); fi", "else", " remote_branch=$(git ls-remote --heads \"$remote\" \"$branch\" 2>/dev/null | awk '{print $1}' | head -n 1)", " if [ -n \"$remote_branch\" ]; then remote_branch_exists=true; source_commit=\"$remote_branch\"; fi", "fi", "export workspace_exists remote_branch_exists source_commit", "node <<'NODE'", "console.log(JSON.stringify({ ok: true, workspaceExists: process.env.workspace_exists === 'true', remoteBranchExists: process.env.remote_branch_exists === 'true', sourceCommit: process.env.source_commit === 'null' ? null : process.env.source_commit, valuesPrinted: false }));", "NODE", ].join("\n"); } export function yamlLaneSourceBootstrapSubmitScript(spec: AgentRunLaneSpec): string { const bootstrap = spec.source.bootstrapFromBranch ?? spec.source.branch; const stateDir = `/tmp/unidesk-agentrun-source-${spec.nodeId}-${spec.lane}`; return [ "set -eu", `workspace=${shQuote(spec.source.workspace)}`, `remote=${shQuote(spec.source.remote)}`, `branch=${shQuote(spec.source.branch)}`, `bootstrap_branch=${shQuote(bootstrap)}`, `state_dir=${shQuote(stateDir)}`, "mkdir -p \"$state_dir\" \"$(dirname \"$workspace\")\"", "case \"$remote\" in", " git@*:*)", " mkdir -p \"$HOME/.ssh\"", " chmod 700 \"$HOME/.ssh\" 2>/dev/null || true", " export GIT_SSH_COMMAND=${GIT_SSH_COMMAND:-ssh -o StrictHostKeyChecking=accept-new}", " ;;", " ssh://*)", " mkdir -p \"$HOME/.ssh\"", " chmod 700 \"$HOME/.ssh\" 2>/dev/null || true", " export GIT_SSH_COMMAND=${GIT_SSH_COMMAND:-ssh -o StrictHostKeyChecking=accept-new}", " ;;", "esac", "job_id=\"source-bootstrap-$(date +%s)-$$\"", "status_file=\"$state_dir/$job_id.json\"", "stdout_file=\"$state_dir/$job_id.stdout.log\"", "stderr_file=\"$state_dir/$job_id.stderr.log\"", "cat > \"$status_file\" </dev/null | sed 's/\"/\\\\\"/g' | tr '\\n' ' ' | cut -c1-3000); CODE=\"$code\" ERROR_TAIL=\"$tail_text\" JOB_ID=\"$job_id\" WORKSPACE=\"$workspace\" BRANCH=\"$branch\" node <<'NODE' > \"$status_file\"", "const code = Number(process.env.CODE || 1);", "console.log(JSON.stringify({ ok: false, status: 'failed', exitCode: code, jobId: process.env.JOB_ID, workspace: process.env.WORKSPACE, branch: process.env.BRANCH, errorTail: process.env.ERROR_TAIL || null, valuesPrinted: false }));", "NODE", " fi; exit \"$code\"; }", " trap write_failed_status EXIT", " if [ -d \"$workspace/.git\" ] && git -C \"$workspace\" rev-parse --git-dir >/dev/null 2>&1; then", " :", " else", " rm -rf \"$workspace\"", " git clone --no-checkout \"$remote\" \"$workspace\"", " fi", " cd \"$workspace\"", " git remote set-url origin \"$remote\" || git remote add origin \"$remote\"", " git fetch origin \"$bootstrap_branch\" \"$branch\" || git fetch origin \"$bootstrap_branch\"", " if git rev-parse --verify \"refs/remotes/origin/$branch^{commit}\" >/dev/null 2>&1; then", " git checkout -B \"$branch\" \"refs/remotes/origin/$branch\"", " else", " git checkout -B \"$branch\" \"refs/remotes/origin/$bootstrap_branch\"", " fi", " if [ -f deploy/deploy.json ]; then rm deploy/deploy.json; fi", " git add -A deploy/deploy.json 2>/dev/null || true", " if ! git diff --quiet --cached -- deploy/deploy.json 2>/dev/null; then", " git -c user.email=agentrun@unidesk.local -c user.name='UniDesk AgentRun Ops' commit -m 'chore: remove service deploy json truth'", " fi", " git push -u origin \"$branch\"", " source_commit=$(git rev-parse HEAD)", " status_short=$(git status --short)", " SOURCE_COMMIT=\"$source_commit\" STATUS_SHORT=\"$status_short\" JOB_ID=\"$job_id\" WORKSPACE=\"$workspace\" BRANCH=\"$branch\" node <<'NODE' > \"$status_file\"", "console.log(JSON.stringify({ ok: process.env.STATUS_SHORT === '', status: 'succeeded', jobId: process.env.JOB_ID, workspace: process.env.WORKSPACE, branch: process.env.BRANCH, sourceCommit: process.env.SOURCE_COMMIT, workspaceClean: process.env.STATUS_SHORT === '', statusShort: process.env.STATUS_SHORT || null, removedServiceDeployJson: true, valuesPrinted: false }));", "NODE", " trap - EXIT", ") >\"$stdout_file\" 2>\"$stderr_file\" &", "pid=$!", "JOB_PID=\"$pid\" JOB_ID=\"$job_id\" STATUS_FILE=\"$status_file\" STDOUT_FILE=\"$stdout_file\" STDERR_FILE=\"$stderr_file\" node <<'NODE'", "console.log(JSON.stringify({ ok: true, status: 'submitted', jobId: process.env.JOB_ID, pid: Number(process.env.JOB_PID), statusFile: process.env.STATUS_FILE, stdoutFile: process.env.STDOUT_FILE, stderrFile: process.env.STDERR_FILE, valuesPrinted: false }));", "NODE", ].join("\n"); } export async function waitForYamlLaneSourceBootstrap(config: UniDeskConfig, spec: AgentRunLaneSpec, jobId: string | null): Promise & { ok: boolean; payload: Record }> { if (jobId === null) return { ok: false, payload: { ok: false, degradedReason: "source-bootstrap-job-id-missing", valuesPrinted: false } }; const startedAt = Date.now(); const timeoutMs = spec.source.bootstrapTimeoutSeconds * 1000; const pollMs = spec.source.bootstrapPollSeconds * 1000; let lastPayload: Record = {}; let polls = 0; while (Date.now() - startedAt < timeoutMs) { polls += 1; const probe = await capture(config, spec.nodeRoute, ["sh", "--", yamlLaneSourceBootstrapStatusScript(spec, jobId)]); const payload = captureJsonPayload(probe); lastPayload = payload; progressEvent("agentrun.yaml-lane.source-bootstrap.progress", { node: spec.nodeId, lane: spec.lane, jobId, polls, status: stringOrNull(payload.status) ?? "unknown", sourceCommit: stringOrNull(payload.sourceCommit), elapsedMs: Date.now() - startedAt, }); if (payload.status === "succeeded" && stringOrNull(payload.sourceCommit) !== null) return { ok: payload.ok === true, payload, polls, elapsedMs: Date.now() - startedAt }; if (payload.status === "failed") return { ok: false, payload, polls, elapsedMs: Date.now() - startedAt }; await sleep(pollMs); } return { ok: false, payload: { ...lastPayload, ok: false, status: "timeout", degradedReason: "source-bootstrap-timeout", valuesPrinted: false }, polls, elapsedMs: Date.now() - startedAt }; } export function yamlLaneSourceBootstrapStatusScript(spec: AgentRunLaneSpec, jobId: string): string { const stateDir = `/tmp/unidesk-agentrun-source-${spec.nodeId}-${spec.lane}`; return [ "set +e", `status_file=${shQuote(`${stateDir}/${jobId}.json`)}`, "if [ -f \"$status_file\" ]; then cat \"$status_file\"; else printf '{\"ok\":false,\"status\":\"missing\",\"valuesPrinted\":false}\\n'; fi", ].join("\n"); } export function yamlLaneSourceRestoreScript(spec: AgentRunLaneSpec): string { return [ "set +e", `workspace=${shQuote(spec.source.workspace)}`, `branch=${shQuote(spec.source.branch)}`, "tmp_dir=$(mktemp -d)", "trap 'rm -rf \"$tmp_dir\"' EXIT", "workspace_exists=false", "if git -C \"$workspace\" rev-parse --git-dir >/dev/null 2>&1; then workspace_exists=true; fi", "if [ \"$workspace_exists\" != true ]; then", " WORKSPACE=\"$workspace\" BRANCH=\"$branch\" node <<'NODE'", "console.log(JSON.stringify({ ok: false, status: 'skipped', failureKind: 'source-worktree-missing', workspace: process.env.WORKSPACE, branch: process.env.BRANCH, valuesPrinted: false }));", "NODE", " exit 0", "fi", "before_branch=$(git -C \"$workspace\" rev-parse --abbrev-ref HEAD 2>/dev/null || true)", "before_head=$(git -C \"$workspace\" rev-parse HEAD 2>/dev/null || true)", "status_short=$(git -C \"$workspace\" status --short 2>/dev/null || true)", "if [ -n \"$status_short\" ]; then", " WORKSPACE=\"$workspace\" BRANCH=\"$branch\" BEFORE_BRANCH=\"$before_branch\" BEFORE_HEAD=\"$before_head\" STATUS_SHORT=\"$status_short\" node <<'NODE'", "console.log(JSON.stringify({ ok: false, status: 'skipped', failureKind: 'source-worktree-dirty', workspace: process.env.WORKSPACE, branch: process.env.BRANCH, before: { branch: process.env.BEFORE_BRANCH || null, head: process.env.BEFORE_HEAD || null, detached: process.env.BEFORE_BRANCH === 'HEAD' }, statusShort: process.env.STATUS_SHORT || null, valuesPrinted: false }));", "NODE", " exit 0", "fi", "git -C \"$workspace\" fetch origin \"$branch\" > \"$tmp_dir/fetch.out\" 2> \"$tmp_dir/fetch.err\"", "fetch_exit=$?", "remote_branch_commit=$(git -C \"$workspace\" rev-parse \"refs/remotes/origin/$branch\" 2>/dev/null || true)", "checkout_exit=1", "if [ \"$fetch_exit\" -eq 0 ] && [ -n \"$remote_branch_commit\" ]; then", " git -C \"$workspace\" checkout -B \"$branch\" \"refs/remotes/origin/$branch\" > \"$tmp_dir/checkout.out\" 2> \"$tmp_dir/checkout.err\"", " checkout_exit=$?", "else", " : > \"$tmp_dir/checkout.out\"", " : > \"$tmp_dir/checkout.err\"", "fi", "after_branch=$(git -C \"$workspace\" rev-parse --abbrev-ref HEAD 2>/dev/null || true)", "after_head=$(git -C \"$workspace\" rev-parse HEAD 2>/dev/null || true)", "after_status_short=$(git -C \"$workspace\" status --short 2>/dev/null || true)", "fetch_err=$(tail -n 20 \"$tmp_dir/fetch.err\" 2>/dev/null | tr '\\n' ' ' | cut -c1-1200)", "checkout_err=$(tail -n 20 \"$tmp_dir/checkout.err\" 2>/dev/null | tr '\\n' ' ' | cut -c1-1200)", "WORKSPACE=\"$workspace\" BRANCH=\"$branch\" BEFORE_BRANCH=\"$before_branch\" BEFORE_HEAD=\"$before_head\" FETCH_EXIT=\"$fetch_exit\" CHECKOUT_EXIT=\"$checkout_exit\" REMOTE_BRANCH_COMMIT=\"$remote_branch_commit\" AFTER_BRANCH=\"$after_branch\" AFTER_HEAD=\"$after_head\" AFTER_STATUS_SHORT=\"$after_status_short\" FETCH_ERR=\"$fetch_err\" CHECKOUT_ERR=\"$checkout_err\" node <<'NODE'", "const fetchExit = Number(process.env.FETCH_EXIT || '1');", "const checkoutExit = Number(process.env.CHECKOUT_EXIT || '1');", "const afterClean = !process.env.AFTER_STATUS_SHORT;", "const ok = fetchExit === 0 && checkoutExit === 0 && afterClean && process.env.AFTER_BRANCH === process.env.BRANCH;", "let failureKind = null;", "if (fetchExit !== 0) failureKind = 'source-branch-fetch-failed';", "else if (!process.env.REMOTE_BRANCH_COMMIT) failureKind = 'source-branch-missing';", "else if (checkoutExit !== 0) failureKind = 'source-branch-checkout-failed';", "else if (!afterClean) failureKind = 'source-worktree-dirty-after-restore';", "else if (process.env.AFTER_BRANCH !== process.env.BRANCH) failureKind = 'source-worktree-branch-mismatch';", "console.log(JSON.stringify({", " ok,", " status: ok ? 'restored' : 'failed',", " failureKind,", " workspace: process.env.WORKSPACE,", " branch: process.env.BRANCH,", " before: { branch: process.env.BEFORE_BRANCH || null, head: process.env.BEFORE_HEAD || null, detached: process.env.BEFORE_BRANCH === 'HEAD' },", " after: { branch: process.env.AFTER_BRANCH || null, head: process.env.AFTER_HEAD || null, clean: afterClean },", " remoteBranchCommit: process.env.REMOTE_BRANCH_COMMIT || null,", " fetch: { exitCode: fetchExit, stderrTail: process.env.FETCH_ERR || null },", " checkout: { exitCode: checkoutExit, stderrTail: process.env.CHECKOUT_ERR || null },", " valuesPrinted: false", "}));", "NODE", ].join("\n"); } export function yamlLaneBuildImageSubmitScript(spec: AgentRunLaneSpec, sourceCommit: string): string { const build = spec.deployment.manager.imageBuild; const noProxy = build.noProxy.join(","); const imageRepository = `${spec.ci.registryPrefix}/${build.repository}`; const stateDir = `/tmp/unidesk-agentrun-build-${spec.nodeId}-${spec.lane}`; const buildArgs = Object.entries(build.buildArgs) .sort(([left], [right]) => left.localeCompare(right)) .map(([key, value]) => `${key}=${value}`); const script = [ "set -eu", `workspace=${shQuote(spec.source.workspace)}`, `source_commit=${shQuote(sourceCommit)}`, `state_dir=${shQuote(stateDir)}`, `containerfile=${shQuote(build.containerfile)}`, `context_dir=${shQuote(build.context)}`, `image_repository=${shQuote(imageRepository)}`, `network=${shQuote(build.network)}`, `http_proxy_value=${build.httpProxy === null ? "''" : shQuote(build.httpProxy)}`, `https_proxy_value=${build.httpsProxy === null ? "''" : shQuote(build.httpsProxy)}`, `no_proxy_value=${shQuote(noProxy)}`, `env_identity_files=${shQuote(JSON.stringify(build.envIdentityFiles))}`, `build_args_json=${shQuote(JSON.stringify(buildArgs))}`, "mkdir -p \"$state_dir\"", "cd \"$workspace\"", "git checkout \"$source_commit\"", "env_identity=$(ENV_IDENTITY_FILES=\"$env_identity_files\" BUILD_ARGS_JSON=\"$build_args_json\" node <<'NODE'", "const { createHash } = require('node:crypto');", "const { readFileSync, existsSync, lstatSync, readdirSync } = require('node:fs');", "const { join } = require('node:path');", "const files = JSON.parse(process.env.ENV_IDENTITY_FILES || '[]');", "const buildArgs = JSON.parse(process.env.BUILD_ARGS_JSON || '[]');", "const hash = createHash('sha256');", "const skipDirNames = new Set(['.git', '.worktree', '.state', 'node_modules', 'coverage', 'tmp', '.tmp']);", "function collectIdentityFiles(input) {", " if (!existsSync(input)) return [{ path: input, missing: true }];", " const stat = lstatSync(input);", " if (stat.isFile()) return [{ path: input, missing: false }];", " if (!stat.isDirectory()) return [{ path: input, missing: true }];", " const out = [];", " const stack = [input];", " while (stack.length > 0) {", " const dir = stack.pop();", " const entries = readdirSync(dir, { withFileTypes: true }).sort((left, right) => left.name.localeCompare(right.name));", " for (const entry of entries) {", " if (entry.isDirectory() && skipDirNames.has(entry.name)) continue;", " const child = join(dir, entry.name);", " if (entry.isDirectory()) stack.push(child);", " else if (entry.isFile()) out.push({ path: child, missing: false });", " }", " }", " return out.sort((left, right) => left.path.localeCompare(right.path));", "}", "for (const item of buildArgs) { hash.update('build-arg'); hash.update('\\0'); hash.update(item); hash.update('\\0'); }", "for (const file of files) {", " for (const entry of collectIdentityFiles(file)) {", " hash.update(entry.path); hash.update('\\0');", " if (!entry.missing) hash.update(readFileSync(entry.path));", " hash.update('\\0');", " }", "}", "process.stdout.write(hash.digest('hex').slice(0, 24));", "NODE", ")", "source_short=$(printf '%s' \"$source_commit\" | cut -c1-12)", "job_id=\"$source_short-$env_identity\"", "status_file=\"$state_dir/$job_id.json\"", "stdout_file=\"$state_dir/$job_id.stdout.log\"", "stderr_file=\"$state_dir/$job_id.stderr.log\"", "cat > \"$status_file\" </dev/null | sed 's/\"/\\\\\"/g' | tr '\\n' ' ' | cut -c1-2000); CODE=\"$code\" ERROR_TAIL=\"$tail_text\" JOB_ID=\"$job_id\" SOURCE_COMMIT=\"$source_commit\" ENV_IDENTITY=\"$env_identity\" IMAGE_REPOSITORY=\"$image_repository\" node <<'NODE' > \"$status_file\"", "const code = Number(process.env.CODE || 1);", "console.log(JSON.stringify({ ok: false, status: 'failed', exitCode: code, jobId: process.env.JOB_ID, sourceCommit: process.env.SOURCE_COMMIT, envIdentity: process.env.ENV_IDENTITY, image: `${process.env.IMAGE_REPOSITORY}:${process.env.ENV_IDENTITY}`, errorTail: process.env.ERROR_TAIL || null, valuesPrinted: false }));", "NODE", " fi; exit \"$code\"; }", " trap write_failed_status EXIT", " cd \"$workspace\"", " image=\"$image_repository:$env_identity\"", " args=\"--network $network\"", " if [ -n \"$http_proxy_value\" ]; then args=\"$args --build-arg HTTP_PROXY=$http_proxy_value --build-arg http_proxy=$http_proxy_value\"; fi", " if [ -n \"$https_proxy_value\" ]; then args=\"$args --build-arg HTTPS_PROXY=$https_proxy_value --build-arg https_proxy=$https_proxy_value\"; fi", " if [ -n \"$no_proxy_value\" ]; then args=\"$args --build-arg NO_PROXY=$no_proxy_value --build-arg no_proxy=$no_proxy_value\"; fi", " build_arg_values=$(BUILD_ARGS_JSON=\"$build_args_json\" node <<'NODE'", "const values = JSON.parse(process.env.BUILD_ARGS_JSON || '[]');", "for (const value of values) console.log(value);", "NODE", " )", " while IFS= read -r build_arg_value; do [ -n \"$build_arg_value\" ] && args=\"$args --build-arg $build_arg_value\"; done </dev/null 2>&1; then build_status=reused; else docker build $args -f \"$containerfile\" -t \"$image\" \"$context_dir\"; build_status=built; fi", " manifest_accept='application/vnd.oci.image.index.v1+json, application/vnd.oci.image.manifest.v1+json, application/vnd.docker.distribution.manifest.v2+json'", " fetch_manifest_digest() {", " ref=\"$1\"", " attempt=1", " max_attempts=30", " while [ \"$attempt\" -le \"$max_attempts\" ]; do", " headers=$(mktemp)", " if curl -fsSI -H \"Accept: $manifest_accept\" \"http://127.0.0.1:5000/v2/${image_repository#127.0.0.1:5000/}/manifests/$ref\" >\"$headers\"; then", " found_digest=$(awk -F': ' 'tolower($1)==\"docker-content-digest\"{print $2}' \"$headers\" | tr -d '\\r' | head -n 1)", " rm -f \"$headers\"", " if [ -n \"$found_digest\" ]; then printf '%s' \"$found_digest\"; return 0; fi", " else", " rm -f \"$headers\"", " fi", " printf '{\"event\":\"agentrun-registry-manifest-probe\",\"status\":\"retrying\",\"ref\":\"%s\",\"attempt\":\"%s/%s\"}\\n' \"$ref\" \"$attempt\" \"$max_attempts\" >&2", " sleep 1", " attempt=$((attempt + 1))", " done", " printf '{\"event\":\"agentrun-registry-manifest-probe\",\"status\":\"exhausted\",\"ref\":\"%s\",\"attempt\":\"%s/%s\"}\\n' \"$ref\" \"$max_attempts\" \"$max_attempts\" >&2", " return 1", " }", " docker push \"$image\"", " digest=$(fetch_manifest_digest \"$env_identity\" || true)", " if [ -n \"$digest\" ]; then verified_digest=$(fetch_manifest_digest \"$digest\" || true); if [ \"$verified_digest\" != \"$digest\" ]; then digest=; fi; fi", " STATUS=\"$build_status\" DIGEST=\"$digest\" JOB_ID=\"$job_id\" SOURCE_COMMIT=\"$source_commit\" ENV_IDENTITY=\"$env_identity\" IMAGE_REPOSITORY=\"$image_repository\" node <<'NODE' > \"$status_file\"", "const digest = process.env.DIGEST || null;", "console.log(JSON.stringify({ ok: Boolean(digest), status: process.env.STATUS || 'built', jobId: process.env.JOB_ID, sourceCommit: process.env.SOURCE_COMMIT, envIdentity: process.env.ENV_IDENTITY, image: `${process.env.IMAGE_REPOSITORY}:${process.env.ENV_IDENTITY}`, digest, repositoryDigest: digest ? `${process.env.IMAGE_REPOSITORY}@${digest}` : null, valuesPrinted: false }));", "NODE", " trap - EXIT", ") >\"$stdout_file\" 2>\"$stderr_file\" &", "pid=$!", "JOB_PID=\"$pid\" JOB_ID=\"$job_id\" STATUS_FILE=\"$status_file\" STDOUT_FILE=\"$stdout_file\" STDERR_FILE=\"$stderr_file\" node <<'NODE'", "console.log(JSON.stringify({ ok: true, status: 'submitted', jobId: process.env.JOB_ID, pid: Number(process.env.JOB_PID), statusFile: process.env.STATUS_FILE, stdoutFile: process.env.STDOUT_FILE, stderrFile: process.env.STDERR_FILE, valuesPrinted: false }));", "NODE", ].join("\n"); return script; } export async function waitForYamlLaneBuildImage(config: UniDeskConfig, spec: AgentRunLaneSpec, sourceCommit: string, jobId: string | null): Promise & { ok: boolean; payload: Record }> { if (jobId === null) return { ok: false, payload: { ok: false, degradedReason: "build-job-id-missing", valuesPrinted: false } }; const startedAt = Date.now(); const timeoutMs = Math.max(60, spec.deployment.manager.imageBuild.timeoutSeconds) * 1000; let lastPayload: Record = {}; let polls = 0; while (Date.now() - startedAt < timeoutMs) { polls += 1; const probe = await capture(config, spec.nodeRoute, ["sh", "--", yamlLaneBuildImageStatusScript(spec, jobId)]); const payload = captureJsonPayload(probe); lastPayload = payload; progressEvent("agentrun.yaml-lane.image-build.progress", { node: spec.nodeId, lane: spec.lane, sourceCommit, jobId, polls, status: stringOrNull(payload.status) ?? "unknown", elapsedMs: Date.now() - startedAt, }); if (payload.ok === true && stringOrNull(payload.digest) !== null) return { ok: true, payload, polls, elapsedMs: Date.now() - startedAt }; if (payload.status === "failed") return { ok: false, payload, polls, elapsedMs: Date.now() - startedAt }; await sleep(spec.deployment.manager.imageBuild.pollSeconds * 1000); } return { ok: false, payload: { ...lastPayload, ok: false, status: "timeout", degradedReason: "image-build-timeout", valuesPrinted: false }, polls, elapsedMs: Date.now() - startedAt }; } export function yamlLaneBuildImageStatusScript(spec: AgentRunLaneSpec, jobId: string): string { const stateDir = `/tmp/unidesk-agentrun-build-${spec.nodeId}-${spec.lane}`; return [ "set +e", `status_file=${shQuote(`${stateDir}/${jobId}.json`)}`, `stdout_file=${shQuote(`${stateDir}/${jobId}.stdout.log`)}`, `stderr_file=${shQuote(`${stateDir}/${jobId}.stderr.log`)}`, "if [ -f \"$status_file\" ]; then cat \"$status_file\"; else printf '{\"ok\":false,\"status\":\"missing\",\"valuesPrinted\":false}\\n'; fi", "if [ -f \"$stderr_file\" ] && tail -n 20 \"$stderr_file\" | grep -Eq 'ERROR|error|failed|denied'; then :; fi", ].join("\n"); } export async function runYamlLaneGitopsPublishJob(config: UniDeskConfig, spec: AgentRunLaneSpec, sourceCommit: string, files: readonly { path: string; content: string }[]): Promise & { ok: boolean; payload: Record }> { const jobName = `gitops-publish-${spec.nodeId.toLowerCase()}-${spec.lane}-${Date.now().toString(36)}`.slice(0, 63); const manifest = yamlLaneGitopsPublishJobManifest(spec, files, jobName); const created = await capture(config, spec.nodeKubeRoute, ["sh", "--", createYamlLaneJobScript(spec.gitMirror.namespace, jobName, manifest)]); if (created.exitCode !== 0) { return { ok: false, payload: { ok: false, status: "create-failed", jobName, degradedReason: "gitops-publish-job-create-failed", valuesPrinted: false }, jobName, sourceCommit, phase: "create-job", capture: compactCapture(created, { full: true, stdoutTailChars: 5000, stderrTailChars: 5000 }), valuesPrinted: false, }; } const startedAt = Date.now(); const timeoutMs = 300_000; let polls = 0; let lastPayload: Record = {}; let lastProbe: SshCaptureResult | null = null; while (Date.now() - startedAt < timeoutMs) { polls += 1; lastProbe = await capture(config, spec.nodeKubeRoute, ["sh", "--", yamlLaneJobProbeScript(spec.gitMirror.namespace, jobName)]); const probePayload = captureJsonPayload(lastProbe); const publishPayload = yamlLaneGitopsPublishPayloadFromProbe(probePayload); if (Object.keys(publishPayload).length > 0) lastPayload = publishPayload; progressEvent("agentrun.yaml-lane.gitops-publish.progress", { node: spec.nodeId, lane: spec.lane, sourceCommit, jobName, polls, status: stringOrNull(publishPayload.status) ?? (probePayload.succeeded === true ? "succeeded" : probePayload.failed === true ? "failed" : "running"), gitopsCommit: stringOrNull(publishPayload.gitopsCommit), elapsedMs: Date.now() - startedAt, }); if (probePayload.succeeded === true) { if (publishPayload.ok === true && stringOrNull(publishPayload.gitopsCommit) !== null) { return { ok: true, payload: publishPayload, jobName, polls, elapsedMs: Date.now() - startedAt, probe: compactCapture(lastProbe), valuesPrinted: false }; } return { ok: false, payload: { ...publishPayload, ok: false, status: stringOrNull(publishPayload.status) ?? "succeeded-without-result", degradedReason: "gitops-publish-result-missing", jobName, valuesPrinted: false }, jobName, polls, elapsedMs: Date.now() - startedAt, probe: compactCapture(lastProbe, { full: true, stdoutTailChars: 5000, stderrTailChars: 5000 }), valuesPrinted: false, }; } if (probePayload.failed === true) { const payload = Object.keys(publishPayload).length > 0 ? publishPayload : { ok: false, status: "failed", degradedReason: "gitops-publish-job-failed", jobName, valuesPrinted: false }; return { ok: false, payload, jobName, polls, elapsedMs: Date.now() - startedAt, probe: compactCapture(lastProbe, { full: true, stdoutTailChars: 5000, stderrTailChars: 5000 }), valuesPrinted: false, }; } await sleep(5_000); } return { ok: false, payload: { ...lastPayload, ok: false, status: "timeout", degradedReason: "gitops-publish-job-timeout", jobName, valuesPrinted: false }, jobName, polls, elapsedMs: Date.now() - startedAt, probe: lastProbe === null ? null : compactCapture(lastProbe, { full: true, stdoutTailChars: 5000, stderrTailChars: 5000 }), valuesPrinted: false, }; }