// SPEC: PJ2026-01060307 控制面模块化 draft-2026-06-25-p0. yaml-lane module for scripts/src/agentrun.ts. // Moved mechanically from scripts/src/agentrun.ts:3311-4219 for #903. // SPEC: PJ2026-01020108 cancel lifecycle + PJ2026-01020205 AipodSpec binding + PJ2026-01020302 session policy + PJ2026-01020305 cancel control + PJ2026-01060305/06 YAML execution policy and bounded output draft-2026-06-25-p0. // Exposes AgentRun lane-scoped policy, AipodSpec SecretRef binding, cancel lifecycle, and bounded default output in the UniDesk CLI. import { chmodSync, copyFileSync, existsSync, readFileSync, statSync, writeFileSync } from "node:fs"; import { join } from "node:path"; import { spawnSync } from "node:child_process"; import { rootPath, type UniDeskConfig } from "../config"; import type { RenderedCliResult } from "../output"; import { applyLocalCaddyManagedSite } from "../pk01-caddy"; import { runSshCommandCapture, type SshCaptureResult } from "../ssh"; import { runRemoteSshCommandCapture } from "../remote"; import { startJob } from "../jobs"; import { AGENTRUN_CONFIG_PATH, agentRunLaneSummary, agentRunPipelineRunName, agentRunProviderCredentialRefs, agentRunSourceSnapshotRef, agentRunSourceSnapshotStageRefPrefix, resolveAgentRunLaneTarget, type AgentRunCancelLifecycleSpec, type AgentRunLaneSpec, } from "../agentrun-lanes"; import { agentRunImageArtifact, placeholderAgentRunImage, renderedFilesDigest, renderedObjectsDigest, renderAgentRunControlPlaneManifests, renderAgentRunGitopsFiles, type AgentRunArtifactService, } from "../agentrun-manifests"; import { sha256Fingerprint } from "../platform-infra-ops-library"; import type { CleanupReleasedPvOptions, CleanupRunnersOptions, CleanupRunsOptions, CleanupSessionPvcsOptions, RefreshOptions } from "./options"; import { cleanupReleasedPvsFinalizeNodeScript, cleanupReleasedPvsPlanNodeScript, cleanupRunnersFinalizeNodeScript, cleanupRunsFinalizeNodeScript, cleanupRunsPlanNodeScript, refreshYamlLaneScript } from "./git-mirror"; import { cleanupRunnersFactsNodeScript, cleanupRunnersPlanNodeScript, collectLaneSecretSources, createYamlLaneJobScript, yamlLaneGitopsPublishJobManifest, yamlLaneGitopsPublishPayloadFromProbe, yamlLaneJobProbeScript } from "./secrets"; import { capture, captureJsonPayload, compactCapture, progressEvent, shQuote, sleep, stringOrNull } from "./utils"; export async function refresh(config: UniDeskConfig, options: RefreshOptions): Promise> { return await refreshYamlLane(config, options); } export async function refreshYamlLane(config: UniDeskConfig, options: RefreshOptions): Promise> { const { configPath, spec } = resolveAgentRunLaneTarget(options); const plan = { node: spec.nodeId, lane: spec.lane, version: spec.version, argoNamespace: spec.gitops.argoNamespace, argoApplication: spec.gitops.argoApplication, gitopsBranch: spec.gitops.branch, mutation: "argocd-hard-refresh", valuesPrinted: false, }; if (options.dryRun || !options.confirm) { return { ok: true, command: "agentrun control-plane refresh", mode: "dry-run", mutation: false, configPath, target: agentRunLaneSummary(spec), plan, next: { confirm: `bun scripts/cli.ts agentrun control-plane refresh --node ${spec.nodeId} --lane ${spec.lane} --confirm`, status: `bun scripts/cli.ts agentrun control-plane status --node ${spec.nodeId} --lane ${spec.lane}`, }, valuesPrinted: false, }; } const refreshed = await capture(config, spec.nodeKubeRoute, ["sh", "--", refreshYamlLaneScript(spec)]); const payload = captureJsonPayload(refreshed); return { ok: refreshed.exitCode === 0 && payload.ok !== false, command: "agentrun control-plane refresh", mode: "confirmed-yaml-lane", mutation: true, configPath, target: agentRunLaneSummary(spec), plan, result: payload, refreshed: compactCapture(refreshed, { full: refreshed.exitCode !== 0, stdoutTailChars: 3000, stderrTailChars: 3000 }), next: { status: `bun scripts/cli.ts agentrun control-plane status --node ${spec.nodeId} --lane ${spec.lane}`, }, valuesPrinted: false, }; } export async function cleanupRunners(config: UniDeskConfig, options: CleanupRunnersOptions): Promise> { const { configPath, spec } = resolveAgentRunLaneTarget(options); const result = await capture(config, spec.nodeKubeRoute, ["sh", "--", cleanupRunnersScript(options, spec)]); const payload = captureJsonPayload(result); const ok = result.exitCode === 0 && payload.ok !== false; const base = { ...payload, ok, command: "agentrun control-plane cleanup-runners", configPath, target: agentRunLaneSummary(spec), mode: options.dryRun || !options.confirm ? "dry-run" : "confirmed-cleanup", namespace: spec.runtime.namespace, retention: spec.deployment.runner.retention, probe: compactCapture(result, { full: result.exitCode !== 0, stdoutTailChars: 3000, stderrTailChars: 3000 }), }; if (options.dryRun || !options.confirm) { return { ...base, dryRun: true, mutation: false, next: { confirm: `bun scripts/cli.ts agentrun control-plane cleanup-runners --node ${spec.nodeId} --lane ${spec.lane}${options.forceActive ? " --force-active" : ""} --confirm`, }, }; } return { ...base, dryRun: false, mutation: true, followUp: { dryRun: `bun scripts/cli.ts agentrun control-plane cleanup-runners --node ${spec.nodeId} --lane ${spec.lane}${options.forceActive ? " --force-active" : ""} --dry-run`, status: `bun scripts/cli.ts agentrun control-plane status --node ${spec.nodeId} --lane ${spec.lane}`, }, }; } export async function cleanupRuns(config: UniDeskConfig, options: CleanupRunsOptions): Promise> { const { configPath, spec } = resolveAgentRunLaneTarget(options); const result = await capture(config, spec.nodeKubeRoute, ["sh", "--", cleanupRunsScript(options, spec.ci.namespace, spec.ci.pipelineRunPrefix)]); const payload = captureJsonPayload(result); const ok = result.exitCode === 0 && payload.ok !== false; const base = { ...payload, ok, command: "agentrun control-plane cleanup-runs", configPath, target: agentRunLaneSummary(spec), mode: options.dryRun || !options.confirm ? "dry-run" : "confirmed-cleanup", namespace: spec.ci.namespace, minAgeMinutes: options.minAgeMinutes, limit: options.limit, probe: compactCapture(result, { full: result.exitCode !== 0, stdoutTailChars: 3000, stderrTailChars: 3000 }), }; if (options.dryRun || !options.confirm) { return { ...base, dryRun: true, mutation: false, next: { confirm: `bun scripts/cli.ts agentrun control-plane cleanup-runs --node ${spec.nodeId} --lane ${spec.lane} --min-age-minutes ${options.minAgeMinutes} --limit ${options.limit} --confirm`, }, }; } return { ...base, dryRun: false, mutation: true, followUp: { status: `bun scripts/cli.ts agentrun control-plane status --node ${spec.nodeId} --lane ${spec.lane}`, releasedPvs: `bun scripts/cli.ts agentrun control-plane cleanup-released-pvs --node ${spec.nodeId} --lane ${spec.lane} --limit ${options.limit} --dry-run`, diskPressure: `trans ${spec.nodeKubeRoute} kubectl get node -o jsonpath='{range .items[*]}{.metadata.name}{\"\\t\"}{range .status.conditions[*]}{.type}{\"=\"}{.status}{\" \"}{.reason}{\";\"}{end}{\"\\n\"}{end}'`, }, }; } export async function cleanupReleasedPvs(config: UniDeskConfig, options: CleanupReleasedPvOptions): Promise> { const { configPath, spec } = resolveAgentRunLaneTarget(options); const result = await capture(config, spec.nodeKubeRoute, ["sh", "--", cleanupReleasedPvsScript(options, spec.ci.namespace)]); const payload = captureJsonPayload(result); const ok = result.exitCode === 0 && payload.ok !== false; const base = { ...payload, ok, command: "agentrun control-plane cleanup-released-pvs", configPath, target: agentRunLaneSummary(spec), mode: options.dryRun || !options.confirm ? "dry-run" : "confirmed-cleanup", namespace: spec.ci.namespace, limit: options.limit, probe: compactCapture(result, { full: result.exitCode !== 0, stdoutTailChars: 3000, stderrTailChars: 3000 }), }; if (options.dryRun || !options.confirm) { return { ...base, dryRun: true, mutation: false, next: { confirm: `bun scripts/cli.ts agentrun control-plane cleanup-released-pvs --node ${spec.nodeId} --lane ${spec.lane} --limit ${options.limit} --confirm`, }, }; } return { ...base, dryRun: false, mutation: true, followUp: { cleanupRuns: `bun scripts/cli.ts agentrun control-plane cleanup-runs --node ${spec.nodeId} --lane ${spec.lane} --min-age-minutes 30 --limit ${options.limit} --dry-run`, diskPressure: `trans ${spec.nodeKubeRoute} kubectl get node -o jsonpath='{range .items[*]}{.metadata.name}{\"\\t\"}{range .status.conditions[*]}{.type}{\"=\"}{.status}{\" \"}{.reason}{\";\"}{end}{\"\\n\"}{end}'`, }, }; } export async function cleanupSessionPvcs(config: UniDeskConfig, options: CleanupSessionPvcsOptions): Promise> { const { configPath, spec } = resolveAgentRunLaneTarget(options); const result = await capture(config, spec.nodeKubeRoute, ["sh", "--", cleanupSessionPvcsScript(options, spec)]); const payload = captureJsonPayload(result); const ok = result.exitCode === 0 && payload.ok !== false; const base = { ...payload, ok, command: "agentrun control-plane cleanup-session-pvcs", configPath, target: { node: spec.nodeId, lane: spec.lane, namespace: spec.runtime.namespace }, mode: options.dryRun || !options.confirm ? "dry-run" : "confirmed-cleanup", namespace: spec.runtime.namespace, retention: spec.deployment.runner.retention.sessionPvcRetention, probe: result.exitCode === 0 ? undefined : compactCapture(result, { full: true, stdoutTailChars: 3000, stderrTailChars: 3000 }), }; if (options.dryRun || !options.confirm) { return { ...base, dryRun: true, mutation: false, next: { confirm: `bun scripts/cli.ts agentrun control-plane cleanup-session-pvcs --node ${spec.nodeId} --lane ${spec.lane} --limit ${options.limit} --confirm` } }; } return { ...base, dryRun: false, mutation: true, followUp: { dryRun: `bun scripts/cli.ts agentrun control-plane cleanup-session-pvcs --node ${spec.nodeId} --lane ${spec.lane} --limit ${options.limit} --dry-run`, diskPressure: `bun scripts/cli.ts gc remote ${spec.nodeId} status --limit 20`, }, }; } export function cleanupSessionPvcsScript(options: CleanupSessionPvcsOptions, spec: AgentRunLaneSpec): string { const retention = spec.deployment.runner.retention.sessionPvcRetention; const script = readFileSync(rootPath("scripts/src/agentrun/cleanup-session-pvcs.mjs"), "utf8"); return [ "set -eu", `namespace=${shQuote(spec.runtime.namespace)}`, `confirm=${options.confirm && !options.dryRun ? "true" : "false"}`, `limit=${String(Math.min(options.limit, retention.maxDeletePerRun))}`, `enabled=${retention.enabled ? "true" : "false"}`, `prefixes_json_b64=${shQuote(Buffer.from(JSON.stringify(retention.prefixes), "utf8").toString("base64"))}`, "tmp_dir=$(mktemp -d)", "trap 'rm -rf \"$tmp_dir\"' EXIT", "cat > \"$tmp_dir/cleanup-session-pvcs.mjs\" <<'NODE'", script, "NODE", "env NAMESPACE=\"$namespace\" CONFIRM=\"$confirm\" LIMIT=\"$limit\" ENABLED=\"$enabled\" PREFIXES_JSON_B64=\"$prefixes_json_b64\" node \"$tmp_dir/cleanup-session-pvcs.mjs\"", ].join("\n"); } export function cleanupRunnersScript(options: CleanupRunnersOptions, spec: AgentRunLaneSpec): string { const retention = spec.deployment.runner.retention; const matchLabelsB64 = Buffer.from(JSON.stringify(retention.selectors.matchLabels), "utf8").toString("base64"); const jobNamePrefixesB64 = Buffer.from(JSON.stringify(retention.selectors.jobNamePrefixes), "utf8").toString("base64"); return [ "set -eu", `namespace=${shQuote(spec.runtime.namespace)}`, `manager_deployment=${shQuote(spec.runtime.managerDeployment)}`, `max_runners=${String(retention.maxRunners)}`, `cleanup_order=${shQuote(retention.cleanupOrder)}`, `active_heartbeat_max_age_ms=${String(retention.activeHeartbeatMaxAgeMs)}`, `age_based_cleanup_enabled=${retention.ageBasedCleanup.enabled ? "true" : "false"}`, `age_based_max_age_hours=${retention.ageBasedCleanup.maxAgeHours === null ? "" : String(retention.ageBasedCleanup.maxAgeHours)}`, `timeout_seconds=${String(options.timeoutSeconds)}`, `force_active=${options.forceActive ? "true" : "false"}`, `match_labels_json_b64=${shQuote(matchLabelsB64)}`, `job_name_prefixes_json_b64=${shQuote(jobNamePrefixesB64)}`, "match_labels_json=$(printf '%s' \"$match_labels_json_b64\" | base64 -d)", "job_name_prefixes_json=$(printf '%s' \"$job_name_prefixes_json_b64\" | base64 -d)", "tmp_dir=$(mktemp -d)", "trap 'rm -rf \"$tmp_dir\"' EXIT", "kubectl -n \"$namespace\" get job -o json > \"$tmp_dir/jobs.json\"", "kubectl -n \"$namespace\" get pod -o json > \"$tmp_dir/pods.json\"", "facts_exit=0", "set +e", "kubectl -n \"$namespace\" exec -i deploy/\"$manager_deployment\" -- env RETENTION_NAMESPACE=\"$namespace\" sh -lc 'cat >/tmp/agentrun-runner-retention.mjs && bun /tmp/agentrun-runner-retention.mjs' > \"$tmp_dir/runner-facts.json\" 2> \"$tmp_dir/runner-facts.err\" <<'NODE'", cleanupRunnersFactsNodeScript(), "NODE", "facts_exit=$?", "set -e", "if [ \"$facts_exit\" -ne 0 ]; then", " printf '%s\\n' '{\"ok\":false,\"items\":[],\"failureKind\":\"manager-facts-unavailable\",\"valuesPrinted\":false}' > \"$tmp_dir/runner-facts.json\"", "fi", "MATCH_LABELS_JSON=\"$match_labels_json\" JOB_NAME_PREFIXES_JSON=\"$job_name_prefixes_json\" MAX_RUNNERS=\"$max_runners\" CLEANUP_ORDER=\"$cleanup_order\" ACTIVE_HEARTBEAT_MAX_AGE_MS=\"$active_heartbeat_max_age_ms\" AGE_BASED_CLEANUP_ENABLED=\"$age_based_cleanup_enabled\" AGE_BASED_MAX_AGE_HOURS=\"$age_based_max_age_hours\" FORCE_ACTIVE=\"$force_active\" FACTS_EXIT=\"$facts_exit\" TMP_DIR=\"$tmp_dir\" NAMESPACE=\"$namespace\" node <<'NODE' > \"$tmp_dir/plan.json\"", cleanupRunnersPlanNodeScript(), "NODE", "if [ " + shQuote(options.confirm && !options.dryRun ? "true" : "false") + " != true ]; then", " cat \"$tmp_dir/plan.json\"", " exit 0", "fi", "node -e 'const fs=require(\"node:fs\"); const plan=JSON.parse(fs.readFileSync(process.argv[1],\"utf8\")); const names=Array.isArray(plan.selectedRunnerJobs)?plan.selectedRunnerJobs:[]; fs.writeFileSync(process.argv[2], names.join(\"\\n\") + (names.length>0?\"\\n\":\"\"));' \"$tmp_dir/plan.json\" \"$tmp_dir/selected-jobs.txt\"", "delete_exit=0", "if [ -s \"$tmp_dir/selected-jobs.txt\" ]; then", " xargs -r kubectl -n \"$namespace\" delete job --ignore-not-found=true --wait=true --timeout=\"${timeout_seconds}s\" < \"$tmp_dir/selected-jobs.txt\" > \"$tmp_dir/delete.out\" 2> \"$tmp_dir/delete.err\" || delete_exit=$?", "else", " : > \"$tmp_dir/delete.out\"", " : > \"$tmp_dir/delete.err\"", "fi", "kubectl -n \"$namespace\" get job -o json > \"$tmp_dir/jobs-after.json\"", "kubectl -n \"$namespace\" get pod -o json > \"$tmp_dir/pods-after.json\"", "DELETE_EXIT=\"$delete_exit\" TMP_DIR=\"$tmp_dir\" node <<'NODE'", cleanupRunnersFinalizeNodeScript(), "NODE", ].join("\n"); } export function cleanupRunsScript(options: CleanupRunsOptions, namespace: string, pipelineRunPrefix: string): string { return [ "set -eu", `namespace=${shQuote(namespace)}`, `pipeline_run_prefix=${shQuote(pipelineRunPrefix)}`, `min_age_minutes=${String(options.minAgeMinutes)}`, `limit=${String(options.limit)}`, `timeout_seconds=${String(options.timeoutSeconds)}`, "tmp_dir=$(mktemp -d)", "trap 'rm -rf \"$tmp_dir\"' EXIT", "kubectl -n \"$namespace\" get pipelinerun -o json > \"$tmp_dir/pipelineruns.json\"", "kubectl -n \"$namespace\" get pvc -o json > \"$tmp_dir/pvcs.json\"", "kubectl get pv -o json > \"$tmp_dir/pvs.json\"", "kubectl -n \"$namespace\" get pod -o json > \"$tmp_dir/pods.json\"", "NAMESPACE=\"$namespace\" PIPELINE_RUN_PREFIX=\"$pipeline_run_prefix\" MIN_AGE_MINUTES=\"$min_age_minutes\" LIMIT=\"$limit\" TMP_DIR=\"$tmp_dir\" node <<'NODE' > \"$tmp_dir/plan.json\"", cleanupRunsPlanNodeScript(), "NODE", "if [ " + shQuote(options.confirm && !options.dryRun ? "true" : "false") + " != true ]; then", " cat \"$tmp_dir/plan.json\"", " exit 0", "fi", "node -e 'const fs=require(\"node:fs\"); const plan=JSON.parse(fs.readFileSync(process.argv[1],\"utf8\")); const names=Array.isArray(plan.selectedPipelineRuns)?plan.selectedPipelineRuns:[]; fs.writeFileSync(process.argv[2], names.join(\"\\n\") + (names.length>0?\"\\n\":\"\"));' \"$tmp_dir/plan.json\" \"$tmp_dir/selected-names.txt\"", "delete_exit=0", "if [ -s \"$tmp_dir/selected-names.txt\" ]; then", " xargs -r kubectl -n \"$namespace\" delete pipelinerun --ignore-not-found=true --wait=true --timeout=\"${timeout_seconds}s\" < \"$tmp_dir/selected-names.txt\" > \"$tmp_dir/delete.out\" 2> \"$tmp_dir/delete.err\" || delete_exit=$?", "else", " : > \"$tmp_dir/delete.out\"", " : > \"$tmp_dir/delete.err\"", "fi", "kubectl -n \"$namespace\" get pvc -o json > \"$tmp_dir/pvcs-after.json\"", "DELETE_EXIT=\"$delete_exit\" TMP_DIR=\"$tmp_dir\" node <<'NODE'", cleanupRunsFinalizeNodeScript(), "NODE", ].join("\n"); } export function cleanupReleasedPvsScript(options: CleanupReleasedPvOptions, namespace: string): string { return [ "set -eu", `namespace=${shQuote(namespace)}`, `limit=${String(options.limit)}`, `timeout_seconds=${String(options.timeoutSeconds)}`, "tmp_dir=$(mktemp -d)", "trap 'rm -rf \"$tmp_dir\"' EXIT", "kubectl get pv -o json > \"$tmp_dir/pvs.json\"", "NAMESPACE=\"$namespace\" LIMIT=\"$limit\" TMP_DIR=\"$tmp_dir\" node <<'NODE' > \"$tmp_dir/plan.json\"", cleanupReleasedPvsPlanNodeScript(), "NODE", "if [ " + shQuote(options.confirm && !options.dryRun ? "true" : "false") + " != true ]; then", " cat \"$tmp_dir/plan.json\"", " exit 0", "fi", "node -e 'const fs=require(\"node:fs\"); const plan=JSON.parse(fs.readFileSync(process.argv[1],\"utf8\")); const names=Array.isArray(plan.selectedPersistentVolumes)?plan.selectedPersistentVolumes:[]; fs.writeFileSync(process.argv[2], names.join(\"\\n\") + (names.length>0?\"\\n\":\"\"));' \"$tmp_dir/plan.json\" \"$tmp_dir/selected-pvs.txt\"", "delete_exit=0", "if [ -s \"$tmp_dir/selected-pvs.txt\" ]; then", " xargs -r kubectl delete pv --ignore-not-found=true --wait=true --timeout=\"${timeout_seconds}s\" < \"$tmp_dir/selected-pvs.txt\" > \"$tmp_dir/delete.out\" 2> \"$tmp_dir/delete.err\" || delete_exit=$?", "else", " : > \"$tmp_dir/delete.out\"", " : > \"$tmp_dir/delete.err\"", "fi", "kubectl get pv -o json > \"$tmp_dir/pvs-after.json\"", "DELETE_EXIT=\"$delete_exit\" TMP_DIR=\"$tmp_dir\" node <<'NODE'", cleanupReleasedPvsFinalizeNodeScript(), "NODE", ].join("\n"); } export function yamlLaneSourceStatusScript(spec: AgentRunLaneSpec): string { return [ "set +e", `expected_workspace=${shQuote(spec.source.workspace)}`, `source_branch=${shQuote(spec.source.branch)}`, "workspace_exists=false", "workspace_clean=null", "local_head=null", "branch=null", "remote_url=null", "remote_branch_exists=false", "remote_branch_commit=null", "status_short=''", "git config --global --add safe.directory \"$expected_workspace\" 2>/dev/null || true", "if [ -d .git ] || git rev-parse --show-toplevel >/dev/null 2>&1; then", " actual_workspace=$(pwd)", " workspace_exists=true", " branch=$(git rev-parse --abbrev-ref HEAD 2>/dev/null || true)", " remote_url=$(git remote get-url origin 2>/dev/null || true)", " local_head=$(git rev-parse HEAD 2>/dev/null || true)", " status_short=$(git status --short 2>/dev/null || true)", " if [ -z \"$status_short\" ]; then workspace_clean=true; else workspace_clean=false; fi", " git fetch origin \"$source_branch\" >/dev/null 2>&1 || true", " remote_branch_commit=$(git rev-parse \"refs/remotes/origin/$source_branch\" 2>/dev/null || true)", " if [ -n \"$remote_branch_commit\" ]; then remote_branch_exists=true; fi", "else", " actual_workspace=$(pwd)", "fi", "export expected_workspace source_branch workspace_exists workspace_clean local_head branch remote_url remote_branch_exists remote_branch_commit status_short actual_workspace", "python3 - <<'PY'", "import json, os", "def nullable(value):", " return value if value and value != 'null' else None", "def boolean_value(value):", " if value == 'true': return True", " if value == 'false': return False", " return None", "env = os.environ", "print(json.dumps({", " 'ok': env.get('workspace_exists') == 'true',", " 'statusMode': 'host-worktree',", " 'expectedWorkspace': env.get('expected_workspace'),", " 'actualWorkspace': env.get('actual_workspace') or None,", " 'workspaceExists': env.get('workspace_exists') == 'true',", " 'workspaceClean': boolean_value(env.get('workspace_clean')),", " 'branch': nullable(env.get('branch')),", " 'remoteUrl': nullable(env.get('remote_url')),", " 'localHead': nullable(env.get('local_head')),", " 'remoteBranch': env.get('source_branch'),", " 'remoteBranchExists': env.get('remote_branch_exists') == 'true',", " 'remoteBranchCommit': nullable(env.get('remote_branch_commit')),", " 'statusShort': nullable(env.get('status_short')),", " 'valuesPrinted': False,", "}, ensure_ascii=False))", "PY", ].join("\n"); } export function yamlLaneK3sSourceStatusScript(spec: AgentRunLaneSpec): string { return [ "set +e", `namespace=${shQuote(spec.gitMirror.namespace)}`, `read_deployment=${shQuote(spec.gitMirror.readDeployment)}`, `repository=${shQuote(spec.source.repository)}`, `source_branch=${shQuote(spec.source.branch)}`, `source_stage_ref_prefix=${shQuote(agentRunSourceSnapshotStageRefPrefix(spec))}`, "repo_path=\"/cache/${repository}.git\"", "kubectl -n \"$namespace\" get deploy \"$read_deployment\" >/tmp/agentrun-source-read-deploy.txt 2>/dev/null", "read_exit=$?", "source_commit=''", "source_ref=\"refs/mirror-stage/heads/$source_branch\"", "source_stage_ref=''", "snapshot_commit=''", "if [ \"$read_exit\" -eq 0 ]; then", " refs=$(kubectl -n \"$namespace\" exec deploy/\"$read_deployment\" -- sh -lc 'repo_path=\"$1\"; source_branch=\"$2\"; stage_ref_prefix=\"$3\"; source_ref=\"refs/mirror-stage/heads/$source_branch\"; source_commit=$(git --git-dir=\"$repo_path\" rev-parse --verify \"$source_ref^{commit}\" 2>/dev/null || true); source_stage_ref=\"\"; snapshot_commit=\"\"; if printf \"%s\" \"$source_commit\" | grep -Eq \"^[0-9a-fA-F]{40}$\"; then source_stage_ref=\"${stage_ref_prefix%/}/$source_commit\"; snapshot_commit=$(git --git-dir=\"$repo_path\" rev-parse --verify \"$source_stage_ref^{commit}\" 2>/dev/null || true); fi; printf \"source_commit=%s\\nsource_stage_ref=%s\\nsnapshot_commit=%s\\n\" \"$source_commit\" \"$source_stage_ref\" \"$snapshot_commit\"' sh \"$repo_path\" \"$source_branch\" \"$source_stage_ref_prefix\" 2>/dev/null)", " source_commit=$(printf '%s\\n' \"$refs\" | sed -n 's/^source_commit=//p' | tail -n 1 | tr -d '\\r')", " source_stage_ref=$(printf '%s\\n' \"$refs\" | sed -n 's/^source_stage_ref=//p' | tail -n 1 | tr -d '\\r')", " snapshot_commit=$(printf '%s\\n' \"$refs\" | sed -n 's/^snapshot_commit=//p' | tail -n 1 | tr -d '\\r')", "fi", "export namespace read_deployment repository source_branch read_exit source_commit source_ref source_stage_ref snapshot_commit", "python3 - <<'PY'", "import json, os, re", "def sha(value):", " return (value or '').lower() if re.match(r'^[0-9a-f]{40}$', value or '', re.I) else None", "source_commit = sha(os.environ.get('source_commit'))", "snapshot_commit = sha(os.environ.get('snapshot_commit'))", "source_stage_ref = os.environ.get('source_stage_ref') or None", "read_ready = os.environ.get('read_exit') == '0'", "snapshot_present = source_commit is not None and snapshot_commit == source_commit", "degraded_reason = None", "if not read_ready:", " degraded_reason = 'git-mirror-read-not-ready'", "elif source_commit is None:", " degraded_reason = 'source-branch-missing'", "elif not snapshot_present:", " degraded_reason = 'source-snapshot-missing'", "print(json.dumps({", " 'ok': read_ready and snapshot_present,", " 'statusMode': 'k3s-git-mirror',", " 'sourceAuthority': 'git-mirror-snapshot',", " 'expectedWorkspace': None,", " 'actualWorkspace': None,", " 'workspaceExists': None,", " 'workspaceClean': None,", " 'branch': os.environ.get('source_branch'),", " 'remoteUrl': None,", " 'localHead': source_commit,", " 'remoteBranch': os.environ.get('source_branch'),", " 'remoteBranchExists': source_commit is not None,", " 'remoteBranchCommit': source_commit,", " 'sourceCommit': snapshot_commit if snapshot_present else source_commit,", " 'sourceRef': os.environ.get('source_ref'),", " 'sourceStageRef': source_stage_ref,", " 'snapshotPresent': snapshot_present,", " 'degradedReason': degraded_reason,", " 'gitMirror': {", " 'namespace': os.environ.get('namespace'),", " 'readDeployment': os.environ.get('read_deployment'),", " 'readReady': read_ready,", " 'repository': os.environ.get('repository'),", " 'sourceRef': os.environ.get('source_ref'),", " 'sourceStageRef': source_stage_ref,", " },", " 'statusShort': None,", " 'valuesPrinted': False,", "}, ensure_ascii=False))", "PY", ].join("\n"); } export function yamlLaneRuntimeStatusScript(spec: AgentRunLaneSpec, pipelineRun: string | null): string { return [ "set +e", `runtime_namespace=${shQuote(spec.runtime.namespace)}`, `ci_namespace=${shQuote(spec.ci.namespace)}`, `pipeline_name=${shQuote(spec.ci.pipeline)}`, `pipeline_run=${pipelineRun === null ? "''" : shQuote(pipelineRun)}`, `service_account=${shQuote(spec.ci.serviceAccountName)}`, `argo_namespace=${shQuote(spec.gitops.argoNamespace)}`, `argo_application=${shQuote(spec.gitops.argoApplication)}`, `manager_deployment=${shQuote(spec.runtime.managerDeployment)}`, `manager_service=${shQuote(spec.runtime.managerService)}`, `database_secret=${shQuote(spec.database.secretRef.name)}`, `database_key=${shQuote(spec.database.secretRef.key)}`, `secrets_json=${shQuote(JSON.stringify(collectLaneSecretSources(spec).map((item) => item.targetRef)))}`, "export runtime_namespace ci_namespace pipeline_name pipeline_run service_account argo_namespace argo_application manager_deployment manager_service database_secret database_key secrets_json", "tmp_dir=$(mktemp -d)", "trap 'rm -rf \"$tmp_dir\"' EXIT", "kubectl get ns \"$runtime_namespace\" -o json > \"$tmp_dir/runtime-ns.json\" 2>/dev/null", "runtime_ns_exit=$?", "kubectl get ns \"$ci_namespace\" -o json > \"$tmp_dir/ci-ns.json\" 2>/dev/null", "ci_ns_exit=$?", "kubectl -n \"$ci_namespace\" get pipeline \"$pipeline_name\" -o json > \"$tmp_dir/pipeline.json\" 2>/dev/null", "pipeline_exit=$?", "kubectl -n \"$ci_namespace\" get serviceaccount \"$service_account\" -o json > \"$tmp_dir/serviceaccount.json\" 2>/dev/null", "sa_exit=$?", "if [ -n \"$pipeline_run\" ]; then kubectl -n \"$ci_namespace\" get pipelinerun \"$pipeline_run\" -o json > \"$tmp_dir/pipelinerun.json\" 2>/dev/null; pr_exit=$?; else pr_exit=2; fi", "kubectl -n \"$argo_namespace\" get application \"$argo_application\" -o json > \"$tmp_dir/argo.json\" 2>/dev/null", "argo_exit=$?", "kubectl -n \"$runtime_namespace\" get deploy \"$manager_deployment\" -o json > \"$tmp_dir/manager-deploy.json\" 2>/dev/null", "manager_deploy_exit=$?", "kubectl -n \"$runtime_namespace\" get svc \"$manager_service\" -o json > \"$tmp_dir/manager-svc.json\" 2>/dev/null", "manager_svc_exit=$?", "kubectl -n \"$runtime_namespace\" get secret \"$database_secret\" -o json > \"$tmp_dir/db-secret.json\" 2>/dev/null", "db_secret_exit=$?", "SECRET_REFS_JSON=\"$secrets_json\" python3 - <<'PY' > \"$tmp_dir/secrets.json\"", "import json, os, subprocess", "refs = json.loads(os.environ.get('SECRET_REFS_JSON') or '[]')", "result = []", "for ref in refs:", " out = subprocess.run(['kubectl', '-n', ref.get('namespace', ''), 'get', 'secret', ref.get('name', ''), '-o', 'json'], text=True, capture_output=True)", " key_present = False", " if out.returncode == 0:", " try:", " key_present = ref.get('key') in (json.loads(out.stdout).get('data') or {})", " except Exception:", " key_present = False", " result.append({'namespace': ref.get('namespace'), 'name': ref.get('name'), 'key': ref.get('key'), 'present': out.returncode == 0, 'keyPresent': key_present, 'valuesPrinted': False})", "print(json.dumps({'ready': all(item['present'] and item['keyPresent'] for item in result), 'count': len(result), 'items': result, 'valuesPrinted': False}, ensure_ascii=False))", "PY", "kubectl -n \"$runtime_namespace\" get deploy,sts,svc,secret -o name > \"$tmp_dir/runtime-names.txt\" 2>/dev/null", "NODE_TMP=\"$tmp_dir\" RUNTIME_NS_EXIT=\"$runtime_ns_exit\" CI_NS_EXIT=\"$ci_ns_exit\" PIPELINE_EXIT=\"$pipeline_exit\" SERVICE_ACCOUNT_EXIT=\"$sa_exit\" PIPELINERUN_EXIT=\"$pr_exit\" ARGO_EXIT=\"$argo_exit\" MANAGER_DEPLOY_EXIT=\"$manager_deploy_exit\" MANAGER_SVC_EXIT=\"$manager_svc_exit\" DB_SECRET_EXIT=\"$db_secret_exit\" python3 - <<'PY'", "import json, os, pathlib, re", "base = pathlib.Path(os.environ['NODE_TMP'])", "def read_json(name):", " try: return json.loads((base / name).read_text())", " except Exception: return None", "def exists(name): return os.environ.get(name) == '0'", "def condition(obj):", " conds = (((obj or {}).get('status') or {}).get('conditions') or [])", " return conds[0] if conds else {}", "def env_value(deploy, name):", " containers = (((((deploy or {}).get('spec') or {}).get('template') or {}).get('spec') or {}).get('containers') or [])", " envs = (containers[0].get('env') or []) if containers else []", " for entry in envs:", " if entry.get('name') == name: return entry.get('value')", " return None", "def pipeline_run_param(obj, name):", " for entry in (((obj or {}).get('spec') or {}).get('params') or []):", " if entry.get('name') == name: return entry.get('value')", " return None", "pipeline_run = read_json('pipelinerun.json')", "argo = read_json('argo.json')", "manager_deploy = read_json('manager-deploy.json')", "manager_svc = read_json('manager-svc.json')", "db_secret = read_json('db-secret.json')", "secrets = read_json('secrets.json') or {'ready': True, 'count': 0, 'items': [], 'valuesPrinted': False}", "try: names = (base / 'runtime-names.txt').read_text()", "except Exception: names = ''", "c = condition(pipeline_run)", "ports = []", "for port in ((((manager_svc or {}).get('spec') or {}).get('ports')) or []):", " ports.append({'name': port.get('name'), 'port': port.get('port'), 'targetPort': port.get('targetPort')})", "matching_postgres = [line for line in re.split(r'\\r?\\n', names) if re.search('postgres', line, re.I)][:20]", "print(json.dumps({", " 'ok': exists('RUNTIME_NS_EXIT') and exists('CI_NS_EXIT'),", " 'runtimeNamespaceExists': exists('RUNTIME_NS_EXIT'),", " 'ciNamespaceExists': exists('CI_NS_EXIT'),", " 'serviceAccountExists': exists('SERVICE_ACCOUNT_EXIT'),", " 'pipeline': {'exists': exists('PIPELINE_EXIT'), 'name': os.environ.get('pipeline_name')},", " 'pipelineRun': {'exists': exists('PIPELINERUN_EXIT'), 'name': os.environ.get('pipeline_run') or None, 'status': c.get('status'), 'reason': c.get('reason'), 'sourceCommit': pipeline_run_param(pipeline_run, 'revision'), 'startTime': ((pipeline_run or {}).get('status') or {}).get('startTime'), 'completionTime': ((pipeline_run or {}).get('status') or {}).get('completionTime')},", " 'argo': {'exists': exists('ARGO_EXIT'), 'namespace': os.environ.get('argo_namespace'), 'application': os.environ.get('argo_application'), 'revision': (((argo or {}).get('status') or {}).get('sync') or {}).get('revision'), 'syncStatus': (((argo or {}).get('status') or {}).get('sync') or {}).get('status'), 'healthStatus': (((argo or {}).get('status') or {}).get('health') or {}).get('status')},", " 'manager': {'deploymentExists': exists('MANAGER_DEPLOY_EXIT'), 'serviceExists': exists('MANAGER_SVC_EXIT'), 'deployment': os.environ.get('manager_deployment'), 'service': os.environ.get('manager_service'), 'image': ((((((manager_deploy or {}).get('spec') or {}).get('template') or {}).get('spec') or {}).get('containers') or [{}])[0]).get('image'), 'sourceCommit': env_value(manager_deploy, 'AGENTRUN_SOURCE_COMMIT'), 'servicePorts': ports},", " 'database': {'secretPresent': exists('DB_SECRET_EXIT'), 'secretName': os.environ.get('database_secret'), 'key': os.environ.get('database_key'), 'keyPresent': os.environ.get('database_key') in (((db_secret or {}).get('data') or {})), 'valuesPrinted': False},", " 'secrets': secrets,", " 'localPostgres': {'absent': len(matching_postgres) == 0, 'matchingObjects': matching_postgres},", " 'valuesPrinted': False,", "}, ensure_ascii=False))", "PY", ].join("\n"); } export function yamlLaneSourceBootstrapProbeScript(spec: AgentRunLaneSpec): string { return [ "set +e", `workspace=${shQuote(spec.source.workspace)}`, `remote=${shQuote(spec.source.remote)}`, `branch=${shQuote(spec.source.branch)}`, "workspace_exists=false", "remote_branch_exists=false", "source_commit=null", "if [ -d \"$workspace/.git\" ]; then", " workspace_exists=true", " cd \"$workspace\"", " remote_branch=$(git ls-remote --heads \"$remote\" \"$branch\" 2>/dev/null | awk '{print $1}' | head -n 1)", " if [ -n \"$remote_branch\" ]; then remote_branch_exists=true; source_commit=\"$remote_branch\"; else source_commit=$(git rev-parse HEAD 2>/dev/null || printf null); fi", "else", " remote_branch=$(git ls-remote --heads \"$remote\" \"$branch\" 2>/dev/null | awk '{print $1}' | head -n 1)", " if [ -n \"$remote_branch\" ]; then remote_branch_exists=true; source_commit=\"$remote_branch\"; fi", "fi", "export workspace_exists remote_branch_exists source_commit", "node <<'NODE'", "console.log(JSON.stringify({ ok: true, workspaceExists: process.env.workspace_exists === 'true', remoteBranchExists: process.env.remote_branch_exists === 'true', sourceCommit: process.env.source_commit === 'null' ? null : process.env.source_commit, valuesPrinted: false }));", "NODE", ].join("\n"); } export function yamlLaneSourceBootstrapSubmitScript(spec: AgentRunLaneSpec): string { const bootstrap = spec.source.bootstrapFromBranch ?? spec.source.branch; const stateDir = `/tmp/unidesk-agentrun-source-${spec.nodeId}-${spec.lane}`; return [ "set -eu", `workspace=${shQuote(spec.source.workspace)}`, `remote=${shQuote(spec.source.remote)}`, `branch=${shQuote(spec.source.branch)}`, `bootstrap_branch=${shQuote(bootstrap)}`, `state_dir=${shQuote(stateDir)}`, "mkdir -p \"$state_dir\" \"$(dirname \"$workspace\")\"", "git_user=''", "git_home=''", "case \"$workspace\" in", " /home/*/*)", " git_user=${workspace#/home/}", " git_user=${git_user%%/*}", " git_home=/home/$git_user", " if ! id \"$git_user\" >/dev/null 2>&1; then git_user=''; git_home=''; fi", " ;;", "esac", "if [ -n \"$git_user\" ]; then chown \"$git_user:$git_user\" \"$(dirname \"$workspace\")\" 2>/dev/null || true; fi", "case \"$remote\" in", " git@*:*)", " if [ -n \"$git_user\" ] && [ -f \"$git_home/.ssh/id_ed25519\" ]; then", " if [ -z \"${GIT_SSH_COMMAND:-}\" ]; then GIT_SSH_COMMAND=\"ssh -i $git_home/.ssh/id_ed25519 -o IdentitiesOnly=yes -o StrictHostKeyChecking=accept-new -o UserKnownHostsFile=$git_home/.ssh/known_hosts\"; fi", " export GIT_SSH_COMMAND", " else", " mkdir -p \"$HOME/.ssh\"", " chmod 700 \"$HOME/.ssh\" 2>/dev/null || true", " if [ -z \"${GIT_SSH_COMMAND:-}\" ]; then GIT_SSH_COMMAND=\"ssh -o StrictHostKeyChecking=accept-new\"; fi", " export GIT_SSH_COMMAND", " fi", " ;;", " ssh://*)", " if [ -n \"$git_user\" ] && [ -f \"$git_home/.ssh/id_ed25519\" ]; then", " if [ -z \"${GIT_SSH_COMMAND:-}\" ]; then GIT_SSH_COMMAND=\"ssh -i $git_home/.ssh/id_ed25519 -o IdentitiesOnly=yes -o StrictHostKeyChecking=accept-new -o UserKnownHostsFile=$git_home/.ssh/known_hosts\"; fi", " export GIT_SSH_COMMAND", " else", " mkdir -p \"$HOME/.ssh\"", " chmod 700 \"$HOME/.ssh\" 2>/dev/null || true", " if [ -z \"${GIT_SSH_COMMAND:-}\" ]; then GIT_SSH_COMMAND=\"ssh -o StrictHostKeyChecking=accept-new\"; fi", " export GIT_SSH_COMMAND", " fi", " ;;", "esac", "git_cmd() {", " if [ -n \"$git_user\" ]; then", " sudo -u \"$git_user\" env HOME=\"$git_home\" GIT_SSH_COMMAND=\"$GIT_SSH_COMMAND\" git \"$@\"", " else", " git \"$@\"", " fi", "}", "job_id=\"source-bootstrap-$(date +%s)-$$\"", "status_file=\"$state_dir/$job_id.json\"", "stdout_file=\"$state_dir/$job_id.stdout.log\"", "stderr_file=\"$state_dir/$job_id.stderr.log\"", "cat > \"$status_file\" </dev/null | sed 's/\"/\\\\\"/g' | tr '\\n' ' ' | cut -c1-3000); CODE=\"$code\" ERROR_TAIL=\"$tail_text\" JOB_ID=\"$job_id\" WORKSPACE=\"$workspace\" BRANCH=\"$branch\" node <<'NODE' > \"$status_file\"", "const code = Number(process.env.CODE || 1);", "console.log(JSON.stringify({ ok: false, status: 'failed', exitCode: code, jobId: process.env.JOB_ID, workspace: process.env.WORKSPACE, branch: process.env.BRANCH, errorTail: process.env.ERROR_TAIL || null, valuesPrinted: false }));", "NODE", " fi; exit \"$code\"; }", " trap write_failed_status EXIT", " if [ -d \"$workspace/.git\" ] && git_cmd -C \"$workspace\" rev-parse --git-dir >/dev/null 2>&1; then", " :", " else", " rm -rf \"$workspace\"", " git_cmd clone --no-checkout \"$remote\" \"$workspace\"", " fi", " cd \"$workspace\"", " git_cmd remote set-url origin \"$remote\" || git_cmd remote add origin \"$remote\"", " git_cmd fetch origin \"$bootstrap_branch\" \"$branch\" || git_cmd fetch origin \"$bootstrap_branch\"", " if git_cmd rev-parse --verify \"refs/remotes/origin/$branch^{commit}\" >/dev/null 2>&1; then", " git_cmd checkout -B \"$branch\" \"refs/remotes/origin/$branch\"", " else", " git_cmd checkout -B \"$branch\" \"refs/remotes/origin/$bootstrap_branch\"", " fi", " if [ -f deploy/deploy.json ]; then rm deploy/deploy.json; fi", " git_cmd add -A deploy/deploy.json 2>/dev/null || true", " if ! git_cmd diff --quiet --cached -- deploy/deploy.json 2>/dev/null; then", " git_cmd -c user.email=agentrun@unidesk.local -c user.name='UniDesk AgentRun Ops' commit -m 'chore: remove service deploy json truth'", " fi", " git_cmd push -u origin \"$branch\"", " source_commit=$(git_cmd rev-parse HEAD)", " status_short=$(git_cmd status --short)", " SOURCE_COMMIT=\"$source_commit\" STATUS_SHORT=\"$status_short\" JOB_ID=\"$job_id\" WORKSPACE=\"$workspace\" BRANCH=\"$branch\" node <<'NODE' > \"$status_file\"", "console.log(JSON.stringify({ ok: process.env.STATUS_SHORT === '', status: 'succeeded', jobId: process.env.JOB_ID, workspace: process.env.WORKSPACE, branch: process.env.BRANCH, sourceCommit: process.env.SOURCE_COMMIT, workspaceClean: process.env.STATUS_SHORT === '', statusShort: process.env.STATUS_SHORT || null, removedServiceDeployJson: true, valuesPrinted: false }));", "NODE", " trap - EXIT", ") >\"$stdout_file\" 2>\"$stderr_file\" &", "pid=$!", "JOB_PID=\"$pid\" JOB_ID=\"$job_id\" STATUS_FILE=\"$status_file\" STDOUT_FILE=\"$stdout_file\" STDERR_FILE=\"$stderr_file\" node <<'NODE'", "console.log(JSON.stringify({ ok: true, status: 'submitted', jobId: process.env.JOB_ID, pid: Number(process.env.JOB_PID), statusFile: process.env.STATUS_FILE, stdoutFile: process.env.STDOUT_FILE, stderrFile: process.env.STDERR_FILE, valuesPrinted: false }));", "NODE", ].join("\n"); } export async function waitForYamlLaneSourceBootstrap(config: UniDeskConfig, spec: AgentRunLaneSpec, jobId: string | null): Promise & { ok: boolean; payload: Record }> { if (jobId === null) return { ok: false, payload: { ok: false, degradedReason: "source-bootstrap-job-id-missing", valuesPrinted: false } }; const startedAt = Date.now(); const timeoutMs = spec.source.bootstrapTimeoutSeconds * 1000; const pollMs = spec.source.bootstrapPollSeconds * 1000; let lastPayload: Record = {}; let polls = 0; while (Date.now() - startedAt < timeoutMs) { polls += 1; const probe = await capture(config, spec.nodeRoute, ["sh", "--", yamlLaneSourceBootstrapStatusScript(spec, jobId)]); const payload = captureJsonPayload(probe); lastPayload = payload; progressEvent("agentrun.yaml-lane.source-bootstrap.progress", { node: spec.nodeId, lane: spec.lane, jobId, polls, status: stringOrNull(payload.status) ?? "unknown", sourceCommit: stringOrNull(payload.sourceCommit), elapsedMs: Date.now() - startedAt, }); if (payload.status === "succeeded" && stringOrNull(payload.sourceCommit) !== null) return { ok: payload.ok === true, payload, polls, elapsedMs: Date.now() - startedAt }; if (payload.status === "failed") return { ok: false, payload, polls, elapsedMs: Date.now() - startedAt }; await sleep(pollMs); } return { ok: false, payload: { ...lastPayload, ok: false, status: "timeout", degradedReason: "source-bootstrap-timeout", valuesPrinted: false }, polls, elapsedMs: Date.now() - startedAt }; } export function yamlLaneSourceBootstrapStatusScript(spec: AgentRunLaneSpec, jobId: string): string { const stateDir = `/tmp/unidesk-agentrun-source-${spec.nodeId}-${spec.lane}`; return [ "set +e", `status_file=${shQuote(`${stateDir}/${jobId}.json`)}`, "if [ -f \"$status_file\" ]; then cat \"$status_file\"; else printf '{\"ok\":false,\"status\":\"missing\",\"valuesPrinted\":false}\\n'; fi", ].join("\n"); } export function yamlLaneSourceRestoreScript(spec: AgentRunLaneSpec): string { return [ "set +e", `workspace=${shQuote(spec.source.workspace)}`, `remote=${shQuote(spec.source.remote)}`, `branch=${shQuote(spec.source.branch)}`, "tmp_dir=$(mktemp -d)", "trap 'rm -rf \"$tmp_dir\"' EXIT", "git_user=''", "git_home=''", "case \"$workspace\" in", " /home/*/*)", " git_user=${workspace#/home/}", " git_user=${git_user%%/*}", " git_home=/home/$git_user", " if ! id \"$git_user\" >/dev/null 2>&1; then git_user=''; git_home=''; fi", " ;;", "esac", "case \"$remote\" in", " git@*:*)", " if [ -n \"$git_user\" ] && [ -f \"$git_home/.ssh/id_ed25519\" ]; then", " if [ -z \"${GIT_SSH_COMMAND:-}\" ]; then GIT_SSH_COMMAND=\"ssh -i $git_home/.ssh/id_ed25519 -o IdentitiesOnly=yes -o StrictHostKeyChecking=accept-new -o UserKnownHostsFile=$git_home/.ssh/known_hosts\"; fi", " export GIT_SSH_COMMAND", " else", " mkdir -p \"$HOME/.ssh\"", " chmod 700 \"$HOME/.ssh\" 2>/dev/null || true", " if [ -z \"${GIT_SSH_COMMAND:-}\" ]; then GIT_SSH_COMMAND=\"ssh -o StrictHostKeyChecking=accept-new\"; fi", " export GIT_SSH_COMMAND", " fi", " ;;", " ssh://*)", " if [ -n \"$git_user\" ] && [ -f \"$git_home/.ssh/id_ed25519\" ]; then", " if [ -z \"${GIT_SSH_COMMAND:-}\" ]; then GIT_SSH_COMMAND=\"ssh -i $git_home/.ssh/id_ed25519 -o IdentitiesOnly=yes -o StrictHostKeyChecking=accept-new -o UserKnownHostsFile=$git_home/.ssh/known_hosts\"; fi", " export GIT_SSH_COMMAND", " else", " mkdir -p \"$HOME/.ssh\"", " chmod 700 \"$HOME/.ssh\" 2>/dev/null || true", " if [ -z \"${GIT_SSH_COMMAND:-}\" ]; then GIT_SSH_COMMAND=\"ssh -o StrictHostKeyChecking=accept-new\"; fi", " export GIT_SSH_COMMAND", " fi", " ;;", "esac", "git_cmd() {", " if [ -n \"$git_user\" ]; then", " sudo -u \"$git_user\" env HOME=\"$git_home\" GIT_SSH_COMMAND=\"$GIT_SSH_COMMAND\" git \"$@\"", " else", " git \"$@\"", " fi", "}", "workspace_exists=false", "git config --global --add safe.directory \"$workspace\" 2>/dev/null || true", "git_cmd config --global --add safe.directory \"$workspace\" 2>/dev/null || true", "if git_cmd -C \"$workspace\" rev-parse --git-dir >/dev/null 2>&1; then workspace_exists=true; fi", "if [ \"$workspace_exists\" != true ]; then", " WORKSPACE=\"$workspace\" BRANCH=\"$branch\" node <<'NODE'", "console.log(JSON.stringify({ ok: false, status: 'skipped', failureKind: 'source-worktree-missing', workspace: process.env.WORKSPACE, branch: process.env.BRANCH, valuesPrinted: false }));", "NODE", " exit 0", "fi", "before_branch=$(git_cmd -C \"$workspace\" rev-parse --abbrev-ref HEAD 2>/dev/null || true)", "before_head=$(git_cmd -C \"$workspace\" rev-parse HEAD 2>/dev/null || true)", "status_short=$(git_cmd -C \"$workspace\" status --short 2>/dev/null || true)", "if [ -n \"$status_short\" ]; then", " WORKSPACE=\"$workspace\" BRANCH=\"$branch\" BEFORE_BRANCH=\"$before_branch\" BEFORE_HEAD=\"$before_head\" STATUS_SHORT=\"$status_short\" node <<'NODE'", "console.log(JSON.stringify({ ok: false, status: 'skipped', failureKind: 'source-worktree-dirty', workspace: process.env.WORKSPACE, branch: process.env.BRANCH, before: { branch: process.env.BEFORE_BRANCH || null, head: process.env.BEFORE_HEAD || null, detached: process.env.BEFORE_BRANCH === 'HEAD' }, statusShort: process.env.STATUS_SHORT || null, valuesPrinted: false }));", "NODE", " exit 0", "fi", "git_cmd -C \"$workspace\" fetch origin \"$branch\" > \"$tmp_dir/fetch.out\" 2> \"$tmp_dir/fetch.err\"", "fetch_exit=$?", "remote_branch_commit=$(git_cmd -C \"$workspace\" rev-parse \"refs/remotes/origin/$branch\" 2>/dev/null || true)", "checkout_exit=1", "if [ \"$fetch_exit\" -eq 0 ] && [ -n \"$remote_branch_commit\" ]; then", " git_cmd -C \"$workspace\" checkout -B \"$branch\" \"refs/remotes/origin/$branch\" > \"$tmp_dir/checkout.out\" 2> \"$tmp_dir/checkout.err\"", " checkout_exit=$?", "else", " : > \"$tmp_dir/checkout.out\"", " : > \"$tmp_dir/checkout.err\"", "fi", "after_branch=$(git_cmd -C \"$workspace\" rev-parse --abbrev-ref HEAD 2>/dev/null || true)", "after_head=$(git_cmd -C \"$workspace\" rev-parse HEAD 2>/dev/null || true)", "after_status_short=$(git_cmd -C \"$workspace\" status --short 2>/dev/null || true)", "fetch_err=$(tail -n 20 \"$tmp_dir/fetch.err\" 2>/dev/null | tr '\\n' ' ' | cut -c1-1200)", "checkout_err=$(tail -n 20 \"$tmp_dir/checkout.err\" 2>/dev/null | tr '\\n' ' ' | cut -c1-1200)", "WORKSPACE=\"$workspace\" BRANCH=\"$branch\" BEFORE_BRANCH=\"$before_branch\" BEFORE_HEAD=\"$before_head\" FETCH_EXIT=\"$fetch_exit\" CHECKOUT_EXIT=\"$checkout_exit\" REMOTE_BRANCH_COMMIT=\"$remote_branch_commit\" AFTER_BRANCH=\"$after_branch\" AFTER_HEAD=\"$after_head\" AFTER_STATUS_SHORT=\"$after_status_short\" FETCH_ERR=\"$fetch_err\" CHECKOUT_ERR=\"$checkout_err\" node <<'NODE'", "const fetchExit = Number(process.env.FETCH_EXIT || '1');", "const checkoutExit = Number(process.env.CHECKOUT_EXIT || '1');", "const afterClean = !process.env.AFTER_STATUS_SHORT;", "const ok = fetchExit === 0 && checkoutExit === 0 && afterClean && process.env.AFTER_BRANCH === process.env.BRANCH;", "let failureKind = null;", "if (fetchExit !== 0) failureKind = 'source-branch-fetch-failed';", "else if (!process.env.REMOTE_BRANCH_COMMIT) failureKind = 'source-branch-missing';", "else if (checkoutExit !== 0) failureKind = 'source-branch-checkout-failed';", "else if (!afterClean) failureKind = 'source-worktree-dirty-after-restore';", "else if (process.env.AFTER_BRANCH !== process.env.BRANCH) failureKind = 'source-worktree-branch-mismatch';", "console.log(JSON.stringify({", " ok,", " status: ok ? 'restored' : 'failed',", " failureKind,", " workspace: process.env.WORKSPACE,", " branch: process.env.BRANCH,", " before: { branch: process.env.BEFORE_BRANCH || null, head: process.env.BEFORE_HEAD || null, detached: process.env.BEFORE_BRANCH === 'HEAD' },", " after: { branch: process.env.AFTER_BRANCH || null, head: process.env.AFTER_HEAD || null, clean: afterClean },", " remoteBranchCommit: process.env.REMOTE_BRANCH_COMMIT || null,", " fetch: { exitCode: fetchExit, stderrTail: process.env.FETCH_ERR || null },", " checkout: { exitCode: checkoutExit, stderrTail: process.env.CHECKOUT_ERR || null },", " valuesPrinted: false", "}));", "NODE", ].join("\n"); } export function yamlLaneBuildImageSubmitScript(spec: AgentRunLaneSpec, sourceCommit: string): string { const build = spec.deployment.manager.imageBuild; const noProxy = build.noProxy.join(","); const buildContainerNoProxy = build.buildContainerProxy.noProxy.join(","); const imageRepository = `${spec.ci.registryPrefix}/${build.repository}`; const stateDir = `/tmp/unidesk-agentrun-build-${spec.nodeId}-${spec.lane}`; const buildArgs = Object.entries(build.buildArgs) .sort(([left], [right]) => left.localeCompare(right)) .map(([key, value]) => `${key}=${value}`); const buildContainerProxyIdentity = [ `HTTP_PROXY=${build.buildContainerProxy.httpProxy ?? ""}`, `HTTPS_PROXY=${build.buildContainerProxy.httpsProxy ?? ""}`, `NO_PROXY=${buildContainerNoProxy}`, ]; const script = [ "set -eu", `workspace=${shQuote(spec.source.workspace)}`, `source_commit=${shQuote(sourceCommit)}`, `state_dir=${shQuote(stateDir)}`, `containerfile=${shQuote(build.containerfile)}`, `context_dir=${shQuote(build.context)}`, `image_repository=${shQuote(imageRepository)}`, `network=${shQuote(build.network)}`, `http_proxy_value=${build.httpProxy === null ? "''" : shQuote(build.httpProxy)}`, `https_proxy_value=${build.httpsProxy === null ? "''" : shQuote(build.httpsProxy)}`, `no_proxy_value=${shQuote(noProxy)}`, `build_container_http_proxy_value=${build.buildContainerProxy.httpProxy === null ? "''" : shQuote(build.buildContainerProxy.httpProxy)}`, `build_container_https_proxy_value=${build.buildContainerProxy.httpsProxy === null ? "''" : shQuote(build.buildContainerProxy.httpsProxy)}`, `build_container_no_proxy_value=${shQuote(buildContainerNoProxy)}`, `env_identity_files=${shQuote(JSON.stringify(build.envIdentityFiles))}`, `build_args_json=${shQuote(JSON.stringify(buildArgs))}`, `build_container_proxy_identity_json=${shQuote(JSON.stringify(buildContainerProxyIdentity))}`, "mkdir -p \"$state_dir\"", "git config --global --add safe.directory \"$workspace\" 2>/dev/null || true", "cd \"$workspace\"", "git checkout \"$source_commit\"", "env_identity=$(ENV_IDENTITY_FILES=\"$env_identity_files\" BUILD_ARGS_JSON=\"$build_args_json\" BUILD_CONTAINER_PROXY_IDENTITY_JSON=\"$build_container_proxy_identity_json\" node <<'NODE'", "const { createHash } = require('node:crypto');", "const { readFileSync, existsSync, lstatSync, readdirSync } = require('node:fs');", "const { join } = require('node:path');", "const files = JSON.parse(process.env.ENV_IDENTITY_FILES || '[]');", "const buildArgs = JSON.parse(process.env.BUILD_ARGS_JSON || '[]');", "const buildContainerProxyIdentity = JSON.parse(process.env.BUILD_CONTAINER_PROXY_IDENTITY_JSON || '[]');", "const hash = createHash('sha256');", "const skipDirNames = new Set(['.git', '.worktree', '.state', 'node_modules', 'coverage', 'tmp', '.tmp']);", "function collectIdentityFiles(input) {", " if (!existsSync(input)) return [{ path: input, missing: true }];", " const stat = lstatSync(input);", " if (stat.isFile()) return [{ path: input, missing: false }];", " if (!stat.isDirectory()) return [{ path: input, missing: true }];", " const out = [];", " const stack = [input];", " while (stack.length > 0) {", " const dir = stack.pop();", " const entries = readdirSync(dir, { withFileTypes: true }).sort((left, right) => left.name.localeCompare(right.name));", " for (const entry of entries) {", " if (entry.isDirectory() && skipDirNames.has(entry.name)) continue;", " const child = join(dir, entry.name);", " if (entry.isDirectory()) stack.push(child);", " else if (entry.isFile()) out.push({ path: child, missing: false });", " }", " }", " return out.sort((left, right) => left.path.localeCompare(right.path));", "}", "for (const item of buildArgs) { hash.update('build-arg'); hash.update('\\0'); hash.update(item); hash.update('\\0'); }", "for (const item of buildContainerProxyIdentity) { hash.update('build-container-proxy'); hash.update('\\0'); hash.update(item); hash.update('\\0'); }", "for (const file of files) {", " for (const entry of collectIdentityFiles(file)) {", " hash.update(entry.path); hash.update('\\0');", " if (!entry.missing) hash.update(readFileSync(entry.path));", " hash.update('\\0');", " }", "}", "process.stdout.write(hash.digest('hex').slice(0, 24));", "NODE", ")", "source_short=$(printf '%s' \"$source_commit\" | cut -c1-12)", "job_id=\"$source_short-$env_identity\"", "status_file=\"$state_dir/$job_id.json\"", "stdout_file=\"$state_dir/$job_id.stdout.log\"", "stderr_file=\"$state_dir/$job_id.stderr.log\"", "cat > \"$status_file\" </dev/null | sed 's/\"/\\\\\"/g' | tr '\\n' ' ' | cut -c1-2000); CODE=\"$code\" ERROR_TAIL=\"$tail_text\" JOB_ID=\"$job_id\" SOURCE_COMMIT=\"$source_commit\" ENV_IDENTITY=\"$env_identity\" IMAGE_REPOSITORY=\"$image_repository\" node <<'NODE' > \"$status_file\"", "const code = Number(process.env.CODE || 1);", "console.log(JSON.stringify({ ok: false, status: 'failed', exitCode: code, jobId: process.env.JOB_ID, sourceCommit: process.env.SOURCE_COMMIT, envIdentity: process.env.ENV_IDENTITY, image: `${process.env.IMAGE_REPOSITORY}:${process.env.ENV_IDENTITY}`, errorTail: process.env.ERROR_TAIL || null, valuesPrinted: false }));", "NODE", " fi; exit \"$code\"; }", " trap write_failed_status EXIT", " cd \"$workspace\"", " image=\"$image_repository:$env_identity\"", " all_proxy_value=${https_proxy_value:-$http_proxy_value}", " if [ -n \"$http_proxy_value\" ]; then export HTTP_PROXY=\"$http_proxy_value\" http_proxy=\"$http_proxy_value\"; fi", " if [ -n \"$https_proxy_value\" ]; then export HTTPS_PROXY=\"$https_proxy_value\" https_proxy=\"$https_proxy_value\"; fi", " if [ -n \"$all_proxy_value\" ]; then export ALL_PROXY=\"$all_proxy_value\" all_proxy=\"$all_proxy_value\"; fi", " if [ -n \"$no_proxy_value\" ]; then export NO_PROXY=\"$no_proxy_value\" no_proxy=\"$no_proxy_value\"; fi", " args=\"--network $network\"", " build_container_all_proxy_value=${build_container_https_proxy_value:-$build_container_http_proxy_value}", " if [ -n \"$build_container_http_proxy_value\" ]; then args=\"$args --build-arg HTTP_PROXY=$build_container_http_proxy_value --build-arg http_proxy=$build_container_http_proxy_value\"; fi", " if [ -n \"$build_container_https_proxy_value\" ]; then args=\"$args --build-arg HTTPS_PROXY=$build_container_https_proxy_value --build-arg https_proxy=$build_container_https_proxy_value\"; fi", " if [ -n \"$build_container_all_proxy_value\" ]; then args=\"$args --build-arg ALL_PROXY=$build_container_all_proxy_value --build-arg all_proxy=$build_container_all_proxy_value\"; fi", " if [ -n \"$build_container_no_proxy_value\" ]; then args=\"$args --build-arg NO_PROXY=$build_container_no_proxy_value --build-arg no_proxy=$build_container_no_proxy_value\"; fi", " build_arg_values=$(BUILD_ARGS_JSON=\"$build_args_json\" node <<'NODE'", "const values = JSON.parse(process.env.BUILD_ARGS_JSON || '[]');", "for (const value of values) console.log(value);", "NODE", " )", " while IFS= read -r build_arg_value; do [ -n \"$build_arg_value\" ] && args=\"$args --build-arg $build_arg_value\"; done </dev/null 2>&1; then build_status=reused; else docker build $args -f \"$containerfile\" -t \"$image\" \"$context_dir\"; build_status=built; fi", " manifest_accept='application/vnd.oci.image.index.v1+json, application/vnd.oci.image.manifest.v1+json, application/vnd.docker.distribution.manifest.v2+json'", " fetch_manifest_digest() {", " ref=\"$1\"", " attempt=1", " max_attempts=30", " while [ \"$attempt\" -le \"$max_attempts\" ]; do", " headers=$(mktemp)", " if curl -fsSI -H \"Accept: $manifest_accept\" \"http://127.0.0.1:5000/v2/${image_repository#127.0.0.1:5000/}/manifests/$ref\" >\"$headers\"; then", " found_digest=$(awk -F': ' 'tolower($1)==\"docker-content-digest\"{print $2}' \"$headers\" | tr -d '\\r' | head -n 1)", " rm -f \"$headers\"", " if [ -n \"$found_digest\" ]; then printf '%s' \"$found_digest\"; return 0; fi", " else", " rm -f \"$headers\"", " fi", " printf '{\"event\":\"agentrun-registry-manifest-probe\",\"status\":\"retrying\",\"ref\":\"%s\",\"attempt\":\"%s/%s\"}\\n' \"$ref\" \"$attempt\" \"$max_attempts\" >&2", " sleep 1", " attempt=$((attempt + 1))", " done", " printf '{\"event\":\"agentrun-registry-manifest-probe\",\"status\":\"exhausted\",\"ref\":\"%s\",\"attempt\":\"%s/%s\"}\\n' \"$ref\" \"$max_attempts\" \"$max_attempts\" >&2", " return 1", " }", " docker push \"$image\"", " digest=$(fetch_manifest_digest \"$env_identity\" || true)", " if [ -n \"$digest\" ]; then verified_digest=$(fetch_manifest_digest \"$digest\" || true); if [ \"$verified_digest\" != \"$digest\" ]; then digest=; fi; fi", " STATUS=\"$build_status\" DIGEST=\"$digest\" JOB_ID=\"$job_id\" SOURCE_COMMIT=\"$source_commit\" ENV_IDENTITY=\"$env_identity\" IMAGE_REPOSITORY=\"$image_repository\" node <<'NODE' > \"$status_file\"", "const digest = process.env.DIGEST || null;", "console.log(JSON.stringify({ ok: Boolean(digest), status: process.env.STATUS || 'built', jobId: process.env.JOB_ID, sourceCommit: process.env.SOURCE_COMMIT, envIdentity: process.env.ENV_IDENTITY, image: `${process.env.IMAGE_REPOSITORY}:${process.env.ENV_IDENTITY}`, digest, repositoryDigest: digest ? `${process.env.IMAGE_REPOSITORY}@${digest}` : null, valuesPrinted: false }));", "NODE", " trap - EXIT", ") >\"$stdout_file\" 2>\"$stderr_file\" &", "pid=$!", "JOB_PID=\"$pid\" JOB_ID=\"$job_id\" STATUS_FILE=\"$status_file\" STDOUT_FILE=\"$stdout_file\" STDERR_FILE=\"$stderr_file\" node <<'NODE'", "console.log(JSON.stringify({ ok: true, status: 'submitted', jobId: process.env.JOB_ID, pid: Number(process.env.JOB_PID), statusFile: process.env.STATUS_FILE, stdoutFile: process.env.STDOUT_FILE, stderrFile: process.env.STDERR_FILE, valuesPrinted: false }));", "NODE", ].join("\n"); return script; } export async function waitForYamlLaneBuildImage(config: UniDeskConfig, spec: AgentRunLaneSpec, sourceCommit: string, jobId: string | null): Promise & { ok: boolean; payload: Record }> { if (jobId === null) return { ok: false, payload: { ok: false, degradedReason: "build-job-id-missing", valuesPrinted: false } }; const startedAt = Date.now(); const timeoutMs = Math.max(60, spec.deployment.manager.imageBuild.timeoutSeconds) * 1000; let lastPayload: Record = {}; let polls = 0; while (Date.now() - startedAt < timeoutMs) { polls += 1; const probe = await capture(config, spec.nodeRoute, ["sh", "--", yamlLaneBuildImageStatusScript(spec, jobId)]); const payload = captureJsonPayload(probe); lastPayload = payload; progressEvent("agentrun.yaml-lane.image-build.progress", { node: spec.nodeId, lane: spec.lane, sourceCommit, jobId, polls, status: stringOrNull(payload.status) ?? "unknown", elapsedMs: Date.now() - startedAt, }); if (payload.ok === true && stringOrNull(payload.digest) !== null) return { ok: true, payload, polls, elapsedMs: Date.now() - startedAt }; if (payload.status === "failed") return { ok: false, payload, polls, elapsedMs: Date.now() - startedAt }; await sleep(spec.deployment.manager.imageBuild.pollSeconds * 1000); } return { ok: false, payload: { ...lastPayload, ok: false, status: "timeout", degradedReason: "image-build-timeout", valuesPrinted: false }, polls, elapsedMs: Date.now() - startedAt }; } export function yamlLaneBuildImageStatusScript(spec: AgentRunLaneSpec, jobId: string): string { const stateDir = `/tmp/unidesk-agentrun-build-${spec.nodeId}-${spec.lane}`; return [ "set +e", `status_file=${shQuote(`${stateDir}/${jobId}.json`)}`, `stdout_file=${shQuote(`${stateDir}/${jobId}.stdout.log`)}`, `stderr_file=${shQuote(`${stateDir}/${jobId}.stderr.log`)}`, "if [ -f \"$status_file\" ]; then cat \"$status_file\"; else printf '{\"ok\":false,\"status\":\"missing\",\"valuesPrinted\":false}\\n'; fi", "if [ -f \"$stderr_file\" ] && tail -n 20 \"$stderr_file\" | grep -Eq 'ERROR|error|failed|denied'; then :; fi", ].join("\n"); } export function yamlLaneK3sBuildImageJobManifest(spec: AgentRunLaneSpec, sourceCommit: string, jobName: string): Record { const build = spec.deployment.manager.imageBuild; if (spec.ci.buildkitImage === null) throw new Error(`config/agentrun.yaml controlPlane.lanes.${spec.lane}.ci.buildkitImage is required when source.statusMode=k3s-git-mirror`); const imageRepository = `${spec.ci.registryPrefix}/${build.repository}`; const envIdentity = `source-${sourceCommit.slice(0, 12)}`; const image = `${imageRepository}:${envIdentity}`; const proxyEnv = yamlLaneK3sBuildProxyEnv(spec); return { apiVersion: "batch/v1", kind: "Job", metadata: { name: jobName, namespace: spec.ci.namespace, labels: { "app.kubernetes.io/name": "agentrun-manager-image-build", "app.kubernetes.io/part-of": "agentrun", "agentrun.pikastech.local/lane": spec.version, "agentrun.pikastech.local/node": spec.nodeId, "agentrun.pikastech.local/source-commit": sourceCommit, }, }, spec: { backoffLimit: 0, activeDeadlineSeconds: Math.max(60, build.timeoutSeconds), ttlSecondsAfterFinished: 3600, template: { metadata: { labels: { "app.kubernetes.io/name": "agentrun-manager-image-build", "app.kubernetes.io/part-of": "agentrun", "agentrun.pikastech.local/lane": spec.version, "agentrun.pikastech.local/node": spec.nodeId, "agentrun.pikastech.local/source-commit": sourceCommit, }, }, spec: { restartPolicy: "Never", serviceAccountName: spec.ci.serviceAccountName, hostNetwork: true, dnsPolicy: "ClusterFirstWithHostNet", securityContext: { fsGroup: 1000 }, volumes: [ { name: "workspace", emptyDir: { sizeLimit: "8Gi" } }, { name: "buildkit-state", emptyDir: { sizeLimit: "8Gi" } }, { name: "tmp", emptyDir: {} }, ], initContainers: [{ name: "source", image: spec.ci.toolsImage, imagePullPolicy: "IfNotPresent", env: proxyEnv, command: ["/bin/sh", "-ec", yamlLaneK3sBuildSourceShell(spec, sourceCommit)], volumeMounts: [{ name: "workspace", mountPath: "/workspace" }], }], containers: [{ name: "buildkit", image: spec.ci.buildkitImage, imagePullPolicy: "IfNotPresent", env: [ ...proxyEnv, { name: "BUILDKITD_FLAGS", value: "--oci-worker-no-process-sandbox --oci-worker-net=host --allow-insecure-entitlement network.host" }, ], command: ["/bin/sh", "-ec", yamlLaneK3sBuildImageShell(spec, sourceCommit, image, envIdentity)], securityContext: { privileged: true, runAsUser: 1000, runAsGroup: 1000 }, volumeMounts: [ { name: "workspace", mountPath: "/workspace" }, { name: "buildkit-state", mountPath: "/home/user/.local/share/buildkit" }, { name: "tmp", mountPath: "/tmp" }, ], }], }, }, }, }; } export async function runYamlLaneK3sBuildImageJob(config: UniDeskConfig, spec: AgentRunLaneSpec, sourceCommit: string): Promise & { ok: boolean; payload: Record }> { const jobName = `agentrun-build-${spec.nodeId.toLowerCase()}-${spec.lane}-${sourceCommit.slice(0, 12)}`.slice(0, 63); const manifest = yamlLaneK3sBuildImageJobManifest(spec, sourceCommit, jobName); const created = await capture(config, spec.nodeKubeRoute, ["sh", "--", createYamlLaneJobScript(spec.ci.namespace, jobName, manifest)]); const createPayload = captureJsonPayload(created); if (created.exitCode !== 0 || createPayload.ok === false) { return { ok: false, payload: { ok: false, status: "create-failed", jobName, degradedReason: "k3s-buildkit-job-create-failed", valuesPrinted: false }, create: createPayload, capture: compactCapture(created, { full: true, stdoutTailChars: 5000, stderrTailChars: 5000 }), valuesPrinted: false, }; } const startedAt = Date.now(); const timeoutMs = Math.max(60, spec.deployment.manager.imageBuild.timeoutSeconds) * 1000; const pollMs = Math.max(1, spec.deployment.manager.imageBuild.pollSeconds) * 1000; let polls = 0; let lastPayload: Record = {}; let lastProbe: SshCaptureResult | null = null; while (Date.now() - startedAt < timeoutMs) { polls += 1; lastProbe = await capture(config, spec.nodeKubeRoute, ["sh", "--", yamlLaneJobProbeScript(spec.ci.namespace, jobName)]); const probePayload = captureJsonPayload(lastProbe); const buildPayload = yamlLaneGitopsPublishPayloadFromProbe(probePayload); if (Object.keys(buildPayload).length > 0) lastPayload = buildPayload; progressEvent("agentrun.yaml-lane.k3s-image-build.progress", { node: spec.nodeId, lane: spec.lane, sourceCommit, jobName, polls, status: stringOrNull(buildPayload.status) ?? (probePayload.succeeded === true ? "succeeded" : probePayload.failed === true ? "failed" : "running"), digest: stringOrNull(buildPayload.digest), elapsedMs: Date.now() - startedAt, }); if (probePayload.succeeded === true) { if (buildPayload.ok === true && stringOrNull(buildPayload.digest) !== null && stringOrNull(buildPayload.envIdentity) !== null) { return { ok: true, payload: buildPayload, jobName, create: createPayload, polls, elapsedMs: Date.now() - startedAt, probe: compactCapture(lastProbe), valuesPrinted: false }; } return { ok: false, payload: { ...buildPayload, ok: false, status: stringOrNull(buildPayload.status) ?? "succeeded-without-result", degradedReason: "k3s-buildkit-result-missing", jobName, valuesPrinted: false }, jobName, create: createPayload, polls, elapsedMs: Date.now() - startedAt, probe: compactCapture(lastProbe, { full: true, stdoutTailChars: 5000, stderrTailChars: 5000 }), valuesPrinted: false, }; } if (probePayload.failed === true) { const payload = Object.keys(buildPayload).length > 0 ? buildPayload : { ok: false, status: "failed", degradedReason: "k3s-buildkit-job-failed", jobName, valuesPrinted: false }; return { ok: false, payload, jobName, create: createPayload, polls, elapsedMs: Date.now() - startedAt, probe: compactCapture(lastProbe, { full: true, stdoutTailChars: 5000, stderrTailChars: 5000 }), valuesPrinted: false, }; } await sleep(pollMs); } return { ok: false, payload: { ...lastPayload, ok: false, status: "timeout", degradedReason: "k3s-buildkit-job-timeout", jobName, valuesPrinted: false }, jobName, create: createPayload, polls, elapsedMs: Date.now() - startedAt, probe: lastProbe === null ? null : compactCapture(lastProbe, { full: true, stdoutTailChars: 5000, stderrTailChars: 5000 }), valuesPrinted: false, }; } function yamlLaneK3sBuildProxyEnv(spec: AgentRunLaneSpec): Array<{ name: string; value: string }> { const build = spec.deployment.manager.imageBuild; const result: Array<{ name: string; value: string }> = []; const noProxy = build.noProxy.join(","); const allProxy = build.httpsProxy ?? build.httpProxy; if (build.httpProxy !== null) result.push({ name: "HTTP_PROXY", value: build.httpProxy }, { name: "http_proxy", value: build.httpProxy }); if (build.httpsProxy !== null) result.push({ name: "HTTPS_PROXY", value: build.httpsProxy }, { name: "https_proxy", value: build.httpsProxy }); if (allProxy !== null) result.push({ name: "ALL_PROXY", value: allProxy }, { name: "all_proxy", value: allProxy }); result.push({ name: "NO_PROXY", value: noProxy }, { name: "no_proxy", value: noProxy }); return result; } function yamlLaneK3sBuildSourceShell(spec: AgentRunLaneSpec, sourceCommit: string): string { return [ "set -eu", `read_url=${shQuote(spec.gitMirror.readUrl)}`, `source_commit=${shQuote(sourceCommit)}`, `source_stage_ref=${shQuote(agentRunSourceSnapshotRef(spec, sourceCommit))}`, "rm -rf /workspace/repo", "git clone --no-checkout \"$read_url\" /workspace/repo", "cd /workspace/repo", "git fetch origin \"+$source_stage_ref:refs/remotes/origin/unidesk-source-snapshot\"", "git checkout --detach \"$source_commit\"", "actual=$(git rev-parse HEAD)", "test \"$actual\" = \"$source_commit\"", "chmod -R a+rwX /workspace", ].join("\n"); } function yamlLaneK3sBuildImageShell(spec: AgentRunLaneSpec, sourceCommit: string, image: string, envIdentity: string): string { const build = spec.deployment.manager.imageBuild; const contextPath = build.context === "." ? "/workspace/repo" : `/workspace/repo/${build.context.replace(/^\.\//u, "")}`; const buildContainerNoProxy = build.buildContainerProxy.noProxy.join(","); const buildArgs = [ ...Object.entries(build.buildArgs).sort(([left], [right]) => left.localeCompare(right)).map(([key, value]) => `build-arg:${key}=${value}`), ...yamlLaneK3sBuildProxyBuildArgs(build.buildContainerProxy.httpProxy, build.buildContainerProxy.httpsProxy, buildContainerNoProxy), ]; const buildctlArgs = [ "build", "--allow", "network.host", "--frontend", "dockerfile.v0", "--local", `context=${contextPath}`, "--local", "dockerfile=/workspace/repo", "--opt", `filename=${build.containerfile}`, "--opt", `network=${build.network}`, ...buildArgs.flatMap((arg) => ["--opt", arg]), "--metadata-file", "/workspace/build-metadata.json", "--output", `type=image,name=${image},push=true,registry.insecure=true`, ]; const repositoryDigestBase = image.slice(0, image.lastIndexOf(":")); return [ "set -eu", "cd /workspace/repo", `buildctl-daemonless.sh ${buildctlArgs.map((arg) => shQuote(arg)).join(" ")}`, "metadata_compact=$(tr -d '\\n' < /workspace/build-metadata.json)", "digest=$(printf '%s' \"$metadata_compact\" | sed -n 's/.*\"containerimage.digest\"[[:space:]]*:[[:space:]]*\"\\([^\"]*\\)\".*/\\1/p' | head -n 1)", "if [ -z \"$digest\" ]; then", ` printf '{"ok":false,"status":"failed","failureKind":"image-digest-missing","sourceCommit":"%s","envIdentity":"%s","image":"%s","valuesPrinted":false}\\n' ${shQuote(sourceCommit)} ${shQuote(envIdentity)} ${shQuote(image)}`, " exit 1", "fi", `printf '{"ok":true,"status":"built","sourceCommit":"%s","envIdentity":"%s","image":"%s","digest":"%s","repositoryDigest":"%s@%s","valuesPrinted":false}\\n' ${shQuote(sourceCommit)} ${shQuote(envIdentity)} ${shQuote(image)} "$digest" ${shQuote(repositoryDigestBase)} "$digest"`, ].join("\n"); } function yamlLaneK3sBuildProxyBuildArgs(httpProxy: string | null, httpsProxy: string | null, noProxy: string): string[] { const result: string[] = []; const allProxy = httpsProxy ?? httpProxy; if (httpProxy !== null) result.push(`build-arg:HTTP_PROXY=${httpProxy}`, `build-arg:http_proxy=${httpProxy}`); if (httpsProxy !== null) result.push(`build-arg:HTTPS_PROXY=${httpsProxy}`, `build-arg:https_proxy=${httpsProxy}`); if (allProxy !== null) result.push(`build-arg:ALL_PROXY=${allProxy}`, `build-arg:all_proxy=${allProxy}`); if (noProxy.length > 0) result.push(`build-arg:NO_PROXY=${noProxy}`, `build-arg:no_proxy=${noProxy}`); return result; } export async function runYamlLaneGitopsPublishJob(config: UniDeskConfig, spec: AgentRunLaneSpec, sourceCommit: string, files: readonly { path: string; content: string }[]): Promise & { ok: boolean; payload: Record }> { const jobName = `gitops-publish-${spec.nodeId.toLowerCase()}-${spec.lane}-${Date.now().toString(36)}`.slice(0, 63); const manifest = yamlLaneGitopsPublishJobManifest(spec, files, jobName); const created = await capture(config, spec.nodeKubeRoute, ["sh", "--", createYamlLaneJobScript(spec.gitMirror.namespace, jobName, manifest)]); if (created.exitCode !== 0) { return { ok: false, payload: { ok: false, status: "create-failed", jobName, degradedReason: "gitops-publish-job-create-failed", valuesPrinted: false }, jobName, sourceCommit, phase: "create-job", capture: compactCapture(created, { full: true, stdoutTailChars: 5000, stderrTailChars: 5000 }), valuesPrinted: false, }; } const startedAt = Date.now(); const timeoutMs = 300_000; let polls = 0; let lastPayload: Record = {}; let lastProbe: SshCaptureResult | null = null; while (Date.now() - startedAt < timeoutMs) { polls += 1; lastProbe = await capture(config, spec.nodeKubeRoute, ["sh", "--", yamlLaneJobProbeScript(spec.gitMirror.namespace, jobName)]); const probePayload = captureJsonPayload(lastProbe); const publishPayload = yamlLaneGitopsPublishPayloadFromProbe(probePayload); if (Object.keys(publishPayload).length > 0) lastPayload = publishPayload; progressEvent("agentrun.yaml-lane.gitops-publish.progress", { node: spec.nodeId, lane: spec.lane, sourceCommit, jobName, polls, status: stringOrNull(publishPayload.status) ?? (probePayload.succeeded === true ? "succeeded" : probePayload.failed === true ? "failed" : "running"), gitopsCommit: stringOrNull(publishPayload.gitopsCommit), elapsedMs: Date.now() - startedAt, }); if (probePayload.succeeded === true) { if (publishPayload.ok === true && stringOrNull(publishPayload.gitopsCommit) !== null) { return { ok: true, payload: publishPayload, jobName, polls, elapsedMs: Date.now() - startedAt, probe: compactCapture(lastProbe), valuesPrinted: false }; } return { ok: false, payload: { ...publishPayload, ok: false, status: stringOrNull(publishPayload.status) ?? "succeeded-without-result", degradedReason: "gitops-publish-result-missing", jobName, valuesPrinted: false }, jobName, polls, elapsedMs: Date.now() - startedAt, probe: compactCapture(lastProbe, { full: true, stdoutTailChars: 5000, stderrTailChars: 5000 }), valuesPrinted: false, }; } if (probePayload.failed === true) { const payload = Object.keys(publishPayload).length > 0 ? publishPayload : { ok: false, status: "failed", degradedReason: "gitops-publish-job-failed", jobName, valuesPrinted: false }; return { ok: false, payload, jobName, polls, elapsedMs: Date.now() - startedAt, probe: compactCapture(lastProbe, { full: true, stdoutTailChars: 5000, stderrTailChars: 5000 }), valuesPrinted: false, }; } await sleep(5_000); } return { ok: false, payload: { ...lastPayload, ok: false, status: "timeout", degradedReason: "gitops-publish-job-timeout", jobName, valuesPrinted: false }, jobName, polls, elapsedMs: Date.now() - startedAt, probe: lastProbe === null ? null : compactCapture(lastProbe, { full: true, stdoutTailChars: 5000, stderrTailChars: 5000 }), valuesPrinted: false, }; }