mirrors/pikasTech-unidesk
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity
Files
ab3566435ced8780215b48eb7a1160f324c87cf8
pikasTech-unidesk/scripts/src/agentrun/yaml-lane.ts
T
Codex ab3566435c fix: add JD01 GC retention controls
2026-07-05 04:18:22 +00:00

1380 lines
77 KiB
TypeScript
Raw Blame History

// SPEC: PJ2026-01060307 控制面模块化 draft-2026-06-25-p0. yaml-lane module for scripts/src/agentrun.ts.
// Moved mechanically from scripts/src/agentrun.ts:3311-4219 for #903.
// SPEC: PJ2026-01020108 cancel lifecycle + PJ2026-01020205 AipodSpec binding + PJ2026-01020302 session policy + PJ2026-01020305 cancel control + PJ2026-01060305/06 YAML execution policy and bounded output draft-2026-06-25-p0.
// Exposes AgentRun lane-scoped policy, AipodSpec SecretRef binding, cancel lifecycle, and bounded default output in the UniDesk CLI.
import { chmodSync, copyFileSync, existsSync, readFileSync, statSync, writeFileSync } from "node:fs";
import { join } from "node:path";
import { spawnSync } from "node:child_process";
import { rootPath, type UniDeskConfig } from "../config";
import type { RenderedCliResult } from "../output";
import { applyLocalCaddyManagedSite } from "../pk01-caddy";
import { runSshCommandCapture, type SshCaptureResult } from "../ssh";
import { runRemoteSshCommandCapture } from "../remote";
import { startJob } from "../jobs";
import {
AGENTRUN_CONFIG_PATH,
agentRunLaneSummary,
agentRunPipelineRunName,
agentRunProviderCredentialRefs,
agentRunSourceSnapshotRef,
agentRunSourceSnapshotStageRefPrefix,
resolveAgentRunLaneTarget,
type AgentRunCancelLifecycleSpec,
type AgentRunLaneSpec,
} from "../agentrun-lanes";
import {
agentRunImageArtifact,
placeholderAgentRunImage,
renderedFilesDigest,
renderedObjectsDigest,
renderAgentRunControlPlaneManifests,
renderAgentRunGitopsFiles,
type AgentRunArtifactService,
} from "../agentrun-manifests";
import { sha256Fingerprint } from "../platform-infra-ops-library";
import type { CleanupReleasedPvOptions, CleanupRunnersOptions, CleanupRunsOptions, CleanupSessionPvcsOptions, RefreshOptions } from "./options";
import { cleanupReleasedPvsFinalizeNodeScript, cleanupReleasedPvsPlanNodeScript, cleanupRunnersFinalizeNodeScript, cleanupRunsFinalizeNodeScript, cleanupRunsPlanNodeScript, refreshYamlLaneScript } from "./git-mirror";
import { cleanupRunnersFactsNodeScript, cleanupRunnersPlanNodeScript, collectLaneSecretSources, createYamlLaneJobScript, yamlLaneGitopsPublishJobManifest, yamlLaneGitopsPublishPayloadFromProbe, yamlLaneJobProbeScript } from "./secrets";
import { capture, captureJsonPayload, compactCapture, progressEvent, shQuote, sleep, stringOrNull } from "./utils";
export async function refresh(config: UniDeskConfig, options: RefreshOptions): Promise<Record<string, unknown>> {
return await refreshYamlLane(config, options);
}
export async function refreshYamlLane(config: UniDeskConfig, options: RefreshOptions): Promise<Record<string, unknown>> {
const { configPath, spec } = resolveAgentRunLaneTarget(options);
const plan = {
node: spec.nodeId,
lane: spec.lane,
version: spec.version,
argoNamespace: spec.gitops.argoNamespace,
argoApplication: spec.gitops.argoApplication,
gitopsBranch: spec.gitops.branch,
mutation: "argocd-hard-refresh",
valuesPrinted: false,
};
if (options.dryRun || !options.confirm) {
return {
ok: true,
command: "agentrun control-plane refresh",
mode: "dry-run",
mutation: false,
configPath,
target: agentRunLaneSummary(spec),
plan,
next: {
confirm: `bun scripts/cli.ts agentrun control-plane refresh --node ${spec.nodeId} --lane ${spec.lane} --confirm`,
status: `bun scripts/cli.ts agentrun control-plane status --node ${spec.nodeId} --lane ${spec.lane}`,
},
valuesPrinted: false,
};
}
const refreshed = await capture(config, spec.nodeKubeRoute, ["sh", "--", refreshYamlLaneScript(spec)]);
const payload = captureJsonPayload(refreshed);
return {
ok: refreshed.exitCode === 0 && payload.ok !== false,
command: "agentrun control-plane refresh",
mode: "confirmed-yaml-lane",
mutation: true,
configPath,
target: agentRunLaneSummary(spec),
plan,
result: payload,
refreshed: compactCapture(refreshed, { full: refreshed.exitCode !== 0, stdoutTailChars: 3000, stderrTailChars: 3000 }),
next: {
status: `bun scripts/cli.ts agentrun control-plane status --node ${spec.nodeId} --lane ${spec.lane}`,
},
valuesPrinted: false,
};
}
export async function cleanupRunners(config: UniDeskConfig, options: CleanupRunnersOptions): Promise<Record<string, unknown>> {
const { configPath, spec } = resolveAgentRunLaneTarget(options);
const result = await capture(config, spec.nodeKubeRoute, ["sh", "--", cleanupRunnersScript(options, spec)]);
const payload = captureJsonPayload(result);
const ok = result.exitCode === 0 && payload.ok !== false;
const base = {
...payload,
ok,
command: "agentrun control-plane cleanup-runners",
configPath,
target: agentRunLaneSummary(spec),
mode: options.dryRun || !options.confirm ? "dry-run" : "confirmed-cleanup",
namespace: spec.runtime.namespace,
retention: spec.deployment.runner.retention,
probe: compactCapture(result, { full: result.exitCode !== 0, stdoutTailChars: 3000, stderrTailChars: 3000 }),
};
if (options.dryRun || !options.confirm) {
return {
...base,
dryRun: true,
mutation: false,
next: {
confirm: `bun scripts/cli.ts agentrun control-plane cleanup-runners --node ${spec.nodeId} --lane ${spec.lane}${options.forceActive ? " --force-active" : ""} --confirm`,
},
};
}
return {
...base,
dryRun: false,
mutation: true,
followUp: {
dryRun: `bun scripts/cli.ts agentrun control-plane cleanup-runners --node ${spec.nodeId} --lane ${spec.lane}${options.forceActive ? " --force-active" : ""} --dry-run`,
status: `bun scripts/cli.ts agentrun control-plane status --node ${spec.nodeId} --lane ${spec.lane}`,
},
};
}
export async function cleanupRuns(config: UniDeskConfig, options: CleanupRunsOptions): Promise<Record<string, unknown>> {
const { configPath, spec } = resolveAgentRunLaneTarget(options);
const result = await capture(config, spec.nodeKubeRoute, ["sh", "--", cleanupRunsScript(options, spec.ci.namespace, spec.ci.pipelineRunPrefix)]);
const payload = captureJsonPayload(result);
const ok = result.exitCode === 0 && payload.ok !== false;
const base = {
...payload,
ok,
command: "agentrun control-plane cleanup-runs",
configPath,
target: agentRunLaneSummary(spec),
mode: options.dryRun || !options.confirm ? "dry-run" : "confirmed-cleanup",
namespace: spec.ci.namespace,
minAgeMinutes: options.minAgeMinutes,
limit: options.limit,
probe: compactCapture(result, { full: result.exitCode !== 0, stdoutTailChars: 3000, stderrTailChars: 3000 }),
};
if (options.dryRun || !options.confirm) {
return {
...base,
dryRun: true,
mutation: false,
next: {
confirm: `bun scripts/cli.ts agentrun control-plane cleanup-runs --node ${spec.nodeId} --lane ${spec.lane} --min-age-minutes ${options.minAgeMinutes} --limit ${options.limit} --confirm`,
},
};
}
return {
...base,
dryRun: false,
mutation: true,
followUp: {
status: `bun scripts/cli.ts agentrun control-plane status --node ${spec.nodeId} --lane ${spec.lane}`,
releasedPvs: `bun scripts/cli.ts agentrun control-plane cleanup-released-pvs --node ${spec.nodeId} --lane ${spec.lane} --limit ${options.limit} --dry-run`,
diskPressure: `trans ${spec.nodeKubeRoute} kubectl get node -o jsonpath='{range .items[*]}{.metadata.name}{\"\\t\"}{range .status.conditions[*]}{.type}{\"=\"}{.status}{\" \"}{.reason}{\";\"}{end}{\"\\n\"}{end}'`,
},
};
}
export async function cleanupReleasedPvs(config: UniDeskConfig, options: CleanupReleasedPvOptions): Promise<Record<string, unknown>> {
const { configPath, spec } = resolveAgentRunLaneTarget(options);
const result = await capture(config, spec.nodeKubeRoute, ["sh", "--", cleanupReleasedPvsScript(options, spec.ci.namespace)]);
const payload = captureJsonPayload(result);
const ok = result.exitCode === 0 && payload.ok !== false;
const base = {
...payload,
ok,
command: "agentrun control-plane cleanup-released-pvs",
configPath,
target: agentRunLaneSummary(spec),
mode: options.dryRun || !options.confirm ? "dry-run" : "confirmed-cleanup",
namespace: spec.ci.namespace,
limit: options.limit,
probe: compactCapture(result, { full: result.exitCode !== 0, stdoutTailChars: 3000, stderrTailChars: 3000 }),
};
if (options.dryRun || !options.confirm) {
return {
...base,
dryRun: true,
mutation: false,
next: {
confirm: `bun scripts/cli.ts agentrun control-plane cleanup-released-pvs --node ${spec.nodeId} --lane ${spec.lane} --limit ${options.limit} --confirm`,
},
};
}
return {
...base,
dryRun: false,
mutation: true,
followUp: {
cleanupRuns: `bun scripts/cli.ts agentrun control-plane cleanup-runs --node ${spec.nodeId} --lane ${spec.lane} --min-age-minutes 30 --limit ${options.limit} --dry-run`,
diskPressure: `trans ${spec.nodeKubeRoute} kubectl get node -o jsonpath='{range .items[*]}{.metadata.name}{\"\\t\"}{range .status.conditions[*]}{.type}{\"=\"}{.status}{\" \"}{.reason}{\";\"}{end}{\"\\n\"}{end}'`,
},
};
}
export async function cleanupSessionPvcs(config: UniDeskConfig, options: CleanupSessionPvcsOptions): Promise<Record<string, unknown>> {
const { configPath, spec } = resolveAgentRunLaneTarget(options);
const result = await capture(config, spec.nodeKubeRoute, ["sh", "--", cleanupSessionPvcsScript(options, spec)]);
const payload = captureJsonPayload(result);
const ok = result.exitCode === 0 && payload.ok !== false;
const base = {
...payload,
ok,
command: "agentrun control-plane cleanup-session-pvcs",
configPath,
target: { node: spec.nodeId, lane: spec.lane, namespace: spec.runtime.namespace },
mode: options.dryRun || !options.confirm ? "dry-run" : "confirmed-cleanup",
namespace: spec.runtime.namespace,
retention: spec.deployment.runner.retention.sessionPvcRetention,
probe: result.exitCode === 0 ? undefined : compactCapture(result, { full: true, stdoutTailChars: 3000, stderrTailChars: 3000 }),
};
if (options.dryRun || !options.confirm) {
return { ...base, dryRun: true, mutation: false, next: { confirm: `bun scripts/cli.ts agentrun control-plane cleanup-session-pvcs --node ${spec.nodeId} --lane ${spec.lane} --limit ${options.limit} --confirm` } };
}
return {
...base,
dryRun: false,
mutation: true,
followUp: {
dryRun: `bun scripts/cli.ts agentrun control-plane cleanup-session-pvcs --node ${spec.nodeId} --lane ${spec.lane} --limit ${options.limit} --dry-run`,
diskPressure: `bun scripts/cli.ts gc remote ${spec.nodeId} status --limit 20`,
},
};
}
export function cleanupSessionPvcsScript(options: CleanupSessionPvcsOptions, spec: AgentRunLaneSpec): string {
const retention = spec.deployment.runner.retention.sessionPvcRetention;
const script = readFileSync(rootPath("scripts/src/agentrun/cleanup-session-pvcs.mjs"), "utf8");
return [
"set -eu",
`namespace=${shQuote(spec.runtime.namespace)}`,
`confirm=${options.confirm && !options.dryRun ? "true" : "false"}`,
`limit=${String(Math.min(options.limit, retention.maxDeletePerRun))}`,
`enabled=${retention.enabled ? "true" : "false"}`,
`prefixes_json_b64=${shQuote(Buffer.from(JSON.stringify(retention.prefixes), "utf8").toString("base64"))}`,
"tmp_dir=$(mktemp -d)",
"trap 'rm -rf \"$tmp_dir\"' EXIT",
"cat > \"$tmp_dir/cleanup-session-pvcs.mjs\" <<'NODE'",
script,
"NODE",
"env NAMESPACE=\"$namespace\" CONFIRM=\"$confirm\" LIMIT=\"$limit\" ENABLED=\"$enabled\" PREFIXES_JSON_B64=\"$prefixes_json_b64\" node \"$tmp_dir/cleanup-session-pvcs.mjs\"",
].join("\n");
}
export function cleanupRunnersScript(options: CleanupRunnersOptions, spec: AgentRunLaneSpec): string {
const retention = spec.deployment.runner.retention;
const matchLabelsB64 = Buffer.from(JSON.stringify(retention.selectors.matchLabels), "utf8").toString("base64");
const jobNamePrefixesB64 = Buffer.from(JSON.stringify(retention.selectors.jobNamePrefixes), "utf8").toString("base64");
return [
"set -eu",
`namespace=${shQuote(spec.runtime.namespace)}`,
`manager_deployment=${shQuote(spec.runtime.managerDeployment)}`,
`max_runners=${String(retention.maxRunners)}`,
`cleanup_order=${shQuote(retention.cleanupOrder)}`,
`active_heartbeat_max_age_ms=${String(retention.activeHeartbeatMaxAgeMs)}`,
`age_based_cleanup_enabled=${retention.ageBasedCleanup.enabled ? "true" : "false"}`,
`age_based_max_age_hours=${retention.ageBasedCleanup.maxAgeHours === null ? "" : String(retention.ageBasedCleanup.maxAgeHours)}`,
`timeout_seconds=${String(options.timeoutSeconds)}`,
`force_active=${options.forceActive ? "true" : "false"}`,
`match_labels_json_b64=${shQuote(matchLabelsB64)}`,
`job_name_prefixes_json_b64=${shQuote(jobNamePrefixesB64)}`,
"match_labels_json=$(printf '%s' \"$match_labels_json_b64\" | base64 -d)",
"job_name_prefixes_json=$(printf '%s' \"$job_name_prefixes_json_b64\" | base64 -d)",
"tmp_dir=$(mktemp -d)",
"trap 'rm -rf \"$tmp_dir\"' EXIT",
"kubectl -n \"$namespace\" get job -o json > \"$tmp_dir/jobs.json\"",
"kubectl -n \"$namespace\" get pod -o json > \"$tmp_dir/pods.json\"",
"facts_exit=0",
"set +e",
"kubectl -n \"$namespace\" exec -i deploy/\"$manager_deployment\" -- env RETENTION_NAMESPACE=\"$namespace\" sh -lc 'cat >/tmp/agentrun-runner-retention.mjs && bun /tmp/agentrun-runner-retention.mjs' > \"$tmp_dir/runner-facts.json\" 2> \"$tmp_dir/runner-facts.err\" <<'NODE'",
cleanupRunnersFactsNodeScript(),
"NODE",
"facts_exit=$?",
"set -e",
"if [ \"$facts_exit\" -ne 0 ]; then",
" printf '%s\\n' '{\"ok\":false,\"items\":[],\"failureKind\":\"manager-facts-unavailable\",\"valuesPrinted\":false}' > \"$tmp_dir/runner-facts.json\"",
"fi",
"MATCH_LABELS_JSON=\"$match_labels_json\" JOB_NAME_PREFIXES_JSON=\"$job_name_prefixes_json\" MAX_RUNNERS=\"$max_runners\" CLEANUP_ORDER=\"$cleanup_order\" ACTIVE_HEARTBEAT_MAX_AGE_MS=\"$active_heartbeat_max_age_ms\" AGE_BASED_CLEANUP_ENABLED=\"$age_based_cleanup_enabled\" AGE_BASED_MAX_AGE_HOURS=\"$age_based_max_age_hours\" FORCE_ACTIVE=\"$force_active\" FACTS_EXIT=\"$facts_exit\" TMP_DIR=\"$tmp_dir\" NAMESPACE=\"$namespace\" node <<'NODE' > \"$tmp_dir/plan.json\"",
cleanupRunnersPlanNodeScript(),
"NODE",
"if [ " + shQuote(options.confirm && !options.dryRun ? "true" : "false") + " != true ]; then",
" cat \"$tmp_dir/plan.json\"",
" exit 0",
"fi",
"node -e 'const fs=require(\"node:fs\"); const plan=JSON.parse(fs.readFileSync(process.argv[1],\"utf8\")); const names=Array.isArray(plan.selectedRunnerJobs)?plan.selectedRunnerJobs:[]; fs.writeFileSync(process.argv[2], names.join(\"\\n\") + (names.length>0?\"\\n\":\"\"));' \"$tmp_dir/plan.json\" \"$tmp_dir/selected-jobs.txt\"",
"delete_exit=0",
"if [ -s \"$tmp_dir/selected-jobs.txt\" ]; then",
" xargs -r kubectl -n \"$namespace\" delete job --ignore-not-found=true --wait=true --timeout=\"${timeout_seconds}s\" < \"$tmp_dir/selected-jobs.txt\" > \"$tmp_dir/delete.out\" 2> \"$tmp_dir/delete.err\" || delete_exit=$?",
"else",
" : > \"$tmp_dir/delete.out\"",
" : > \"$tmp_dir/delete.err\"",
"fi",
"kubectl -n \"$namespace\" get job -o json > \"$tmp_dir/jobs-after.json\"",
"kubectl -n \"$namespace\" get pod -o json > \"$tmp_dir/pods-after.json\"",
"DELETE_EXIT=\"$delete_exit\" TMP_DIR=\"$tmp_dir\" node <<'NODE'",
cleanupRunnersFinalizeNodeScript(),
"NODE",
].join("\n");
}
export function cleanupRunsScript(options: CleanupRunsOptions, namespace: string, pipelineRunPrefix: string): string {
return [
"set -eu",
`namespace=${shQuote(namespace)}`,
`pipeline_run_prefix=${shQuote(pipelineRunPrefix)}`,
`min_age_minutes=${String(options.minAgeMinutes)}`,
`limit=${String(options.limit)}`,
`timeout_seconds=${String(options.timeoutSeconds)}`,
"tmp_dir=$(mktemp -d)",
"trap 'rm -rf \"$tmp_dir\"' EXIT",
"kubectl -n \"$namespace\" get pipelinerun -o json > \"$tmp_dir/pipelineruns.json\"",
"kubectl -n \"$namespace\" get pvc -o json > \"$tmp_dir/pvcs.json\"",
"kubectl get pv -o json > \"$tmp_dir/pvs.json\"",
"kubectl -n \"$namespace\" get pod -o json > \"$tmp_dir/pods.json\"",
"NAMESPACE=\"$namespace\" PIPELINE_RUN_PREFIX=\"$pipeline_run_prefix\" MIN_AGE_MINUTES=\"$min_age_minutes\" LIMIT=\"$limit\" TMP_DIR=\"$tmp_dir\" node <<'NODE' > \"$tmp_dir/plan.json\"",
cleanupRunsPlanNodeScript(),
"NODE",
"if [ " + shQuote(options.confirm && !options.dryRun ? "true" : "false") + " != true ]; then",
" cat \"$tmp_dir/plan.json\"",
" exit 0",
"fi",
"node -e 'const fs=require(\"node:fs\"); const plan=JSON.parse(fs.readFileSync(process.argv[1],\"utf8\")); const names=Array.isArray(plan.selectedPipelineRuns)?plan.selectedPipelineRuns:[]; fs.writeFileSync(process.argv[2], names.join(\"\\n\") + (names.length>0?\"\\n\":\"\"));' \"$tmp_dir/plan.json\" \"$tmp_dir/selected-names.txt\"",
"delete_exit=0",
"if [ -s \"$tmp_dir/selected-names.txt\" ]; then",
" xargs -r kubectl -n \"$namespace\" delete pipelinerun --ignore-not-found=true --wait=true --timeout=\"${timeout_seconds}s\" < \"$tmp_dir/selected-names.txt\" > \"$tmp_dir/delete.out\" 2> \"$tmp_dir/delete.err\" || delete_exit=$?",
"else",
" : > \"$tmp_dir/delete.out\"",
" : > \"$tmp_dir/delete.err\"",
"fi",
"kubectl -n \"$namespace\" get pvc -o json > \"$tmp_dir/pvcs-after.json\"",
"DELETE_EXIT=\"$delete_exit\" TMP_DIR=\"$tmp_dir\" node <<'NODE'",
cleanupRunsFinalizeNodeScript(),
"NODE",
].join("\n");
}
export function cleanupReleasedPvsScript(options: CleanupReleasedPvOptions, namespace: string): string {
return [
"set -eu",
`namespace=${shQuote(namespace)}`,
`limit=${String(options.limit)}`,
`timeout_seconds=${String(options.timeoutSeconds)}`,
"tmp_dir=$(mktemp -d)",
"trap 'rm -rf \"$tmp_dir\"' EXIT",
"kubectl get pv -o json > \"$tmp_dir/pvs.json\"",
"NAMESPACE=\"$namespace\" LIMIT=\"$limit\" TMP_DIR=\"$tmp_dir\" node <<'NODE' > \"$tmp_dir/plan.json\"",
cleanupReleasedPvsPlanNodeScript(),
"NODE",
"if [ " + shQuote(options.confirm && !options.dryRun ? "true" : "false") + " != true ]; then",
" cat \"$tmp_dir/plan.json\"",
" exit 0",
"fi",
"node -e 'const fs=require(\"node:fs\"); const plan=JSON.parse(fs.readFileSync(process.argv[1],\"utf8\")); const names=Array.isArray(plan.selectedPersistentVolumes)?plan.selectedPersistentVolumes:[]; fs.writeFileSync(process.argv[2], names.join(\"\\n\") + (names.length>0?\"\\n\":\"\"));' \"$tmp_dir/plan.json\" \"$tmp_dir/selected-pvs.txt\"",
"delete_exit=0",
"if [ -s \"$tmp_dir/selected-pvs.txt\" ]; then",
" xargs -r kubectl delete pv --ignore-not-found=true --wait=true --timeout=\"${timeout_seconds}s\" < \"$tmp_dir/selected-pvs.txt\" > \"$tmp_dir/delete.out\" 2> \"$tmp_dir/delete.err\" || delete_exit=$?",
"else",
" : > \"$tmp_dir/delete.out\"",
" : > \"$tmp_dir/delete.err\"",
"fi",
"kubectl get pv -o json > \"$tmp_dir/pvs-after.json\"",
"DELETE_EXIT=\"$delete_exit\" TMP_DIR=\"$tmp_dir\" node <<'NODE'",
cleanupReleasedPvsFinalizeNodeScript(),
"NODE",
].join("\n");
}
export function yamlLaneSourceStatusScript(spec: AgentRunLaneSpec): string {
return [
"set +e",
`expected_workspace=${shQuote(spec.source.workspace)}`,
`source_branch=${shQuote(spec.source.branch)}`,
"workspace_exists=false",
"workspace_clean=null",
"local_head=null",
"branch=null",
"remote_url=null",
"remote_branch_exists=false",
"remote_branch_commit=null",
"status_short=''",
"git config --global --add safe.directory \"$expected_workspace\" 2>/dev/null || true",
"if [ -d .git ] || git rev-parse --show-toplevel >/dev/null 2>&1; then",
" actual_workspace=$(pwd)",
" workspace_exists=true",
" branch=$(git rev-parse --abbrev-ref HEAD 2>/dev/null || true)",
" remote_url=$(git remote get-url origin 2>/dev/null || true)",
" local_head=$(git rev-parse HEAD 2>/dev/null || true)",
" status_short=$(git status --short 2>/dev/null || true)",
" if [ -z \"$status_short\" ]; then workspace_clean=true; else workspace_clean=false; fi",
" git fetch origin \"$source_branch\" >/dev/null 2>&1 || true",
" remote_branch_commit=$(git rev-parse \"refs/remotes/origin/$source_branch\" 2>/dev/null || true)",
" if [ -n \"$remote_branch_commit\" ]; then remote_branch_exists=true; fi",
"else",
" actual_workspace=$(pwd)",
"fi",
"export expected_workspace source_branch workspace_exists workspace_clean local_head branch remote_url remote_branch_exists remote_branch_commit status_short actual_workspace",
"python3 - <<'PY'",
"import json, os",
"def nullable(value):",
" return value if value and value != 'null' else None",
"def boolean_value(value):",
" if value == 'true': return True",
" if value == 'false': return False",
" return None",
"env = os.environ",
"print(json.dumps({",
" 'ok': env.get('workspace_exists') == 'true',",
" 'statusMode': 'host-worktree',",
" 'expectedWorkspace': env.get('expected_workspace'),",
" 'actualWorkspace': env.get('actual_workspace') or None,",
" 'workspaceExists': env.get('workspace_exists') == 'true',",
" 'workspaceClean': boolean_value(env.get('workspace_clean')),",
" 'branch': nullable(env.get('branch')),",
" 'remoteUrl': nullable(env.get('remote_url')),",
" 'localHead': nullable(env.get('local_head')),",
" 'remoteBranch': env.get('source_branch'),",
" 'remoteBranchExists': env.get('remote_branch_exists') == 'true',",
" 'remoteBranchCommit': nullable(env.get('remote_branch_commit')),",
" 'statusShort': nullable(env.get('status_short')),",
" 'valuesPrinted': False,",
"}, ensure_ascii=False))",
"PY",
].join("\n");
}
export function yamlLaneK3sSourceStatusScript(spec: AgentRunLaneSpec): string {
return [
"set +e",
`namespace=${shQuote(spec.gitMirror.namespace)}`,
`read_deployment=${shQuote(spec.gitMirror.readDeployment)}`,
`repository=${shQuote(spec.source.repository)}`,
`source_branch=${shQuote(spec.source.branch)}`,
`source_stage_ref_prefix=${shQuote(agentRunSourceSnapshotStageRefPrefix(spec))}`,
"repo_path=\"/cache/${repository}.git\"",
"kubectl -n \"$namespace\" get deploy \"$read_deployment\" >/tmp/agentrun-source-read-deploy.txt 2>/dev/null",
"read_exit=$?",
"source_commit=''",
"source_ref=\"refs/mirror-stage/heads/$source_branch\"",
"source_stage_ref=''",
"snapshot_commit=''",
"if [ \"$read_exit\" -eq 0 ]; then",
" refs=$(kubectl -n \"$namespace\" exec deploy/\"$read_deployment\" -- sh -lc 'repo_path=\"$1\"; source_branch=\"$2\"; stage_ref_prefix=\"$3\"; source_ref=\"refs/mirror-stage/heads/$source_branch\"; source_commit=$(git --git-dir=\"$repo_path\" rev-parse --verify \"$source_ref^{commit}\" 2>/dev/null || true); source_stage_ref=\"\"; snapshot_commit=\"\"; if printf \"%s\" \"$source_commit\" | grep -Eq \"^[0-9a-fA-F]{40}$\"; then source_stage_ref=\"${stage_ref_prefix%/}/$source_commit\"; snapshot_commit=$(git --git-dir=\"$repo_path\" rev-parse --verify \"$source_stage_ref^{commit}\" 2>/dev/null || true); fi; printf \"source_commit=%s\\nsource_stage_ref=%s\\nsnapshot_commit=%s\\n\" \"$source_commit\" \"$source_stage_ref\" \"$snapshot_commit\"' sh \"$repo_path\" \"$source_branch\" \"$source_stage_ref_prefix\" 2>/dev/null)",
" source_commit=$(printf '%s\\n' \"$refs\" | sed -n 's/^source_commit=//p' | tail -n 1 | tr -d '\\r')",
" source_stage_ref=$(printf '%s\\n' \"$refs\" | sed -n 's/^source_stage_ref=//p' | tail -n 1 | tr -d '\\r')",
" snapshot_commit=$(printf '%s\\n' \"$refs\" | sed -n 's/^snapshot_commit=//p' | tail -n 1 | tr -d '\\r')",
"fi",
"export namespace read_deployment repository source_branch read_exit source_commit source_ref source_stage_ref snapshot_commit",
"python3 - <<'PY'",
"import json, os, re",
"def sha(value):",
" return (value or '').lower() if re.match(r'^[0-9a-f]{40}$', value or '', re.I) else None",
"source_commit = sha(os.environ.get('source_commit'))",
"snapshot_commit = sha(os.environ.get('snapshot_commit'))",
"source_stage_ref = os.environ.get('source_stage_ref') or None",
"read_ready = os.environ.get('read_exit') == '0'",
"snapshot_present = source_commit is not None and snapshot_commit == source_commit",
"degraded_reason = None",
"if not read_ready:",
" degraded_reason = 'git-mirror-read-not-ready'",
"elif source_commit is None:",
" degraded_reason = 'source-branch-missing'",
"elif not snapshot_present:",
" degraded_reason = 'source-snapshot-missing'",
"print(json.dumps({",
" 'ok': read_ready and snapshot_present,",
" 'statusMode': 'k3s-git-mirror',",
" 'sourceAuthority': 'git-mirror-snapshot',",
" 'expectedWorkspace': None,",
" 'actualWorkspace': None,",
" 'workspaceExists': None,",
" 'workspaceClean': None,",
" 'branch': os.environ.get('source_branch'),",
" 'remoteUrl': None,",
" 'localHead': source_commit,",
" 'remoteBranch': os.environ.get('source_branch'),",
" 'remoteBranchExists': source_commit is not None,",
" 'remoteBranchCommit': source_commit,",
" 'sourceCommit': snapshot_commit if snapshot_present else source_commit,",
" 'sourceRef': os.environ.get('source_ref'),",
" 'sourceStageRef': source_stage_ref,",
" 'snapshotPresent': snapshot_present,",
" 'degradedReason': degraded_reason,",
" 'gitMirror': {",
" 'namespace': os.environ.get('namespace'),",
" 'readDeployment': os.environ.get('read_deployment'),",
" 'readReady': read_ready,",
" 'repository': os.environ.get('repository'),",
" 'sourceRef': os.environ.get('source_ref'),",
" 'sourceStageRef': source_stage_ref,",
" },",
" 'statusShort': None,",
" 'valuesPrinted': False,",
"}, ensure_ascii=False))",
"PY",
].join("\n");
}
export function yamlLaneRuntimeStatusScript(spec: AgentRunLaneSpec, pipelineRun: string | null): string {
return [
"set +e",
`runtime_namespace=${shQuote(spec.runtime.namespace)}`,
`ci_namespace=${shQuote(spec.ci.namespace)}`,
`pipeline_name=${shQuote(spec.ci.pipeline)}`,
`pipeline_run=${pipelineRun === null ? "''" : shQuote(pipelineRun)}`,
`service_account=${shQuote(spec.ci.serviceAccountName)}`,
`argo_namespace=${shQuote(spec.gitops.argoNamespace)}`,
`argo_application=${shQuote(spec.gitops.argoApplication)}`,
`manager_deployment=${shQuote(spec.runtime.managerDeployment)}`,
`manager_service=${shQuote(spec.runtime.managerService)}`,
`database_secret=${shQuote(spec.database.secretRef.name)}`,
`database_key=${shQuote(spec.database.secretRef.key)}`,
`secrets_json=${shQuote(JSON.stringify(collectLaneSecretSources(spec).map((item) => item.targetRef)))}`,
"export runtime_namespace ci_namespace pipeline_name pipeline_run service_account argo_namespace argo_application manager_deployment manager_service database_secret database_key secrets_json",
"tmp_dir=$(mktemp -d)",
"trap 'rm -rf \"$tmp_dir\"' EXIT",
"kubectl get ns \"$runtime_namespace\" -o json > \"$tmp_dir/runtime-ns.json\" 2>/dev/null",
"runtime_ns_exit=$?",
"kubectl get ns \"$ci_namespace\" -o json > \"$tmp_dir/ci-ns.json\" 2>/dev/null",
"ci_ns_exit=$?",
"kubectl -n \"$ci_namespace\" get pipeline \"$pipeline_name\" -o json > \"$tmp_dir/pipeline.json\" 2>/dev/null",
"pipeline_exit=$?",
"kubectl -n \"$ci_namespace\" get serviceaccount \"$service_account\" -o json > \"$tmp_dir/serviceaccount.json\" 2>/dev/null",
"sa_exit=$?",
"if [ -n \"$pipeline_run\" ]; then kubectl -n \"$ci_namespace\" get pipelinerun \"$pipeline_run\" -o json > \"$tmp_dir/pipelinerun.json\" 2>/dev/null; pr_exit=$?; else pr_exit=2; fi",
"kubectl -n \"$argo_namespace\" get application \"$argo_application\" -o json > \"$tmp_dir/argo.json\" 2>/dev/null",
"argo_exit=$?",
"kubectl -n \"$runtime_namespace\" get deploy \"$manager_deployment\" -o json > \"$tmp_dir/manager-deploy.json\" 2>/dev/null",
"manager_deploy_exit=$?",
"kubectl -n \"$runtime_namespace\" get svc \"$manager_service\" -o json > \"$tmp_dir/manager-svc.json\" 2>/dev/null",
"manager_svc_exit=$?",
"kubectl -n \"$runtime_namespace\" get secret \"$database_secret\" -o json > \"$tmp_dir/db-secret.json\" 2>/dev/null",
"db_secret_exit=$?",
"SECRET_REFS_JSON=\"$secrets_json\" python3 - <<'PY' > \"$tmp_dir/secrets.json\"",
"import json, os, subprocess",
"refs = json.loads(os.environ.get('SECRET_REFS_JSON') or '[]')",
"result = []",
"for ref in refs:",
" out = subprocess.run(['kubectl', '-n', ref.get('namespace', ''), 'get', 'secret', ref.get('name', ''), '-o', 'json'], text=True, capture_output=True)",
" key_present = False",
" if out.returncode == 0:",
" try:",
" key_present = ref.get('key') in (json.loads(out.stdout).get('data') or {})",
" except Exception:",
" key_present = False",
" result.append({'namespace': ref.get('namespace'), 'name': ref.get('name'), 'key': ref.get('key'), 'present': out.returncode == 0, 'keyPresent': key_present, 'valuesPrinted': False})",
"print(json.dumps({'ready': all(item['present'] and item['keyPresent'] for item in result), 'count': len(result), 'items': result, 'valuesPrinted': False}, ensure_ascii=False))",
"PY",
"kubectl -n \"$runtime_namespace\" get deploy,sts,svc,secret -o name > \"$tmp_dir/runtime-names.txt\" 2>/dev/null",
"NODE_TMP=\"$tmp_dir\" RUNTIME_NS_EXIT=\"$runtime_ns_exit\" CI_NS_EXIT=\"$ci_ns_exit\" PIPELINE_EXIT=\"$pipeline_exit\" SERVICE_ACCOUNT_EXIT=\"$sa_exit\" PIPELINERUN_EXIT=\"$pr_exit\" ARGO_EXIT=\"$argo_exit\" MANAGER_DEPLOY_EXIT=\"$manager_deploy_exit\" MANAGER_SVC_EXIT=\"$manager_svc_exit\" DB_SECRET_EXIT=\"$db_secret_exit\" python3 - <<'PY'",
"import json, os, pathlib, re",
"base = pathlib.Path(os.environ['NODE_TMP'])",
"def read_json(name):",
" try: return json.loads((base / name).read_text())",
" except Exception: return None",
"def exists(name): return os.environ.get(name) == '0'",
"def condition(obj):",
" conds = (((obj or {}).get('status') or {}).get('conditions') or [])",
" return conds[0] if conds else {}",
"def env_value(deploy, name):",
" containers = (((((deploy or {}).get('spec') or {}).get('template') or {}).get('spec') or {}).get('containers') or [])",
" envs = (containers[0].get('env') or []) if containers else []",
" for entry in envs:",
" if entry.get('name') == name: return entry.get('value')",
" return None",
"def pipeline_run_param(obj, name):",
" for entry in (((obj or {}).get('spec') or {}).get('params') or []):",
" if entry.get('name') == name: return entry.get('value')",
" return None",
"pipeline_run = read_json('pipelinerun.json')",
"argo = read_json('argo.json')",
"manager_deploy = read_json('manager-deploy.json')",
"manager_svc = read_json('manager-svc.json')",
"db_secret = read_json('db-secret.json')",
"secrets = read_json('secrets.json') or {'ready': True, 'count': 0, 'items': [], 'valuesPrinted': False}",
"try: names = (base / 'runtime-names.txt').read_text()",
"except Exception: names = ''",
"c = condition(pipeline_run)",
"ports = []",
"for port in ((((manager_svc or {}).get('spec') or {}).get('ports')) or []):",
" ports.append({'name': port.get('name'), 'port': port.get('port'), 'targetPort': port.get('targetPort')})",
"matching_postgres = [line for line in re.split(r'\\r?\\n', names) if re.search('postgres', line, re.I)][:20]",
"print(json.dumps({",
" 'ok': exists('RUNTIME_NS_EXIT') and exists('CI_NS_EXIT'),",
" 'runtimeNamespaceExists': exists('RUNTIME_NS_EXIT'),",
" 'ciNamespaceExists': exists('CI_NS_EXIT'),",
" 'serviceAccountExists': exists('SERVICE_ACCOUNT_EXIT'),",
" 'pipeline': {'exists': exists('PIPELINE_EXIT'), 'name': os.environ.get('pipeline_name')},",
" 'pipelineRun': {'exists': exists('PIPELINERUN_EXIT'), 'name': os.environ.get('pipeline_run') or None, 'status': c.get('status'), 'reason': c.get('reason'), 'sourceCommit': pipeline_run_param(pipeline_run, 'revision'), 'startTime': ((pipeline_run or {}).get('status') or {}).get('startTime'), 'completionTime': ((pipeline_run or {}).get('status') or {}).get('completionTime')},",
" 'argo': {'exists': exists('ARGO_EXIT'), 'namespace': os.environ.get('argo_namespace'), 'application': os.environ.get('argo_application'), 'revision': (((argo or {}).get('status') or {}).get('sync') or {}).get('revision'), 'syncStatus': (((argo or {}).get('status') or {}).get('sync') or {}).get('status'), 'healthStatus': (((argo or {}).get('status') or {}).get('health') or {}).get('status')},",
" 'manager': {'deploymentExists': exists('MANAGER_DEPLOY_EXIT'), 'serviceExists': exists('MANAGER_SVC_EXIT'), 'deployment': os.environ.get('manager_deployment'), 'service': os.environ.get('manager_service'), 'image': ((((((manager_deploy or {}).get('spec') or {}).get('template') or {}).get('spec') or {}).get('containers') or [{}])[0]).get('image'), 'sourceCommit': env_value(manager_deploy, 'AGENTRUN_SOURCE_COMMIT'), 'servicePorts': ports},",
" 'database': {'secretPresent': exists('DB_SECRET_EXIT'), 'secretName': os.environ.get('database_secret'), 'key': os.environ.get('database_key'), 'keyPresent': os.environ.get('database_key') in (((db_secret or {}).get('data') or {})), 'valuesPrinted': False},",
" 'secrets': secrets,",
" 'localPostgres': {'absent': len(matching_postgres) == 0, 'matchingObjects': matching_postgres},",
" 'valuesPrinted': False,",
"}, ensure_ascii=False))",
"PY",
].join("\n");
}
export function yamlLaneSourceBootstrapProbeScript(spec: AgentRunLaneSpec): string {
return [
"set +e",
`workspace=${shQuote(spec.source.workspace)}`,
`remote=${shQuote(spec.source.remote)}`,
`branch=${shQuote(spec.source.branch)}`,
"workspace_exists=false",
"remote_branch_exists=false",
"source_commit=null",
"if [ -d \"$workspace/.git\" ]; then",
" workspace_exists=true",
" cd \"$workspace\"",
" remote_branch=$(git ls-remote --heads \"$remote\" \"$branch\" 2>/dev/null | awk '{print $1}' | head -n 1)",
" if [ -n \"$remote_branch\" ]; then remote_branch_exists=true; source_commit=\"$remote_branch\"; else source_commit=$(git rev-parse HEAD 2>/dev/null || printf null); fi",
"else",
" remote_branch=$(git ls-remote --heads \"$remote\" \"$branch\" 2>/dev/null | awk '{print $1}' | head -n 1)",
" if [ -n \"$remote_branch\" ]; then remote_branch_exists=true; source_commit=\"$remote_branch\"; fi",
"fi",
"export workspace_exists remote_branch_exists source_commit",
"node <<'NODE'",
"console.log(JSON.stringify({ ok: true, workspaceExists: process.env.workspace_exists === 'true', remoteBranchExists: process.env.remote_branch_exists === 'true', sourceCommit: process.env.source_commit === 'null' ? null : process.env.source_commit, valuesPrinted: false }));",
"NODE",
].join("\n");
}
export function yamlLaneSourceBootstrapSubmitScript(spec: AgentRunLaneSpec): string {
const bootstrap = spec.source.bootstrapFromBranch ?? spec.source.branch;
const stateDir = `/tmp/unidesk-agentrun-source-${spec.nodeId}-${spec.lane}`;
return [
"set -eu",
`workspace=${shQuote(spec.source.workspace)}`,
`remote=${shQuote(spec.source.remote)}`,
`branch=${shQuote(spec.source.branch)}`,
`bootstrap_branch=${shQuote(bootstrap)}`,
`state_dir=${shQuote(stateDir)}`,
"mkdir -p \"$state_dir\" \"$(dirname \"$workspace\")\"",
"git_user=''",
"git_home=''",
"case \"$workspace\" in",
" /home/*/*)",
" git_user=${workspace#/home/}",
" git_user=${git_user%%/*}",
" git_home=/home/$git_user",
" if ! id \"$git_user\" >/dev/null 2>&1; then git_user=''; git_home=''; fi",
" ;;",
"esac",
"if [ -n \"$git_user\" ]; then chown \"$git_user:$git_user\" \"$(dirname \"$workspace\")\" 2>/dev/null || true; fi",
"case \"$remote\" in",
" git@*:*)",
" if [ -n \"$git_user\" ] && [ -f \"$git_home/.ssh/id_ed25519\" ]; then",
" if [ -z \"${GIT_SSH_COMMAND:-}\" ]; then GIT_SSH_COMMAND=\"ssh -i $git_home/.ssh/id_ed25519 -o IdentitiesOnly=yes -o StrictHostKeyChecking=accept-new -o UserKnownHostsFile=$git_home/.ssh/known_hosts\"; fi",
" export GIT_SSH_COMMAND",
" else",
" mkdir -p \"$HOME/.ssh\"",
" chmod 700 \"$HOME/.ssh\" 2>/dev/null || true",
" if [ -z \"${GIT_SSH_COMMAND:-}\" ]; then GIT_SSH_COMMAND=\"ssh -o StrictHostKeyChecking=accept-new\"; fi",
" export GIT_SSH_COMMAND",
" fi",
" ;;",
" ssh://*)",
" if [ -n \"$git_user\" ] && [ -f \"$git_home/.ssh/id_ed25519\" ]; then",
" if [ -z \"${GIT_SSH_COMMAND:-}\" ]; then GIT_SSH_COMMAND=\"ssh -i $git_home/.ssh/id_ed25519 -o IdentitiesOnly=yes -o StrictHostKeyChecking=accept-new -o UserKnownHostsFile=$git_home/.ssh/known_hosts\"; fi",
" export GIT_SSH_COMMAND",
" else",
" mkdir -p \"$HOME/.ssh\"",
" chmod 700 \"$HOME/.ssh\" 2>/dev/null || true",
" if [ -z \"${GIT_SSH_COMMAND:-}\" ]; then GIT_SSH_COMMAND=\"ssh -o StrictHostKeyChecking=accept-new\"; fi",
" export GIT_SSH_COMMAND",
" fi",
" ;;",
"esac",
"git_cmd() {",
" if [ -n \"$git_user\" ]; then",
" sudo -u \"$git_user\" env HOME=\"$git_home\" GIT_SSH_COMMAND=\"$GIT_SSH_COMMAND\" git \"$@\"",
" else",
" git \"$@\"",
" fi",
"}",
"job_id=\"source-bootstrap-$(date +%s)-$$\"",
"status_file=\"$state_dir/$job_id.json\"",
"stdout_file=\"$state_dir/$job_id.stdout.log\"",
"stderr_file=\"$state_dir/$job_id.stderr.log\"",
"cat > \"$status_file\" <<EOF",
"{\"ok\":false,\"status\":\"running\",\"jobId\":\"$job_id\",\"workspace\":\"$workspace\",\"branch\":\"$branch\",\"valuesPrinted\":false}",
"EOF",
"(",
" set -eu",
" write_failed_status() { code=$?; if [ \"$code\" -ne 0 ]; then tail_text=$(tail -n 60 \"$stderr_file\" 2>/dev/null | sed 's/\"/\\\\\"/g' | tr '\\n' ' ' | cut -c1-3000); CODE=\"$code\" ERROR_TAIL=\"$tail_text\" JOB_ID=\"$job_id\" WORKSPACE=\"$workspace\" BRANCH=\"$branch\" node <<'NODE' > \"$status_file\"",
"const code = Number(process.env.CODE || 1);",
"console.log(JSON.stringify({ ok: false, status: 'failed', exitCode: code, jobId: process.env.JOB_ID, workspace: process.env.WORKSPACE, branch: process.env.BRANCH, errorTail: process.env.ERROR_TAIL || null, valuesPrinted: false }));",
"NODE",
" fi; exit \"$code\"; }",
" trap write_failed_status EXIT",
" if [ -d \"$workspace/.git\" ] && git_cmd -C \"$workspace\" rev-parse --git-dir >/dev/null 2>&1; then",
" :",
" else",
" rm -rf \"$workspace\"",
" git_cmd clone --no-checkout \"$remote\" \"$workspace\"",
" fi",
" cd \"$workspace\"",
" git_cmd remote set-url origin \"$remote\" || git_cmd remote add origin \"$remote\"",
" git_cmd fetch origin \"$bootstrap_branch\" \"$branch\" || git_cmd fetch origin \"$bootstrap_branch\"",
" if git_cmd rev-parse --verify \"refs/remotes/origin/$branch^{commit}\" >/dev/null 2>&1; then",
" git_cmd checkout -B \"$branch\" \"refs/remotes/origin/$branch\"",
" else",
" git_cmd checkout -B \"$branch\" \"refs/remotes/origin/$bootstrap_branch\"",
" fi",
" if [ -f deploy/deploy.json ]; then rm deploy/deploy.json; fi",
" git_cmd add -A deploy/deploy.json 2>/dev/null || true",
" if ! git_cmd diff --quiet --cached -- deploy/deploy.json 2>/dev/null; then",
" git_cmd -c user.email=agentrun@unidesk.local -c user.name='UniDesk AgentRun Ops' commit -m 'chore: remove service deploy json truth'",
" fi",
" git_cmd push -u origin \"$branch\"",
" source_commit=$(git_cmd rev-parse HEAD)",
" status_short=$(git_cmd status --short)",
" SOURCE_COMMIT=\"$source_commit\" STATUS_SHORT=\"$status_short\" JOB_ID=\"$job_id\" WORKSPACE=\"$workspace\" BRANCH=\"$branch\" node <<'NODE' > \"$status_file\"",
"console.log(JSON.stringify({ ok: process.env.STATUS_SHORT === '', status: 'succeeded', jobId: process.env.JOB_ID, workspace: process.env.WORKSPACE, branch: process.env.BRANCH, sourceCommit: process.env.SOURCE_COMMIT, workspaceClean: process.env.STATUS_SHORT === '', statusShort: process.env.STATUS_SHORT || null, removedServiceDeployJson: true, valuesPrinted: false }));",
"NODE",
" trap - EXIT",
") >\"$stdout_file\" 2>\"$stderr_file\" &",
"pid=$!",
"JOB_PID=\"$pid\" JOB_ID=\"$job_id\" STATUS_FILE=\"$status_file\" STDOUT_FILE=\"$stdout_file\" STDERR_FILE=\"$stderr_file\" node <<'NODE'",
"console.log(JSON.stringify({ ok: true, status: 'submitted', jobId: process.env.JOB_ID, pid: Number(process.env.JOB_PID), statusFile: process.env.STATUS_FILE, stdoutFile: process.env.STDOUT_FILE, stderrFile: process.env.STDERR_FILE, valuesPrinted: false }));",
"NODE",
].join("\n");
}
export async function waitForYamlLaneSourceBootstrap(config: UniDeskConfig, spec: AgentRunLaneSpec, jobId: string | null): Promise<Record<string, unknown> & { ok: boolean; payload: Record<string, unknown> }> {
if (jobId === null) return { ok: false, payload: { ok: false, degradedReason: "source-bootstrap-job-id-missing", valuesPrinted: false } };
const startedAt = Date.now();
const timeoutMs = spec.source.bootstrapTimeoutSeconds * 1000;
const pollMs = spec.source.bootstrapPollSeconds * 1000;
let lastPayload: Record<string, unknown> = {};
let polls = 0;
while (Date.now() - startedAt < timeoutMs) {
polls += 1;
const probe = await capture(config, spec.nodeRoute, ["sh", "--", yamlLaneSourceBootstrapStatusScript(spec, jobId)]);
const payload = captureJsonPayload(probe);
lastPayload = payload;
progressEvent("agentrun.yaml-lane.source-bootstrap.progress", {
node: spec.nodeId,
lane: spec.lane,
jobId,
polls,
status: stringOrNull(payload.status) ?? "unknown",
sourceCommit: stringOrNull(payload.sourceCommit),
elapsedMs: Date.now() - startedAt,
});
if (payload.status === "succeeded" && stringOrNull(payload.sourceCommit) !== null) return { ok: payload.ok === true, payload, polls, elapsedMs: Date.now() - startedAt };
if (payload.status === "failed") return { ok: false, payload, polls, elapsedMs: Date.now() - startedAt };
await sleep(pollMs);
}
return { ok: false, payload: { ...lastPayload, ok: false, status: "timeout", degradedReason: "source-bootstrap-timeout", valuesPrinted: false }, polls, elapsedMs: Date.now() - startedAt };
}
export function yamlLaneSourceBootstrapStatusScript(spec: AgentRunLaneSpec, jobId: string): string {
const stateDir = `/tmp/unidesk-agentrun-source-${spec.nodeId}-${spec.lane}`;
return [
"set +e",
`status_file=${shQuote(`${stateDir}/${jobId}.json`)}`,
"if [ -f \"$status_file\" ]; then cat \"$status_file\"; else printf '{\"ok\":false,\"status\":\"missing\",\"valuesPrinted\":false}\\n'; fi",
].join("\n");
}
export function yamlLaneSourceRestoreScript(spec: AgentRunLaneSpec): string {
return [
"set +e",
`workspace=${shQuote(spec.source.workspace)}`,
`remote=${shQuote(spec.source.remote)}`,
`branch=${shQuote(spec.source.branch)}`,
"tmp_dir=$(mktemp -d)",
"trap 'rm -rf \"$tmp_dir\"' EXIT",
"git_user=''",
"git_home=''",
"case \"$workspace\" in",
" /home/*/*)",
" git_user=${workspace#/home/}",
" git_user=${git_user%%/*}",
" git_home=/home/$git_user",
" if ! id \"$git_user\" >/dev/null 2>&1; then git_user=''; git_home=''; fi",
" ;;",
"esac",
"case \"$remote\" in",
" git@*:*)",
" if [ -n \"$git_user\" ] && [ -f \"$git_home/.ssh/id_ed25519\" ]; then",
" if [ -z \"${GIT_SSH_COMMAND:-}\" ]; then GIT_SSH_COMMAND=\"ssh -i $git_home/.ssh/id_ed25519 -o IdentitiesOnly=yes -o StrictHostKeyChecking=accept-new -o UserKnownHostsFile=$git_home/.ssh/known_hosts\"; fi",
" export GIT_SSH_COMMAND",
" else",
" mkdir -p \"$HOME/.ssh\"",
" chmod 700 \"$HOME/.ssh\" 2>/dev/null || true",
" if [ -z \"${GIT_SSH_COMMAND:-}\" ]; then GIT_SSH_COMMAND=\"ssh -o StrictHostKeyChecking=accept-new\"; fi",
" export GIT_SSH_COMMAND",
" fi",
" ;;",
" ssh://*)",
" if [ -n \"$git_user\" ] && [ -f \"$git_home/.ssh/id_ed25519\" ]; then",
" if [ -z \"${GIT_SSH_COMMAND:-}\" ]; then GIT_SSH_COMMAND=\"ssh -i $git_home/.ssh/id_ed25519 -o IdentitiesOnly=yes -o StrictHostKeyChecking=accept-new -o UserKnownHostsFile=$git_home/.ssh/known_hosts\"; fi",
" export GIT_SSH_COMMAND",
" else",
" mkdir -p \"$HOME/.ssh\"",
" chmod 700 \"$HOME/.ssh\" 2>/dev/null || true",
" if [ -z \"${GIT_SSH_COMMAND:-}\" ]; then GIT_SSH_COMMAND=\"ssh -o StrictHostKeyChecking=accept-new\"; fi",
" export GIT_SSH_COMMAND",
" fi",
" ;;",
"esac",
"git_cmd() {",
" if [ -n \"$git_user\" ]; then",
" sudo -u \"$git_user\" env HOME=\"$git_home\" GIT_SSH_COMMAND=\"$GIT_SSH_COMMAND\" git \"$@\"",
" else",
" git \"$@\"",
" fi",
"}",
"workspace_exists=false",
"git config --global --add safe.directory \"$workspace\" 2>/dev/null || true",
"git_cmd config --global --add safe.directory \"$workspace\" 2>/dev/null || true",
"if git_cmd -C \"$workspace\" rev-parse --git-dir >/dev/null 2>&1; then workspace_exists=true; fi",
"if [ \"$workspace_exists\" != true ]; then",
" WORKSPACE=\"$workspace\" BRANCH=\"$branch\" node <<'NODE'",
"console.log(JSON.stringify({ ok: false, status: 'skipped', failureKind: 'source-worktree-missing', workspace: process.env.WORKSPACE, branch: process.env.BRANCH, valuesPrinted: false }));",
"NODE",
" exit 0",
"fi",
"before_branch=$(git_cmd -C \"$workspace\" rev-parse --abbrev-ref HEAD 2>/dev/null || true)",
"before_head=$(git_cmd -C \"$workspace\" rev-parse HEAD 2>/dev/null || true)",
"status_short=$(git_cmd -C \"$workspace\" status --short 2>/dev/null || true)",
"if [ -n \"$status_short\" ]; then",
" WORKSPACE=\"$workspace\" BRANCH=\"$branch\" BEFORE_BRANCH=\"$before_branch\" BEFORE_HEAD=\"$before_head\" STATUS_SHORT=\"$status_short\" node <<'NODE'",
"console.log(JSON.stringify({ ok: false, status: 'skipped', failureKind: 'source-worktree-dirty', workspace: process.env.WORKSPACE, branch: process.env.BRANCH, before: { branch: process.env.BEFORE_BRANCH || null, head: process.env.BEFORE_HEAD || null, detached: process.env.BEFORE_BRANCH === 'HEAD' }, statusShort: process.env.STATUS_SHORT || null, valuesPrinted: false }));",
"NODE",
" exit 0",
"fi",
"git_cmd -C \"$workspace\" fetch origin \"$branch\" > \"$tmp_dir/fetch.out\" 2> \"$tmp_dir/fetch.err\"",
"fetch_exit=$?",
"remote_branch_commit=$(git_cmd -C \"$workspace\" rev-parse \"refs/remotes/origin/$branch\" 2>/dev/null || true)",
"checkout_exit=1",
"if [ \"$fetch_exit\" -eq 0 ] && [ -n \"$remote_branch_commit\" ]; then",
" git_cmd -C \"$workspace\" checkout -B \"$branch\" \"refs/remotes/origin/$branch\" > \"$tmp_dir/checkout.out\" 2> \"$tmp_dir/checkout.err\"",
" checkout_exit=$?",
"else",
" : > \"$tmp_dir/checkout.out\"",
" : > \"$tmp_dir/checkout.err\"",
"fi",
"after_branch=$(git_cmd -C \"$workspace\" rev-parse --abbrev-ref HEAD 2>/dev/null || true)",
"after_head=$(git_cmd -C \"$workspace\" rev-parse HEAD 2>/dev/null || true)",
"after_status_short=$(git_cmd -C \"$workspace\" status --short 2>/dev/null || true)",
"fetch_err=$(tail -n 20 \"$tmp_dir/fetch.err\" 2>/dev/null | tr '\\n' ' ' | cut -c1-1200)",
"checkout_err=$(tail -n 20 \"$tmp_dir/checkout.err\" 2>/dev/null | tr '\\n' ' ' | cut -c1-1200)",
"WORKSPACE=\"$workspace\" BRANCH=\"$branch\" BEFORE_BRANCH=\"$before_branch\" BEFORE_HEAD=\"$before_head\" FETCH_EXIT=\"$fetch_exit\" CHECKOUT_EXIT=\"$checkout_exit\" REMOTE_BRANCH_COMMIT=\"$remote_branch_commit\" AFTER_BRANCH=\"$after_branch\" AFTER_HEAD=\"$after_head\" AFTER_STATUS_SHORT=\"$after_status_short\" FETCH_ERR=\"$fetch_err\" CHECKOUT_ERR=\"$checkout_err\" node <<'NODE'",
"const fetchExit = Number(process.env.FETCH_EXIT || '1');",
"const checkoutExit = Number(process.env.CHECKOUT_EXIT || '1');",
"const afterClean = !process.env.AFTER_STATUS_SHORT;",
"const ok = fetchExit === 0 && checkoutExit === 0 && afterClean && process.env.AFTER_BRANCH === process.env.BRANCH;",
"let failureKind = null;",
"if (fetchExit !== 0) failureKind = 'source-branch-fetch-failed';",
"else if (!process.env.REMOTE_BRANCH_COMMIT) failureKind = 'source-branch-missing';",
"else if (checkoutExit !== 0) failureKind = 'source-branch-checkout-failed';",
"else if (!afterClean) failureKind = 'source-worktree-dirty-after-restore';",
"else if (process.env.AFTER_BRANCH !== process.env.BRANCH) failureKind = 'source-worktree-branch-mismatch';",
"console.log(JSON.stringify({",
" ok,",
" status: ok ? 'restored' : 'failed',",
" failureKind,",
" workspace: process.env.WORKSPACE,",
" branch: process.env.BRANCH,",
" before: { branch: process.env.BEFORE_BRANCH || null, head: process.env.BEFORE_HEAD || null, detached: process.env.BEFORE_BRANCH === 'HEAD' },",
" after: { branch: process.env.AFTER_BRANCH || null, head: process.env.AFTER_HEAD || null, clean: afterClean },",
" remoteBranchCommit: process.env.REMOTE_BRANCH_COMMIT || null,",
" fetch: { exitCode: fetchExit, stderrTail: process.env.FETCH_ERR || null },",
" checkout: { exitCode: checkoutExit, stderrTail: process.env.CHECKOUT_ERR || null },",
" valuesPrinted: false",
"}));",
"NODE",
].join("\n");
}
export function yamlLaneBuildImageSubmitScript(spec: AgentRunLaneSpec, sourceCommit: string): string {
const build = spec.deployment.manager.imageBuild;
const noProxy = build.noProxy.join(",");
const buildContainerNoProxy = build.buildContainerProxy.noProxy.join(",");
const imageRepository = `${spec.ci.registryPrefix}/${build.repository}`;
const stateDir = `/tmp/unidesk-agentrun-build-${spec.nodeId}-${spec.lane}`;
const buildArgs = Object.entries(build.buildArgs)
.sort(([left], [right]) => left.localeCompare(right))
.map(([key, value]) => `${key}=${value}`);
const buildContainerProxyIdentity = [
`HTTP_PROXY=${build.buildContainerProxy.httpProxy ?? ""}`,
`HTTPS_PROXY=${build.buildContainerProxy.httpsProxy ?? ""}`,
`NO_PROXY=${buildContainerNoProxy}`,
];
const script = [
"set -eu",
`workspace=${shQuote(spec.source.workspace)}`,
`source_commit=${shQuote(sourceCommit)}`,
`state_dir=${shQuote(stateDir)}`,
`containerfile=${shQuote(build.containerfile)}`,
`context_dir=${shQuote(build.context)}`,
`image_repository=${shQuote(imageRepository)}`,
`network=${shQuote(build.network)}`,
`http_proxy_value=${build.httpProxy === null ? "''" : shQuote(build.httpProxy)}`,
`https_proxy_value=${build.httpsProxy === null ? "''" : shQuote(build.httpsProxy)}`,
`no_proxy_value=${shQuote(noProxy)}`,
`build_container_http_proxy_value=${build.buildContainerProxy.httpProxy === null ? "''" : shQuote(build.buildContainerProxy.httpProxy)}`,
`build_container_https_proxy_value=${build.buildContainerProxy.httpsProxy === null ? "''" : shQuote(build.buildContainerProxy.httpsProxy)}`,
`build_container_no_proxy_value=${shQuote(buildContainerNoProxy)}`,
`env_identity_files=${shQuote(JSON.stringify(build.envIdentityFiles))}`,
`build_args_json=${shQuote(JSON.stringify(buildArgs))}`,
`build_container_proxy_identity_json=${shQuote(JSON.stringify(buildContainerProxyIdentity))}`,
"mkdir -p \"$state_dir\"",
"git config --global --add safe.directory \"$workspace\" 2>/dev/null || true",
"cd \"$workspace\"",
"git checkout \"$source_commit\"",
"env_identity=$(ENV_IDENTITY_FILES=\"$env_identity_files\" BUILD_ARGS_JSON=\"$build_args_json\" BUILD_CONTAINER_PROXY_IDENTITY_JSON=\"$build_container_proxy_identity_json\" node <<'NODE'",
"const { createHash } = require('node:crypto');",
"const { readFileSync, existsSync, lstatSync, readdirSync } = require('node:fs');",
"const { join } = require('node:path');",
"const files = JSON.parse(process.env.ENV_IDENTITY_FILES || '[]');",
"const buildArgs = JSON.parse(process.env.BUILD_ARGS_JSON || '[]');",
"const buildContainerProxyIdentity = JSON.parse(process.env.BUILD_CONTAINER_PROXY_IDENTITY_JSON || '[]');",
"const hash = createHash('sha256');",
"const skipDirNames = new Set(['.git', '.worktree', '.state', 'node_modules', 'coverage', 'tmp', '.tmp']);",
"function collectIdentityFiles(input) {",
" if (!existsSync(input)) return [{ path: input, missing: true }];",
" const stat = lstatSync(input);",
" if (stat.isFile()) return [{ path: input, missing: false }];",
" if (!stat.isDirectory()) return [{ path: input, missing: true }];",
" const out = [];",
" const stack = [input];",
" while (stack.length > 0) {",
" const dir = stack.pop();",
" const entries = readdirSync(dir, { withFileTypes: true }).sort((left, right) => left.name.localeCompare(right.name));",
" for (const entry of entries) {",
" if (entry.isDirectory() && skipDirNames.has(entry.name)) continue;",
" const child = join(dir, entry.name);",
" if (entry.isDirectory()) stack.push(child);",
" else if (entry.isFile()) out.push({ path: child, missing: false });",
" }",
" }",
" return out.sort((left, right) => left.path.localeCompare(right.path));",
"}",
"for (const item of buildArgs) { hash.update('build-arg'); hash.update('\\0'); hash.update(item); hash.update('\\0'); }",
"for (const item of buildContainerProxyIdentity) { hash.update('build-container-proxy'); hash.update('\\0'); hash.update(item); hash.update('\\0'); }",
"for (const file of files) {",
" for (const entry of collectIdentityFiles(file)) {",
" hash.update(entry.path); hash.update('\\0');",
" if (!entry.missing) hash.update(readFileSync(entry.path));",
" hash.update('\\0');",
" }",
"}",
"process.stdout.write(hash.digest('hex').slice(0, 24));",
"NODE",
")",
"source_short=$(printf '%s' \"$source_commit\" | cut -c1-12)",
"job_id=\"$source_short-$env_identity\"",
"status_file=\"$state_dir/$job_id.json\"",
"stdout_file=\"$state_dir/$job_id.stdout.log\"",
"stderr_file=\"$state_dir/$job_id.stderr.log\"",
"cat > \"$status_file\" <<EOF",
"{\"ok\":false,\"status\":\"running\",\"jobId\":\"$job_id\",\"sourceCommit\":\"$source_commit\",\"envIdentity\":\"$env_identity\",\"image\":\"$image_repository:$env_identity\",\"valuesPrinted\":false}",
"EOF",
"(",
" set -eu",
" write_failed_status() { code=$?; if [ \"$code\" -ne 0 ]; then tail_text=$(tail -n 40 \"$stderr_file\" 2>/dev/null | sed 's/\"/\\\\\"/g' | tr '\\n' ' ' | cut -c1-2000); CODE=\"$code\" ERROR_TAIL=\"$tail_text\" JOB_ID=\"$job_id\" SOURCE_COMMIT=\"$source_commit\" ENV_IDENTITY=\"$env_identity\" IMAGE_REPOSITORY=\"$image_repository\" node <<'NODE' > \"$status_file\"",
"const code = Number(process.env.CODE || 1);",
"console.log(JSON.stringify({ ok: false, status: 'failed', exitCode: code, jobId: process.env.JOB_ID, sourceCommit: process.env.SOURCE_COMMIT, envIdentity: process.env.ENV_IDENTITY, image: `${process.env.IMAGE_REPOSITORY}:${process.env.ENV_IDENTITY}`, errorTail: process.env.ERROR_TAIL || null, valuesPrinted: false }));",
"NODE",
" fi; exit \"$code\"; }",
" trap write_failed_status EXIT",
" cd \"$workspace\"",
" image=\"$image_repository:$env_identity\"",
" all_proxy_value=${https_proxy_value:-$http_proxy_value}",
" if [ -n \"$http_proxy_value\" ]; then export HTTP_PROXY=\"$http_proxy_value\" http_proxy=\"$http_proxy_value\"; fi",
" if [ -n \"$https_proxy_value\" ]; then export HTTPS_PROXY=\"$https_proxy_value\" https_proxy=\"$https_proxy_value\"; fi",
" if [ -n \"$all_proxy_value\" ]; then export ALL_PROXY=\"$all_proxy_value\" all_proxy=\"$all_proxy_value\"; fi",
" if [ -n \"$no_proxy_value\" ]; then export NO_PROXY=\"$no_proxy_value\" no_proxy=\"$no_proxy_value\"; fi",
" args=\"--network $network\"",
" build_container_all_proxy_value=${build_container_https_proxy_value:-$build_container_http_proxy_value}",
" if [ -n \"$build_container_http_proxy_value\" ]; then args=\"$args --build-arg HTTP_PROXY=$build_container_http_proxy_value --build-arg http_proxy=$build_container_http_proxy_value\"; fi",
" if [ -n \"$build_container_https_proxy_value\" ]; then args=\"$args --build-arg HTTPS_PROXY=$build_container_https_proxy_value --build-arg https_proxy=$build_container_https_proxy_value\"; fi",
" if [ -n \"$build_container_all_proxy_value\" ]; then args=\"$args --build-arg ALL_PROXY=$build_container_all_proxy_value --build-arg all_proxy=$build_container_all_proxy_value\"; fi",
" if [ -n \"$build_container_no_proxy_value\" ]; then args=\"$args --build-arg NO_PROXY=$build_container_no_proxy_value --build-arg no_proxy=$build_container_no_proxy_value\"; fi",
" build_arg_values=$(BUILD_ARGS_JSON=\"$build_args_json\" node <<'NODE'",
"const values = JSON.parse(process.env.BUILD_ARGS_JSON || '[]');",
"for (const value of values) console.log(value);",
"NODE",
" )",
" while IFS= read -r build_arg_value; do [ -n \"$build_arg_value\" ] && args=\"$args --build-arg $build_arg_value\"; done <<EOF",
"$build_arg_values",
"EOF",
" if docker image inspect \"$image\" >/dev/null 2>&1; then build_status=reused; else docker build $args -f \"$containerfile\" -t \"$image\" \"$context_dir\"; build_status=built; fi",
" manifest_accept='application/vnd.oci.image.index.v1+json, application/vnd.oci.image.manifest.v1+json, application/vnd.docker.distribution.manifest.v2+json'",
" fetch_manifest_digest() {",
" ref=\"$1\"",
" attempt=1",
" max_attempts=30",
" while [ \"$attempt\" -le \"$max_attempts\" ]; do",
" headers=$(mktemp)",
" if curl -fsSI -H \"Accept: $manifest_accept\" \"http://127.0.0.1:5000/v2/${image_repository#127.0.0.1:5000/}/manifests/$ref\" >\"$headers\"; then",
" found_digest=$(awk -F': ' 'tolower($1)==\"docker-content-digest\"{print $2}' \"$headers\" | tr -d '\\r' | head -n 1)",
" rm -f \"$headers\"",
" if [ -n \"$found_digest\" ]; then printf '%s' \"$found_digest\"; return 0; fi",
" else",
" rm -f \"$headers\"",
" fi",
" printf '{\"event\":\"agentrun-registry-manifest-probe\",\"status\":\"retrying\",\"ref\":\"%s\",\"attempt\":\"%s/%s\"}\\n' \"$ref\" \"$attempt\" \"$max_attempts\" >&2",
" sleep 1",
" attempt=$((attempt + 1))",
" done",
" printf '{\"event\":\"agentrun-registry-manifest-probe\",\"status\":\"exhausted\",\"ref\":\"%s\",\"attempt\":\"%s/%s\"}\\n' \"$ref\" \"$max_attempts\" \"$max_attempts\" >&2",
" return 1",
" }",
" docker push \"$image\"",
" digest=$(fetch_manifest_digest \"$env_identity\" || true)",
" if [ -n \"$digest\" ]; then verified_digest=$(fetch_manifest_digest \"$digest\" || true); if [ \"$verified_digest\" != \"$digest\" ]; then digest=; fi; fi",
" STATUS=\"$build_status\" DIGEST=\"$digest\" JOB_ID=\"$job_id\" SOURCE_COMMIT=\"$source_commit\" ENV_IDENTITY=\"$env_identity\" IMAGE_REPOSITORY=\"$image_repository\" node <<'NODE' > \"$status_file\"",
"const digest = process.env.DIGEST || null;",
"console.log(JSON.stringify({ ok: Boolean(digest), status: process.env.STATUS || 'built', jobId: process.env.JOB_ID, sourceCommit: process.env.SOURCE_COMMIT, envIdentity: process.env.ENV_IDENTITY, image: `${process.env.IMAGE_REPOSITORY}:${process.env.ENV_IDENTITY}`, digest, repositoryDigest: digest ? `${process.env.IMAGE_REPOSITORY}@${digest}` : null, valuesPrinted: false }));",
"NODE",
" trap - EXIT",
") >\"$stdout_file\" 2>\"$stderr_file\" &",
"pid=$!",
"JOB_PID=\"$pid\" JOB_ID=\"$job_id\" STATUS_FILE=\"$status_file\" STDOUT_FILE=\"$stdout_file\" STDERR_FILE=\"$stderr_file\" node <<'NODE'",
"console.log(JSON.stringify({ ok: true, status: 'submitted', jobId: process.env.JOB_ID, pid: Number(process.env.JOB_PID), statusFile: process.env.STATUS_FILE, stdoutFile: process.env.STDOUT_FILE, stderrFile: process.env.STDERR_FILE, valuesPrinted: false }));",
"NODE",
].join("\n");
return script;
}
export async function waitForYamlLaneBuildImage(config: UniDeskConfig, spec: AgentRunLaneSpec, sourceCommit: string, jobId: string | null): Promise<Record<string, unknown> & { ok: boolean; payload: Record<string, unknown> }> {
if (jobId === null) return { ok: false, payload: { ok: false, degradedReason: "build-job-id-missing", valuesPrinted: false } };
const startedAt = Date.now();
const timeoutMs = Math.max(60, spec.deployment.manager.imageBuild.timeoutSeconds) * 1000;
let lastPayload: Record<string, unknown> = {};
let polls = 0;
while (Date.now() - startedAt < timeoutMs) {
polls += 1;
const probe = await capture(config, spec.nodeRoute, ["sh", "--", yamlLaneBuildImageStatusScript(spec, jobId)]);
const payload = captureJsonPayload(probe);
lastPayload = payload;
progressEvent("agentrun.yaml-lane.image-build.progress", {
node: spec.nodeId,
lane: spec.lane,
sourceCommit,
jobId,
polls,
status: stringOrNull(payload.status) ?? "unknown",
elapsedMs: Date.now() - startedAt,
});
if (payload.ok === true && stringOrNull(payload.digest) !== null) return { ok: true, payload, polls, elapsedMs: Date.now() - startedAt };
if (payload.status === "failed") return { ok: false, payload, polls, elapsedMs: Date.now() - startedAt };
await sleep(spec.deployment.manager.imageBuild.pollSeconds * 1000);
}
return { ok: false, payload: { ...lastPayload, ok: false, status: "timeout", degradedReason: "image-build-timeout", valuesPrinted: false }, polls, elapsedMs: Date.now() - startedAt };
}
export function yamlLaneBuildImageStatusScript(spec: AgentRunLaneSpec, jobId: string): string {
const stateDir = `/tmp/unidesk-agentrun-build-${spec.nodeId}-${spec.lane}`;
return [
"set +e",
`status_file=${shQuote(`${stateDir}/${jobId}.json`)}`,
`stdout_file=${shQuote(`${stateDir}/${jobId}.stdout.log`)}`,
`stderr_file=${shQuote(`${stateDir}/${jobId}.stderr.log`)}`,
"if [ -f \"$status_file\" ]; then cat \"$status_file\"; else printf '{\"ok\":false,\"status\":\"missing\",\"valuesPrinted\":false}\\n'; fi",
"if [ -f \"$stderr_file\" ] && tail -n 20 \"$stderr_file\" | grep -Eq 'ERROR|error|failed|denied'; then :; fi",
].join("\n");
}
export function yamlLaneK3sBuildImageJobManifest(spec: AgentRunLaneSpec, sourceCommit: string, jobName: string): Record<string, unknown> {
const build = spec.deployment.manager.imageBuild;
if (spec.ci.buildkitImage === null) throw new Error(`config/agentrun.yaml controlPlane.lanes.${spec.lane}.ci.buildkitImage is required when source.statusMode=k3s-git-mirror`);
const imageRepository = `${spec.ci.registryPrefix}/${build.repository}`;
const envIdentity = `source-${sourceCommit.slice(0, 12)}`;
const image = `${imageRepository}:${envIdentity}`;
const proxyEnv = yamlLaneK3sBuildProxyEnv(spec);
return {
apiVersion: "batch/v1",
kind: "Job",
metadata: {
name: jobName,
namespace: spec.ci.namespace,
labels: {
"app.kubernetes.io/name": "agentrun-manager-image-build",
"app.kubernetes.io/part-of": "agentrun",
"agentrun.pikastech.local/lane": spec.version,
"agentrun.pikastech.local/node": spec.nodeId,
"agentrun.pikastech.local/source-commit": sourceCommit,
},
},
spec: {
backoffLimit: 0,
activeDeadlineSeconds: Math.max(60, build.timeoutSeconds),
ttlSecondsAfterFinished: 3600,
template: {
metadata: {
labels: {
"app.kubernetes.io/name": "agentrun-manager-image-build",
"app.kubernetes.io/part-of": "agentrun",
"agentrun.pikastech.local/lane": spec.version,
"agentrun.pikastech.local/node": spec.nodeId,
"agentrun.pikastech.local/source-commit": sourceCommit,
},
},
spec: {
restartPolicy: "Never",
serviceAccountName: spec.ci.serviceAccountName,
hostNetwork: true,
dnsPolicy: "ClusterFirstWithHostNet",
securityContext: { fsGroup: 1000 },
volumes: [
{ name: "workspace", emptyDir: { sizeLimit: "8Gi" } },
{ name: "buildkit-state", emptyDir: { sizeLimit: "8Gi" } },
{ name: "tmp", emptyDir: {} },
],
initContainers: [{
name: "source",
image: spec.ci.toolsImage,
imagePullPolicy: "IfNotPresent",
env: proxyEnv,
command: ["/bin/sh", "-ec", yamlLaneK3sBuildSourceShell(spec, sourceCommit)],
volumeMounts: [{ name: "workspace", mountPath: "/workspace" }],
}],
containers: [{
name: "buildkit",
image: spec.ci.buildkitImage,
imagePullPolicy: "IfNotPresent",
env: [
...proxyEnv,
{ name: "BUILDKITD_FLAGS", value: "--oci-worker-no-process-sandbox --oci-worker-net=host --allow-insecure-entitlement network.host" },
],
command: ["/bin/sh", "-ec", yamlLaneK3sBuildImageShell(spec, sourceCommit, image, envIdentity)],
securityContext: { privileged: true, runAsUser: 1000, runAsGroup: 1000 },
volumeMounts: [
{ name: "workspace", mountPath: "/workspace" },
{ name: "buildkit-state", mountPath: "/home/user/.local/share/buildkit" },
{ name: "tmp", mountPath: "/tmp" },
],
}],
},
},
},
};
}
export async function runYamlLaneK3sBuildImageJob(config: UniDeskConfig, spec: AgentRunLaneSpec, sourceCommit: string): Promise<Record<string, unknown> & { ok: boolean; payload: Record<string, unknown> }> {
const jobName = `agentrun-build-${spec.nodeId.toLowerCase()}-${spec.lane}-${sourceCommit.slice(0, 12)}`.slice(0, 63);
const manifest = yamlLaneK3sBuildImageJobManifest(spec, sourceCommit, jobName);
const created = await capture(config, spec.nodeKubeRoute, ["sh", "--", createYamlLaneJobScript(spec.ci.namespace, jobName, manifest)]);
const createPayload = captureJsonPayload(created);
if (created.exitCode !== 0 || createPayload.ok === false) {
return {
ok: false,
payload: { ok: false, status: "create-failed", jobName, degradedReason: "k3s-buildkit-job-create-failed", valuesPrinted: false },
create: createPayload,
capture: compactCapture(created, { full: true, stdoutTailChars: 5000, stderrTailChars: 5000 }),
valuesPrinted: false,
};
}
const startedAt = Date.now();
const timeoutMs = Math.max(60, spec.deployment.manager.imageBuild.timeoutSeconds) * 1000;
const pollMs = Math.max(1, spec.deployment.manager.imageBuild.pollSeconds) * 1000;
let polls = 0;
let lastPayload: Record<string, unknown> = {};
let lastProbe: SshCaptureResult | null = null;
while (Date.now() - startedAt < timeoutMs) {
polls += 1;
lastProbe = await capture(config, spec.nodeKubeRoute, ["sh", "--", yamlLaneJobProbeScript(spec.ci.namespace, jobName)]);
const probePayload = captureJsonPayload(lastProbe);
const buildPayload = yamlLaneGitopsPublishPayloadFromProbe(probePayload);
if (Object.keys(buildPayload).length > 0) lastPayload = buildPayload;
progressEvent("agentrun.yaml-lane.k3s-image-build.progress", {
node: spec.nodeId,
lane: spec.lane,
sourceCommit,
jobName,
polls,
status: stringOrNull(buildPayload.status) ?? (probePayload.succeeded === true ? "succeeded" : probePayload.failed === true ? "failed" : "running"),
digest: stringOrNull(buildPayload.digest),
elapsedMs: Date.now() - startedAt,
});
if (probePayload.succeeded === true) {
if (buildPayload.ok === true && stringOrNull(buildPayload.digest) !== null && stringOrNull(buildPayload.envIdentity) !== null) {
return { ok: true, payload: buildPayload, jobName, create: createPayload, polls, elapsedMs: Date.now() - startedAt, probe: compactCapture(lastProbe), valuesPrinted: false };
}
return {
ok: false,
payload: { ...buildPayload, ok: false, status: stringOrNull(buildPayload.status) ?? "succeeded-without-result", degradedReason: "k3s-buildkit-result-missing", jobName, valuesPrinted: false },
jobName,
create: createPayload,
polls,
elapsedMs: Date.now() - startedAt,
probe: compactCapture(lastProbe, { full: true, stdoutTailChars: 5000, stderrTailChars: 5000 }),
valuesPrinted: false,
};
}
if (probePayload.failed === true) {
const payload = Object.keys(buildPayload).length > 0 ? buildPayload : { ok: false, status: "failed", degradedReason: "k3s-buildkit-job-failed", jobName, valuesPrinted: false };
return {
ok: false,
payload,
jobName,
create: createPayload,
polls,
elapsedMs: Date.now() - startedAt,
probe: compactCapture(lastProbe, { full: true, stdoutTailChars: 5000, stderrTailChars: 5000 }),
valuesPrinted: false,
};
}
await sleep(pollMs);
}
return {
ok: false,
payload: { ...lastPayload, ok: false, status: "timeout", degradedReason: "k3s-buildkit-job-timeout", jobName, valuesPrinted: false },
jobName,
create: createPayload,
polls,
elapsedMs: Date.now() - startedAt,
probe: lastProbe === null ? null : compactCapture(lastProbe, { full: true, stdoutTailChars: 5000, stderrTailChars: 5000 }),
valuesPrinted: false,
};
}
function yamlLaneK3sBuildProxyEnv(spec: AgentRunLaneSpec): Array<{ name: string; value: string }> {
const build = spec.deployment.manager.imageBuild;
const result: Array<{ name: string; value: string }> = [];
const noProxy = build.noProxy.join(",");
const allProxy = build.httpsProxy ?? build.httpProxy;
if (build.httpProxy !== null) result.push({ name: "HTTP_PROXY", value: build.httpProxy }, { name: "http_proxy", value: build.httpProxy });
if (build.httpsProxy !== null) result.push({ name: "HTTPS_PROXY", value: build.httpsProxy }, { name: "https_proxy", value: build.httpsProxy });
if (allProxy !== null) result.push({ name: "ALL_PROXY", value: allProxy }, { name: "all_proxy", value: allProxy });
result.push({ name: "NO_PROXY", value: noProxy }, { name: "no_proxy", value: noProxy });
return result;
}
function yamlLaneK3sBuildSourceShell(spec: AgentRunLaneSpec, sourceCommit: string): string {
return [
"set -eu",
`read_url=${shQuote(spec.gitMirror.readUrl)}`,
`source_commit=${shQuote(sourceCommit)}`,
`source_stage_ref=${shQuote(agentRunSourceSnapshotRef(spec, sourceCommit))}`,
"rm -rf /workspace/repo",
"git clone --no-checkout \"$read_url\" /workspace/repo",
"cd /workspace/repo",
"git fetch origin \"+$source_stage_ref:refs/remotes/origin/unidesk-source-snapshot\"",
"git checkout --detach \"$source_commit\"",
"actual=$(git rev-parse HEAD)",
"test \"$actual\" = \"$source_commit\"",
"chmod -R a+rwX /workspace",
].join("\n");
}
function yamlLaneK3sBuildImageShell(spec: AgentRunLaneSpec, sourceCommit: string, image: string, envIdentity: string): string {
const build = spec.deployment.manager.imageBuild;
const contextPath = build.context === "." ? "/workspace/repo" : `/workspace/repo/${build.context.replace(/^\.\//u, "")}`;
const buildContainerNoProxy = build.buildContainerProxy.noProxy.join(",");
const buildArgs = [
...Object.entries(build.buildArgs).sort(([left], [right]) => left.localeCompare(right)).map(([key, value]) => `build-arg:${key}=${value}`),
...yamlLaneK3sBuildProxyBuildArgs(build.buildContainerProxy.httpProxy, build.buildContainerProxy.httpsProxy, buildContainerNoProxy),
];
const buildctlArgs = [
"build",
"--allow", "network.host",
"--frontend", "dockerfile.v0",
"--local", `context=${contextPath}`,
"--local", "dockerfile=/workspace/repo",
"--opt", `filename=${build.containerfile}`,
"--opt", `network=${build.network}`,
...buildArgs.flatMap((arg) => ["--opt", arg]),
"--metadata-file", "/workspace/build-metadata.json",
"--output", `type=image,name=${image},push=true,registry.insecure=true`,
];
const repositoryDigestBase = image.slice(0, image.lastIndexOf(":"));
return [
"set -eu",
"cd /workspace/repo",
`buildctl-daemonless.sh ${buildctlArgs.map((arg) => shQuote(arg)).join(" ")}`,
"metadata_compact=$(tr -d '\\n' < /workspace/build-metadata.json)",
"digest=$(printf '%s' \"$metadata_compact\" | sed -n 's/.*\"containerimage.digest\"[[:space:]]*:[[:space:]]*\"\\([^\"]*\\)\".*/\\1/p' | head -n 1)",
"if [ -z \"$digest\" ]; then",
` printf '{"ok":false,"status":"failed","failureKind":"image-digest-missing","sourceCommit":"%s","envIdentity":"%s","image":"%s","valuesPrinted":false}\\n' ${shQuote(sourceCommit)} ${shQuote(envIdentity)} ${shQuote(image)}`,
" exit 1",
"fi",
`printf '{"ok":true,"status":"built","sourceCommit":"%s","envIdentity":"%s","image":"%s","digest":"%s","repositoryDigest":"%s@%s","valuesPrinted":false}\\n' ${shQuote(sourceCommit)} ${shQuote(envIdentity)} ${shQuote(image)} "$digest" ${shQuote(repositoryDigestBase)} "$digest"`,
].join("\n");
}
function yamlLaneK3sBuildProxyBuildArgs(httpProxy: string | null, httpsProxy: string | null, noProxy: string): string[] {
const result: string[] = [];
const allProxy = httpsProxy ?? httpProxy;
if (httpProxy !== null) result.push(`build-arg:HTTP_PROXY=${httpProxy}`, `build-arg:http_proxy=${httpProxy}`);
if (httpsProxy !== null) result.push(`build-arg:HTTPS_PROXY=${httpsProxy}`, `build-arg:https_proxy=${httpsProxy}`);
if (allProxy !== null) result.push(`build-arg:ALL_PROXY=${allProxy}`, `build-arg:all_proxy=${allProxy}`);
if (noProxy.length > 0) result.push(`build-arg:NO_PROXY=${noProxy}`, `build-arg:no_proxy=${noProxy}`);
return result;
}
export async function runYamlLaneGitopsPublishJob(config: UniDeskConfig, spec: AgentRunLaneSpec, sourceCommit: string, files: readonly { path: string; content: string }[]): Promise<Record<string, unknown> & { ok: boolean; payload: Record<string, unknown> }> {
const jobName = `gitops-publish-${spec.nodeId.toLowerCase()}-${spec.lane}-${Date.now().toString(36)}`.slice(0, 63);
const manifest = yamlLaneGitopsPublishJobManifest(spec, files, jobName);
const created = await capture(config, spec.nodeKubeRoute, ["sh", "--", createYamlLaneJobScript(spec.gitMirror.namespace, jobName, manifest)]);
if (created.exitCode !== 0) {
return {
ok: false,
payload: { ok: false, status: "create-failed", jobName, degradedReason: "gitops-publish-job-create-failed", valuesPrinted: false },
jobName,
sourceCommit,
phase: "create-job",
capture: compactCapture(created, { full: true, stdoutTailChars: 5000, stderrTailChars: 5000 }),
valuesPrinted: false,
};
}
const startedAt = Date.now();
const timeoutMs = 300_000;
let polls = 0;
let lastPayload: Record<string, unknown> = {};
let lastProbe: SshCaptureResult | null = null;
while (Date.now() - startedAt < timeoutMs) {
polls += 1;
lastProbe = await capture(config, spec.nodeKubeRoute, ["sh", "--", yamlLaneJobProbeScript(spec.gitMirror.namespace, jobName)]);
const probePayload = captureJsonPayload(lastProbe);
const publishPayload = yamlLaneGitopsPublishPayloadFromProbe(probePayload);
if (Object.keys(publishPayload).length > 0) lastPayload = publishPayload;
progressEvent("agentrun.yaml-lane.gitops-publish.progress", {
node: spec.nodeId,
lane: spec.lane,
sourceCommit,
jobName,
polls,
status: stringOrNull(publishPayload.status) ?? (probePayload.succeeded === true ? "succeeded" : probePayload.failed === true ? "failed" : "running"),
gitopsCommit: stringOrNull(publishPayload.gitopsCommit),
elapsedMs: Date.now() - startedAt,
});
if (probePayload.succeeded === true) {
if (publishPayload.ok === true && stringOrNull(publishPayload.gitopsCommit) !== null) {
return { ok: true, payload: publishPayload, jobName, polls, elapsedMs: Date.now() - startedAt, probe: compactCapture(lastProbe), valuesPrinted: false };
}
return {
ok: false,
payload: { ...publishPayload, ok: false, status: stringOrNull(publishPayload.status) ?? "succeeded-without-result", degradedReason: "gitops-publish-result-missing", jobName, valuesPrinted: false },
jobName,
polls,
elapsedMs: Date.now() - startedAt,
probe: compactCapture(lastProbe, { full: true, stdoutTailChars: 5000, stderrTailChars: 5000 }),
valuesPrinted: false,
};
}
if (probePayload.failed === true) {
const payload = Object.keys(publishPayload).length > 0 ? publishPayload : { ok: false, status: "failed", degradedReason: "gitops-publish-job-failed", jobName, valuesPrinted: false };
return {
ok: false,
payload,
jobName,
polls,
elapsedMs: Date.now() - startedAt,
probe: compactCapture(lastProbe, { full: true, stdoutTailChars: 5000, stderrTailChars: 5000 }),
valuesPrinted: false,
};
}
await sleep(5_000);
}
return {
ok: false,
payload: { ...lastPayload, ok: false, status: "timeout", degradedReason: "gitops-publish-job-timeout", jobName, valuesPrinted: false },
jobName,
polls,
elapsedMs: Date.now() - startedAt,
probe: lastProbe === null ? null : compactCapture(lastProbe, { full: true, stdoutTailChars: 5000, stderrTailChars: 5000 }),
valuesPrinted: false,
};
}
Reference in New Issue View Git Blame Copy Permalink