Codex
912 lines
52 KiB
TypeScript
912 lines
52 KiB
TypeScript
// SPEC: PJ2026-01060307 控制面模块化 draft-2026-06-25-p0. yaml-lane module for scripts/src/agentrun.ts.
|
|
|
|
// Moved mechanically from scripts/src/agentrun.ts:3311-4219 for #903.
|
|
|
|
// SPEC: PJ2026-01020108 cancel lifecycle + PJ2026-01020205 AipodSpec binding + PJ2026-01020302 session policy + PJ2026-01020305 cancel control + PJ2026-01060305/06 YAML execution policy and bounded output draft-2026-06-25-p0.
|
|
// Exposes AgentRun lane-scoped policy, AipodSpec SecretRef binding, cancel lifecycle, and bounded default output in the UniDesk CLI.
|
|
import { chmodSync, copyFileSync, existsSync, readFileSync, statSync, writeFileSync } from "node:fs";
|
|
import { join } from "node:path";
|
|
import { spawnSync } from "node:child_process";
|
|
import { rootPath, type UniDeskConfig } from "../config";
|
|
import type { RenderedCliResult } from "../output";
|
|
import { applyLocalCaddyManagedSite } from "../pk01-caddy";
|
|
import { runSshCommandCapture, type SshCaptureResult } from "../ssh";
|
|
import { runRemoteSshCommandCapture } from "../remote";
|
|
import { startJob } from "../jobs";
|
|
import {
|
|
AGENTRUN_CONFIG_PATH,
|
|
agentRunLaneSummary,
|
|
agentRunPipelineRunName,
|
|
agentRunProviderCredentialRefs,
|
|
resolveAgentRunLaneTarget,
|
|
type AgentRunCancelLifecycleSpec,
|
|
type AgentRunLaneSpec,
|
|
} from "../agentrun-lanes";
|
|
import {
|
|
agentRunImageArtifact,
|
|
placeholderAgentRunImage,
|
|
renderedFilesDigest,
|
|
renderedObjectsDigest,
|
|
renderAgentRunControlPlaneManifests,
|
|
renderAgentRunGitopsFiles,
|
|
type AgentRunArtifactService,
|
|
} from "../agentrun-manifests";
|
|
import { sha256Fingerprint } from "../platform-infra-ops-library";
|
|
|
|
import type { CleanupReleasedPvOptions, CleanupRunnersOptions, CleanupRunsOptions, RefreshOptions } from "./options";
|
|
import { cleanupReleasedPvsFinalizeNodeScript, cleanupReleasedPvsPlanNodeScript, cleanupRunnersFinalizeNodeScript, cleanupRunsFinalizeNodeScript, cleanupRunsPlanNodeScript, refreshYamlLaneScript } from "./git-mirror";
|
|
import { cleanupRunnersFactsNodeScript, cleanupRunnersPlanNodeScript, collectLaneSecretSources, createYamlLaneJobScript, yamlLaneGitopsPublishJobManifest, yamlLaneGitopsPublishPayloadFromProbe, yamlLaneJobProbeScript } from "./secrets";
|
|
import { capture, captureJsonPayload, compactCapture, progressEvent, shQuote, sleep, stringOrNull } from "./utils";
|
|
|
|
export async function refresh(config: UniDeskConfig, options: RefreshOptions): Promise<Record<string, unknown>> {
|
|
return await refreshYamlLane(config, options);
|
|
}
|
|
|
|
export async function refreshYamlLane(config: UniDeskConfig, options: RefreshOptions): Promise<Record<string, unknown>> {
|
|
const { configPath, spec } = resolveAgentRunLaneTarget(options);
|
|
const plan = {
|
|
node: spec.nodeId,
|
|
lane: spec.lane,
|
|
version: spec.version,
|
|
argoNamespace: spec.gitops.argoNamespace,
|
|
argoApplication: spec.gitops.argoApplication,
|
|
gitopsBranch: spec.gitops.branch,
|
|
mutation: "argocd-hard-refresh",
|
|
valuesPrinted: false,
|
|
};
|
|
if (options.dryRun || !options.confirm) {
|
|
return {
|
|
ok: true,
|
|
command: "agentrun control-plane refresh",
|
|
mode: "dry-run",
|
|
mutation: false,
|
|
configPath,
|
|
target: agentRunLaneSummary(spec),
|
|
plan,
|
|
next: {
|
|
confirm: `bun scripts/cli.ts agentrun control-plane refresh --node ${spec.nodeId} --lane ${spec.lane} --confirm`,
|
|
status: `bun scripts/cli.ts agentrun control-plane status --node ${spec.nodeId} --lane ${spec.lane}`,
|
|
},
|
|
valuesPrinted: false,
|
|
};
|
|
}
|
|
const refreshed = await capture(config, spec.nodeKubeRoute, ["sh", "--", refreshYamlLaneScript(spec)]);
|
|
const payload = captureJsonPayload(refreshed);
|
|
return {
|
|
ok: refreshed.exitCode === 0 && payload.ok !== false,
|
|
command: "agentrun control-plane refresh",
|
|
mode: "confirmed-yaml-lane",
|
|
mutation: true,
|
|
configPath,
|
|
target: agentRunLaneSummary(spec),
|
|
plan,
|
|
result: payload,
|
|
refreshed: compactCapture(refreshed, { full: refreshed.exitCode !== 0, stdoutTailChars: 3000, stderrTailChars: 3000 }),
|
|
next: {
|
|
status: `bun scripts/cli.ts agentrun control-plane status --node ${spec.nodeId} --lane ${spec.lane}`,
|
|
},
|
|
valuesPrinted: false,
|
|
};
|
|
}
|
|
|
|
export async function cleanupRunners(config: UniDeskConfig, options: CleanupRunnersOptions): Promise<Record<string, unknown>> {
|
|
const { configPath, spec } = resolveAgentRunLaneTarget(options);
|
|
const result = await capture(config, spec.nodeKubeRoute, ["sh", "--", cleanupRunnersScript(options, spec)]);
|
|
const payload = captureJsonPayload(result);
|
|
const ok = result.exitCode === 0 && payload.ok !== false;
|
|
const base = {
|
|
...payload,
|
|
ok,
|
|
command: "agentrun control-plane cleanup-runners",
|
|
configPath,
|
|
target: agentRunLaneSummary(spec),
|
|
mode: options.dryRun || !options.confirm ? "dry-run" : "confirmed-cleanup",
|
|
namespace: spec.runtime.namespace,
|
|
retention: spec.deployment.runner.retention,
|
|
probe: compactCapture(result, { full: result.exitCode !== 0, stdoutTailChars: 3000, stderrTailChars: 3000 }),
|
|
};
|
|
if (options.dryRun || !options.confirm) {
|
|
return {
|
|
...base,
|
|
dryRun: true,
|
|
mutation: false,
|
|
next: {
|
|
confirm: `bun scripts/cli.ts agentrun control-plane cleanup-runners --node ${spec.nodeId} --lane ${spec.lane}${options.forceActive ? " --force-active" : ""} --confirm`,
|
|
},
|
|
};
|
|
}
|
|
return {
|
|
...base,
|
|
dryRun: false,
|
|
mutation: true,
|
|
followUp: {
|
|
dryRun: `bun scripts/cli.ts agentrun control-plane cleanup-runners --node ${spec.nodeId} --lane ${spec.lane}${options.forceActive ? " --force-active" : ""} --dry-run`,
|
|
status: `bun scripts/cli.ts agentrun control-plane status --node ${spec.nodeId} --lane ${spec.lane}`,
|
|
},
|
|
};
|
|
}
|
|
|
|
export async function cleanupRuns(config: UniDeskConfig, options: CleanupRunsOptions): Promise<Record<string, unknown>> {
|
|
const { configPath, spec } = resolveAgentRunLaneTarget(options);
|
|
const result = await capture(config, spec.nodeKubeRoute, ["sh", "--", cleanupRunsScript(options, spec.ci.namespace, spec.ci.pipelineRunPrefix)]);
|
|
const payload = captureJsonPayload(result);
|
|
const ok = result.exitCode === 0 && payload.ok !== false;
|
|
const base = {
|
|
...payload,
|
|
ok,
|
|
command: "agentrun control-plane cleanup-runs",
|
|
configPath,
|
|
target: agentRunLaneSummary(spec),
|
|
mode: options.dryRun || !options.confirm ? "dry-run" : "confirmed-cleanup",
|
|
namespace: spec.ci.namespace,
|
|
minAgeMinutes: options.minAgeMinutes,
|
|
limit: options.limit,
|
|
probe: compactCapture(result, { full: result.exitCode !== 0, stdoutTailChars: 3000, stderrTailChars: 3000 }),
|
|
};
|
|
if (options.dryRun || !options.confirm) {
|
|
return {
|
|
...base,
|
|
dryRun: true,
|
|
mutation: false,
|
|
next: {
|
|
confirm: `bun scripts/cli.ts agentrun control-plane cleanup-runs --node ${spec.nodeId} --lane ${spec.lane} --min-age-minutes ${options.minAgeMinutes} --limit ${options.limit} --confirm`,
|
|
},
|
|
};
|
|
}
|
|
return {
|
|
...base,
|
|
dryRun: false,
|
|
mutation: true,
|
|
followUp: {
|
|
status: `bun scripts/cli.ts agentrun control-plane status --node ${spec.nodeId} --lane ${spec.lane}`,
|
|
releasedPvs: `bun scripts/cli.ts agentrun control-plane cleanup-released-pvs --node ${spec.nodeId} --lane ${spec.lane} --limit ${options.limit} --dry-run`,
|
|
diskPressure: `trans ${spec.nodeKubeRoute} kubectl get node -o jsonpath='{range .items[*]}{.metadata.name}{\"\\t\"}{range .status.conditions[*]}{.type}{\"=\"}{.status}{\" \"}{.reason}{\";\"}{end}{\"\\n\"}{end}'`,
|
|
},
|
|
};
|
|
}
|
|
|
|
export async function cleanupReleasedPvs(config: UniDeskConfig, options: CleanupReleasedPvOptions): Promise<Record<string, unknown>> {
|
|
const { configPath, spec } = resolveAgentRunLaneTarget(options);
|
|
const result = await capture(config, spec.nodeKubeRoute, ["sh", "--", cleanupReleasedPvsScript(options, spec.ci.namespace)]);
|
|
const payload = captureJsonPayload(result);
|
|
const ok = result.exitCode === 0 && payload.ok !== false;
|
|
const base = {
|
|
...payload,
|
|
ok,
|
|
command: "agentrun control-plane cleanup-released-pvs",
|
|
configPath,
|
|
target: agentRunLaneSummary(spec),
|
|
mode: options.dryRun || !options.confirm ? "dry-run" : "confirmed-cleanup",
|
|
namespace: spec.ci.namespace,
|
|
limit: options.limit,
|
|
probe: compactCapture(result, { full: result.exitCode !== 0, stdoutTailChars: 3000, stderrTailChars: 3000 }),
|
|
};
|
|
if (options.dryRun || !options.confirm) {
|
|
return {
|
|
...base,
|
|
dryRun: true,
|
|
mutation: false,
|
|
next: {
|
|
confirm: `bun scripts/cli.ts agentrun control-plane cleanup-released-pvs --node ${spec.nodeId} --lane ${spec.lane} --limit ${options.limit} --confirm`,
|
|
},
|
|
};
|
|
}
|
|
return {
|
|
...base,
|
|
dryRun: false,
|
|
mutation: true,
|
|
followUp: {
|
|
cleanupRuns: `bun scripts/cli.ts agentrun control-plane cleanup-runs --node ${spec.nodeId} --lane ${spec.lane} --min-age-minutes 30 --limit ${options.limit} --dry-run`,
|
|
diskPressure: `trans ${spec.nodeKubeRoute} kubectl get node -o jsonpath='{range .items[*]}{.metadata.name}{\"\\t\"}{range .status.conditions[*]}{.type}{\"=\"}{.status}{\" \"}{.reason}{\";\"}{end}{\"\\n\"}{end}'`,
|
|
},
|
|
};
|
|
}
|
|
|
|
export function cleanupRunnersScript(options: CleanupRunnersOptions, spec: AgentRunLaneSpec): string {
|
|
const retention = spec.deployment.runner.retention;
|
|
const matchLabelsB64 = Buffer.from(JSON.stringify(retention.selectors.matchLabels), "utf8").toString("base64");
|
|
const jobNamePrefixesB64 = Buffer.from(JSON.stringify(retention.selectors.jobNamePrefixes), "utf8").toString("base64");
|
|
return [
|
|
"set -eu",
|
|
`namespace=${shQuote(spec.runtime.namespace)}`,
|
|
`manager_deployment=${shQuote(spec.runtime.managerDeployment)}`,
|
|
`max_runners=${String(retention.maxRunners)}`,
|
|
`cleanup_order=${shQuote(retention.cleanupOrder)}`,
|
|
`active_heartbeat_max_age_ms=${String(retention.activeHeartbeatMaxAgeMs)}`,
|
|
`age_based_cleanup_enabled=${retention.ageBasedCleanup.enabled ? "true" : "false"}`,
|
|
`age_based_max_age_hours=${retention.ageBasedCleanup.maxAgeHours === null ? "" : String(retention.ageBasedCleanup.maxAgeHours)}`,
|
|
`timeout_seconds=${String(options.timeoutSeconds)}`,
|
|
`force_active=${options.forceActive ? "true" : "false"}`,
|
|
`match_labels_json_b64=${shQuote(matchLabelsB64)}`,
|
|
`job_name_prefixes_json_b64=${shQuote(jobNamePrefixesB64)}`,
|
|
"match_labels_json=$(printf '%s' \"$match_labels_json_b64\" | base64 -d)",
|
|
"job_name_prefixes_json=$(printf '%s' \"$job_name_prefixes_json_b64\" | base64 -d)",
|
|
"tmp_dir=$(mktemp -d)",
|
|
"trap 'rm -rf \"$tmp_dir\"' EXIT",
|
|
"kubectl -n \"$namespace\" get job -o json > \"$tmp_dir/jobs.json\"",
|
|
"kubectl -n \"$namespace\" get pod -o json > \"$tmp_dir/pods.json\"",
|
|
"facts_exit=0",
|
|
"set +e",
|
|
"kubectl -n \"$namespace\" exec -i deploy/\"$manager_deployment\" -- env RETENTION_NAMESPACE=\"$namespace\" sh -lc 'cat >/tmp/agentrun-runner-retention.mjs && bun /tmp/agentrun-runner-retention.mjs' > \"$tmp_dir/runner-facts.json\" 2> \"$tmp_dir/runner-facts.err\" <<'NODE'",
|
|
cleanupRunnersFactsNodeScript(),
|
|
"NODE",
|
|
"facts_exit=$?",
|
|
"set -e",
|
|
"if [ \"$facts_exit\" -ne 0 ]; then",
|
|
" printf '%s\\n' '{\"ok\":false,\"items\":[],\"failureKind\":\"manager-facts-unavailable\",\"valuesPrinted\":false}' > \"$tmp_dir/runner-facts.json\"",
|
|
"fi",
|
|
"MATCH_LABELS_JSON=\"$match_labels_json\" JOB_NAME_PREFIXES_JSON=\"$job_name_prefixes_json\" MAX_RUNNERS=\"$max_runners\" CLEANUP_ORDER=\"$cleanup_order\" ACTIVE_HEARTBEAT_MAX_AGE_MS=\"$active_heartbeat_max_age_ms\" AGE_BASED_CLEANUP_ENABLED=\"$age_based_cleanup_enabled\" AGE_BASED_MAX_AGE_HOURS=\"$age_based_max_age_hours\" FORCE_ACTIVE=\"$force_active\" FACTS_EXIT=\"$facts_exit\" TMP_DIR=\"$tmp_dir\" NAMESPACE=\"$namespace\" node <<'NODE' > \"$tmp_dir/plan.json\"",
|
|
cleanupRunnersPlanNodeScript(),
|
|
"NODE",
|
|
"if [ " + shQuote(options.confirm && !options.dryRun ? "true" : "false") + " != true ]; then",
|
|
" cat \"$tmp_dir/plan.json\"",
|
|
" exit 0",
|
|
"fi",
|
|
"node -e 'const fs=require(\"node:fs\"); const plan=JSON.parse(fs.readFileSync(process.argv[1],\"utf8\")); const names=Array.isArray(plan.selectedRunnerJobs)?plan.selectedRunnerJobs:[]; fs.writeFileSync(process.argv[2], names.join(\"\\n\") + (names.length>0?\"\\n\":\"\"));' \"$tmp_dir/plan.json\" \"$tmp_dir/selected-jobs.txt\"",
|
|
"delete_exit=0",
|
|
"if [ -s \"$tmp_dir/selected-jobs.txt\" ]; then",
|
|
" xargs -r kubectl -n \"$namespace\" delete job --ignore-not-found=true --wait=true --timeout=\"${timeout_seconds}s\" < \"$tmp_dir/selected-jobs.txt\" > \"$tmp_dir/delete.out\" 2> \"$tmp_dir/delete.err\" || delete_exit=$?",
|
|
"else",
|
|
" : > \"$tmp_dir/delete.out\"",
|
|
" : > \"$tmp_dir/delete.err\"",
|
|
"fi",
|
|
"kubectl -n \"$namespace\" get job -o json > \"$tmp_dir/jobs-after.json\"",
|
|
"kubectl -n \"$namespace\" get pod -o json > \"$tmp_dir/pods-after.json\"",
|
|
"DELETE_EXIT=\"$delete_exit\" TMP_DIR=\"$tmp_dir\" node <<'NODE'",
|
|
cleanupRunnersFinalizeNodeScript(),
|
|
"NODE",
|
|
].join("\n");
|
|
}
|
|
|
|
export function cleanupRunsScript(options: CleanupRunsOptions, namespace: string, pipelineRunPrefix: string): string {
|
|
return [
|
|
"set -eu",
|
|
`namespace=${shQuote(namespace)}`,
|
|
`pipeline_run_prefix=${shQuote(pipelineRunPrefix)}`,
|
|
`min_age_minutes=${String(options.minAgeMinutes)}`,
|
|
`limit=${String(options.limit)}`,
|
|
`timeout_seconds=${String(options.timeoutSeconds)}`,
|
|
"tmp_dir=$(mktemp -d)",
|
|
"trap 'rm -rf \"$tmp_dir\"' EXIT",
|
|
"kubectl -n \"$namespace\" get pipelinerun -o json > \"$tmp_dir/pipelineruns.json\"",
|
|
"kubectl -n \"$namespace\" get pvc -o json > \"$tmp_dir/pvcs.json\"",
|
|
"kubectl get pv -o json > \"$tmp_dir/pvs.json\"",
|
|
"kubectl -n \"$namespace\" get pod -o json > \"$tmp_dir/pods.json\"",
|
|
"NAMESPACE=\"$namespace\" PIPELINE_RUN_PREFIX=\"$pipeline_run_prefix\" MIN_AGE_MINUTES=\"$min_age_minutes\" LIMIT=\"$limit\" TMP_DIR=\"$tmp_dir\" node <<'NODE' > \"$tmp_dir/plan.json\"",
|
|
cleanupRunsPlanNodeScript(),
|
|
"NODE",
|
|
"if [ " + shQuote(options.confirm && !options.dryRun ? "true" : "false") + " != true ]; then",
|
|
" cat \"$tmp_dir/plan.json\"",
|
|
" exit 0",
|
|
"fi",
|
|
"node -e 'const fs=require(\"node:fs\"); const plan=JSON.parse(fs.readFileSync(process.argv[1],\"utf8\")); const names=Array.isArray(plan.selectedPipelineRuns)?plan.selectedPipelineRuns:[]; fs.writeFileSync(process.argv[2], names.join(\"\\n\") + (names.length>0?\"\\n\":\"\"));' \"$tmp_dir/plan.json\" \"$tmp_dir/selected-names.txt\"",
|
|
"delete_exit=0",
|
|
"if [ -s \"$tmp_dir/selected-names.txt\" ]; then",
|
|
" xargs -r kubectl -n \"$namespace\" delete pipelinerun --ignore-not-found=true --wait=true --timeout=\"${timeout_seconds}s\" < \"$tmp_dir/selected-names.txt\" > \"$tmp_dir/delete.out\" 2> \"$tmp_dir/delete.err\" || delete_exit=$?",
|
|
"else",
|
|
" : > \"$tmp_dir/delete.out\"",
|
|
" : > \"$tmp_dir/delete.err\"",
|
|
"fi",
|
|
"kubectl -n \"$namespace\" get pvc -o json > \"$tmp_dir/pvcs-after.json\"",
|
|
"DELETE_EXIT=\"$delete_exit\" TMP_DIR=\"$tmp_dir\" node <<'NODE'",
|
|
cleanupRunsFinalizeNodeScript(),
|
|
"NODE",
|
|
].join("\n");
|
|
}
|
|
|
|
export function cleanupReleasedPvsScript(options: CleanupReleasedPvOptions, namespace: string): string {
|
|
return [
|
|
"set -eu",
|
|
`namespace=${shQuote(namespace)}`,
|
|
`limit=${String(options.limit)}`,
|
|
`timeout_seconds=${String(options.timeoutSeconds)}`,
|
|
"tmp_dir=$(mktemp -d)",
|
|
"trap 'rm -rf \"$tmp_dir\"' EXIT",
|
|
"kubectl get pv -o json > \"$tmp_dir/pvs.json\"",
|
|
"NAMESPACE=\"$namespace\" LIMIT=\"$limit\" TMP_DIR=\"$tmp_dir\" node <<'NODE' > \"$tmp_dir/plan.json\"",
|
|
cleanupReleasedPvsPlanNodeScript(),
|
|
"NODE",
|
|
"if [ " + shQuote(options.confirm && !options.dryRun ? "true" : "false") + " != true ]; then",
|
|
" cat \"$tmp_dir/plan.json\"",
|
|
" exit 0",
|
|
"fi",
|
|
"node -e 'const fs=require(\"node:fs\"); const plan=JSON.parse(fs.readFileSync(process.argv[1],\"utf8\")); const names=Array.isArray(plan.selectedPersistentVolumes)?plan.selectedPersistentVolumes:[]; fs.writeFileSync(process.argv[2], names.join(\"\\n\") + (names.length>0?\"\\n\":\"\"));' \"$tmp_dir/plan.json\" \"$tmp_dir/selected-pvs.txt\"",
|
|
"delete_exit=0",
|
|
"if [ -s \"$tmp_dir/selected-pvs.txt\" ]; then",
|
|
" xargs -r kubectl delete pv --ignore-not-found=true --wait=true --timeout=\"${timeout_seconds}s\" < \"$tmp_dir/selected-pvs.txt\" > \"$tmp_dir/delete.out\" 2> \"$tmp_dir/delete.err\" || delete_exit=$?",
|
|
"else",
|
|
" : > \"$tmp_dir/delete.out\"",
|
|
" : > \"$tmp_dir/delete.err\"",
|
|
"fi",
|
|
"kubectl get pv -o json > \"$tmp_dir/pvs-after.json\"",
|
|
"DELETE_EXIT=\"$delete_exit\" TMP_DIR=\"$tmp_dir\" node <<'NODE'",
|
|
cleanupReleasedPvsFinalizeNodeScript(),
|
|
"NODE",
|
|
].join("\n");
|
|
}
|
|
|
|
export function yamlLaneSourceStatusScript(spec: AgentRunLaneSpec): string {
|
|
return [
|
|
"set +e",
|
|
`expected_workspace=${shQuote(spec.source.workspace)}`,
|
|
`source_branch=${shQuote(spec.source.branch)}`,
|
|
"workspace_exists=false",
|
|
"workspace_clean=null",
|
|
"local_head=null",
|
|
"branch=null",
|
|
"remote_url=null",
|
|
"remote_branch_exists=false",
|
|
"remote_branch_commit=null",
|
|
"status_short=''",
|
|
"if [ -d .git ] || git rev-parse --show-toplevel >/dev/null 2>&1; then",
|
|
" actual_workspace=$(pwd)",
|
|
" workspace_exists=true",
|
|
" branch=$(git rev-parse --abbrev-ref HEAD 2>/dev/null || true)",
|
|
" remote_url=$(git remote get-url origin 2>/dev/null || true)",
|
|
" local_head=$(git rev-parse HEAD 2>/dev/null || true)",
|
|
" status_short=$(git status --short 2>/dev/null || true)",
|
|
" if [ -z \"$status_short\" ]; then workspace_clean=true; else workspace_clean=false; fi",
|
|
" git fetch origin \"$source_branch\" >/dev/null 2>&1 || true",
|
|
" remote_branch_commit=$(git rev-parse \"refs/remotes/origin/$source_branch\" 2>/dev/null || true)",
|
|
" if [ -n \"$remote_branch_commit\" ]; then remote_branch_exists=true; fi",
|
|
"else",
|
|
" actual_workspace=$(pwd)",
|
|
"fi",
|
|
"export expected_workspace source_branch workspace_exists workspace_clean local_head branch remote_url remote_branch_exists remote_branch_commit status_short actual_workspace",
|
|
"node <<'NODE'",
|
|
"function nullable(value) { return value && value !== 'null' ? value : null; }",
|
|
"function booleanValue(value) { if (value === 'true') return true; if (value === 'false') return false; return null; }",
|
|
"const env = process.env;",
|
|
"console.log(JSON.stringify({",
|
|
" ok: env.workspace_exists === 'true',",
|
|
" expectedWorkspace: env.expected_workspace,",
|
|
" actualWorkspace: env.actual_workspace || null,",
|
|
" workspaceExists: env.workspace_exists === 'true',",
|
|
" workspaceClean: booleanValue(env.workspace_clean),",
|
|
" branch: nullable(env.branch),",
|
|
" remoteUrl: nullable(env.remote_url),",
|
|
" localHead: nullable(env.local_head),",
|
|
" remoteBranch: env.source_branch,",
|
|
" remoteBranchExists: env.remote_branch_exists === 'true',",
|
|
" remoteBranchCommit: nullable(env.remote_branch_commit),",
|
|
" statusShort: nullable(env.status_short),",
|
|
" valuesPrinted: false",
|
|
"}));",
|
|
"NODE",
|
|
].join("\n");
|
|
}
|
|
|
|
export function yamlLaneRuntimeStatusScript(spec: AgentRunLaneSpec, pipelineRun: string | null): string {
|
|
return [
|
|
"set +e",
|
|
`runtime_namespace=${shQuote(spec.runtime.namespace)}`,
|
|
`ci_namespace=${shQuote(spec.ci.namespace)}`,
|
|
`pipeline_name=${shQuote(spec.ci.pipeline)}`,
|
|
`pipeline_run=${pipelineRun === null ? "''" : shQuote(pipelineRun)}`,
|
|
`service_account=${shQuote(spec.ci.serviceAccountName)}`,
|
|
`argo_namespace=${shQuote(spec.gitops.argoNamespace)}`,
|
|
`argo_application=${shQuote(spec.gitops.argoApplication)}`,
|
|
`manager_deployment=${shQuote(spec.runtime.managerDeployment)}`,
|
|
`manager_service=${shQuote(spec.runtime.managerService)}`,
|
|
`database_secret=${shQuote(spec.database.secretRef.name)}`,
|
|
`database_key=${shQuote(spec.database.secretRef.key)}`,
|
|
`secrets_json=${shQuote(JSON.stringify(collectLaneSecretSources(spec).map((item) => item.targetRef)))}`,
|
|
"export runtime_namespace ci_namespace pipeline_name pipeline_run service_account argo_namespace argo_application manager_deployment manager_service database_secret database_key secrets_json",
|
|
"tmp_dir=$(mktemp -d)",
|
|
"trap 'rm -rf \"$tmp_dir\"' EXIT",
|
|
"kubectl get ns \"$runtime_namespace\" -o json > \"$tmp_dir/runtime-ns.json\" 2>/dev/null",
|
|
"runtime_ns_exit=$?",
|
|
"kubectl get ns \"$ci_namespace\" -o json > \"$tmp_dir/ci-ns.json\" 2>/dev/null",
|
|
"ci_ns_exit=$?",
|
|
"kubectl -n \"$ci_namespace\" get pipeline \"$pipeline_name\" -o json > \"$tmp_dir/pipeline.json\" 2>/dev/null",
|
|
"pipeline_exit=$?",
|
|
"kubectl -n \"$ci_namespace\" get serviceaccount \"$service_account\" -o json > \"$tmp_dir/serviceaccount.json\" 2>/dev/null",
|
|
"sa_exit=$?",
|
|
"if [ -n \"$pipeline_run\" ]; then kubectl -n \"$ci_namespace\" get pipelinerun \"$pipeline_run\" -o json > \"$tmp_dir/pipelinerun.json\" 2>/dev/null; pr_exit=$?; else pr_exit=2; fi",
|
|
"kubectl -n \"$argo_namespace\" get application \"$argo_application\" -o json > \"$tmp_dir/argo.json\" 2>/dev/null",
|
|
"argo_exit=$?",
|
|
"kubectl -n \"$runtime_namespace\" get deploy \"$manager_deployment\" -o json > \"$tmp_dir/manager-deploy.json\" 2>/dev/null",
|
|
"manager_deploy_exit=$?",
|
|
"kubectl -n \"$runtime_namespace\" get svc \"$manager_service\" -o json > \"$tmp_dir/manager-svc.json\" 2>/dev/null",
|
|
"manager_svc_exit=$?",
|
|
"kubectl -n \"$runtime_namespace\" get secret \"$database_secret\" -o json > \"$tmp_dir/db-secret.json\" 2>/dev/null",
|
|
"db_secret_exit=$?",
|
|
"SECRET_REFS_JSON=\"$secrets_json\" NODE_TMP=\"$tmp_dir\" node <<'NODE' > \"$tmp_dir/secrets.json\"",
|
|
"const fs = require('node:fs');",
|
|
"const cp = require('node:child_process');",
|
|
"const refs = JSON.parse(process.env.SECRET_REFS_JSON || '[]');",
|
|
"const result = [];",
|
|
"for (const ref of refs) {",
|
|
" const out = cp.spawnSync('kubectl', ['-n', ref.namespace, 'get', 'secret', ref.name, '-o', 'json'], { encoding: 'utf8' });",
|
|
" let keyPresent = false;",
|
|
" if (out.status === 0) {",
|
|
" try { keyPresent = Object.prototype.hasOwnProperty.call((JSON.parse(out.stdout).data || {}), ref.key); } catch {}",
|
|
" }",
|
|
" result.push({ namespace: ref.namespace, name: ref.name, key: ref.key, present: out.status === 0, keyPresent, valuesPrinted: false });",
|
|
"}",
|
|
"console.log(JSON.stringify({ ready: result.every((item) => item.present && item.keyPresent), count: result.length, items: result, valuesPrinted: false }));",
|
|
"NODE",
|
|
"kubectl -n \"$runtime_namespace\" get deploy,sts,svc,secret -o name > \"$tmp_dir/runtime-names.txt\" 2>/dev/null",
|
|
"NODE_TMP=\"$tmp_dir\" RUNTIME_NS_EXIT=\"$runtime_ns_exit\" CI_NS_EXIT=\"$ci_ns_exit\" PIPELINE_EXIT=\"$pipeline_exit\" SERVICE_ACCOUNT_EXIT=\"$sa_exit\" PIPELINERUN_EXIT=\"$pr_exit\" ARGO_EXIT=\"$argo_exit\" MANAGER_DEPLOY_EXIT=\"$manager_deploy_exit\" MANAGER_SVC_EXIT=\"$manager_svc_exit\" DB_SECRET_EXIT=\"$db_secret_exit\" node <<'NODE'",
|
|
"const fs = require('node:fs');",
|
|
"const path = require('node:path');",
|
|
"const dir = process.env.NODE_TMP;",
|
|
"function readJson(name) { try { return JSON.parse(fs.readFileSync(path.join(dir, name), 'utf8')); } catch { return null; } }",
|
|
"function exists(exitName) { return process.env[exitName] === '0'; }",
|
|
"function condition(obj) { return obj?.status?.conditions?.[0] || null; }",
|
|
"function envValue(deploy, name) { return deploy?.spec?.template?.spec?.containers?.[0]?.env?.find((entry) => entry.name === name)?.value || null; }",
|
|
"function pipelineRunParam(obj, name) { return (obj?.spec?.params || []).find((entry) => entry.name === name)?.value || null; }",
|
|
"const pipelineRun = readJson('pipelinerun.json');",
|
|
"const argo = readJson('argo.json');",
|
|
"const managerDeploy = readJson('manager-deploy.json');",
|
|
"const managerSvc = readJson('manager-svc.json');",
|
|
"const dbSecret = readJson('db-secret.json');",
|
|
"const secrets = readJson('secrets.json') || { ready: true, count: 0, items: [], valuesPrinted: false };",
|
|
"let names = ''; try { names = fs.readFileSync(path.join(dir, 'runtime-names.txt'), 'utf8'); } catch {}",
|
|
"const c = condition(pipelineRun);",
|
|
"console.log(JSON.stringify({",
|
|
" ok: exists('RUNTIME_NS_EXIT') && exists('CI_NS_EXIT'),",
|
|
" runtimeNamespaceExists: exists('RUNTIME_NS_EXIT'),",
|
|
" ciNamespaceExists: exists('CI_NS_EXIT'),",
|
|
" serviceAccountExists: exists('SERVICE_ACCOUNT_EXIT'),",
|
|
" pipeline: { exists: exists('PIPELINE_EXIT'), name: process.env.pipeline_name },",
|
|
" pipelineRun: { exists: exists('PIPELINERUN_EXIT'), name: process.env.pipeline_run || null, status: c?.status || null, reason: c?.reason || null, sourceCommit: pipelineRunParam(pipelineRun, 'revision'), startTime: pipelineRun?.status?.startTime || null, completionTime: pipelineRun?.status?.completionTime || null },",
|
|
" argo: { exists: exists('ARGO_EXIT'), namespace: process.env.argo_namespace, application: process.env.argo_application, revision: argo?.status?.sync?.revision || null, syncStatus: argo?.status?.sync?.status || null, healthStatus: argo?.status?.health?.status || null },",
|
|
" manager: { deploymentExists: exists('MANAGER_DEPLOY_EXIT'), serviceExists: exists('MANAGER_SVC_EXIT'), deployment: process.env.manager_deployment, service: process.env.manager_service, image: managerDeploy?.spec?.template?.spec?.containers?.[0]?.image || null, sourceCommit: envValue(managerDeploy, 'AGENTRUN_SOURCE_COMMIT'), servicePorts: Array.isArray(managerSvc?.spec?.ports) ? managerSvc.spec.ports.map((port) => ({ name: port.name || null, port: port.port || null, targetPort: port.targetPort || null })) : [] },",
|
|
" database: { secretPresent: exists('DB_SECRET_EXIT'), secretName: process.env.database_secret, key: process.env.database_key, keyPresent: Boolean(dbSecret?.data && Object.prototype.hasOwnProperty.call(dbSecret.data, process.env.database_key || '')) , valuesPrinted: false },",
|
|
" secrets,",
|
|
" localPostgres: { absent: !/postgres/i.test(names), matchingObjects: names.split(/\\r?\\n/).filter((line) => /postgres/i.test(line)).slice(0, 20) },",
|
|
" valuesPrinted: false",
|
|
"}));",
|
|
"NODE",
|
|
].join("\n");
|
|
}
|
|
|
|
export function yamlLaneSourceBootstrapProbeScript(spec: AgentRunLaneSpec): string {
|
|
return [
|
|
"set +e",
|
|
`workspace=${shQuote(spec.source.workspace)}`,
|
|
`remote=${shQuote(spec.source.remote)}`,
|
|
`branch=${shQuote(spec.source.branch)}`,
|
|
"workspace_exists=false",
|
|
"remote_branch_exists=false",
|
|
"source_commit=null",
|
|
"if [ -d \"$workspace/.git\" ]; then",
|
|
" workspace_exists=true",
|
|
" cd \"$workspace\"",
|
|
" remote_branch=$(git ls-remote --heads \"$remote\" \"$branch\" 2>/dev/null | awk '{print $1}' | head -n 1)",
|
|
" if [ -n \"$remote_branch\" ]; then remote_branch_exists=true; source_commit=\"$remote_branch\"; else source_commit=$(git rev-parse HEAD 2>/dev/null || printf null); fi",
|
|
"else",
|
|
" remote_branch=$(git ls-remote --heads \"$remote\" \"$branch\" 2>/dev/null | awk '{print $1}' | head -n 1)",
|
|
" if [ -n \"$remote_branch\" ]; then remote_branch_exists=true; source_commit=\"$remote_branch\"; fi",
|
|
"fi",
|
|
"export workspace_exists remote_branch_exists source_commit",
|
|
"node <<'NODE'",
|
|
"console.log(JSON.stringify({ ok: true, workspaceExists: process.env.workspace_exists === 'true', remoteBranchExists: process.env.remote_branch_exists === 'true', sourceCommit: process.env.source_commit === 'null' ? null : process.env.source_commit, valuesPrinted: false }));",
|
|
"NODE",
|
|
].join("\n");
|
|
}
|
|
|
|
export function yamlLaneSourceBootstrapSubmitScript(spec: AgentRunLaneSpec): string {
|
|
const bootstrap = spec.source.bootstrapFromBranch ?? spec.source.branch;
|
|
const stateDir = `/tmp/unidesk-agentrun-source-${spec.nodeId}-${spec.lane}`;
|
|
return [
|
|
"set -eu",
|
|
`workspace=${shQuote(spec.source.workspace)}`,
|
|
`remote=${shQuote(spec.source.remote)}`,
|
|
`branch=${shQuote(spec.source.branch)}`,
|
|
`bootstrap_branch=${shQuote(bootstrap)}`,
|
|
`state_dir=${shQuote(stateDir)}`,
|
|
"mkdir -p \"$state_dir\" \"$(dirname \"$workspace\")\"",
|
|
"case \"$remote\" in",
|
|
" git@*:*)",
|
|
" mkdir -p \"$HOME/.ssh\"",
|
|
" chmod 700 \"$HOME/.ssh\" 2>/dev/null || true",
|
|
" export GIT_SSH_COMMAND=${GIT_SSH_COMMAND:-ssh -o StrictHostKeyChecking=accept-new}",
|
|
" ;;",
|
|
" ssh://*)",
|
|
" mkdir -p \"$HOME/.ssh\"",
|
|
" chmod 700 \"$HOME/.ssh\" 2>/dev/null || true",
|
|
" export GIT_SSH_COMMAND=${GIT_SSH_COMMAND:-ssh -o StrictHostKeyChecking=accept-new}",
|
|
" ;;",
|
|
"esac",
|
|
"job_id=\"source-bootstrap-$(date +%s)-$$\"",
|
|
"status_file=\"$state_dir/$job_id.json\"",
|
|
"stdout_file=\"$state_dir/$job_id.stdout.log\"",
|
|
"stderr_file=\"$state_dir/$job_id.stderr.log\"",
|
|
"cat > \"$status_file\" <<EOF",
|
|
"{\"ok\":false,\"status\":\"running\",\"jobId\":\"$job_id\",\"workspace\":\"$workspace\",\"branch\":\"$branch\",\"valuesPrinted\":false}",
|
|
"EOF",
|
|
"(",
|
|
" set -eu",
|
|
" write_failed_status() { code=$?; if [ \"$code\" -ne 0 ]; then tail_text=$(tail -n 60 \"$stderr_file\" 2>/dev/null | sed 's/\"/\\\\\"/g' | tr '\\n' ' ' | cut -c1-3000); CODE=\"$code\" ERROR_TAIL=\"$tail_text\" JOB_ID=\"$job_id\" WORKSPACE=\"$workspace\" BRANCH=\"$branch\" node <<'NODE' > \"$status_file\"",
|
|
"const code = Number(process.env.CODE || 1);",
|
|
"console.log(JSON.stringify({ ok: false, status: 'failed', exitCode: code, jobId: process.env.JOB_ID, workspace: process.env.WORKSPACE, branch: process.env.BRANCH, errorTail: process.env.ERROR_TAIL || null, valuesPrinted: false }));",
|
|
"NODE",
|
|
" fi; exit \"$code\"; }",
|
|
" trap write_failed_status EXIT",
|
|
" if [ -d \"$workspace/.git\" ] && git -C \"$workspace\" rev-parse --git-dir >/dev/null 2>&1; then",
|
|
" :",
|
|
" else",
|
|
" rm -rf \"$workspace\"",
|
|
" git clone --no-checkout \"$remote\" \"$workspace\"",
|
|
" fi",
|
|
" cd \"$workspace\"",
|
|
" git remote set-url origin \"$remote\" || git remote add origin \"$remote\"",
|
|
" git fetch origin \"$bootstrap_branch\" \"$branch\" || git fetch origin \"$bootstrap_branch\"",
|
|
" if git rev-parse --verify \"refs/remotes/origin/$branch^{commit}\" >/dev/null 2>&1; then",
|
|
" git checkout -B \"$branch\" \"refs/remotes/origin/$branch\"",
|
|
" else",
|
|
" git checkout -B \"$branch\" \"refs/remotes/origin/$bootstrap_branch\"",
|
|
" fi",
|
|
" if [ -f deploy/deploy.json ]; then rm deploy/deploy.json; fi",
|
|
" git add -A deploy/deploy.json 2>/dev/null || true",
|
|
" if ! git diff --quiet --cached -- deploy/deploy.json 2>/dev/null; then",
|
|
" git -c user.email=agentrun@unidesk.local -c user.name='UniDesk AgentRun Ops' commit -m 'chore: remove service deploy json truth'",
|
|
" fi",
|
|
" git push -u origin \"$branch\"",
|
|
" source_commit=$(git rev-parse HEAD)",
|
|
" status_short=$(git status --short)",
|
|
" SOURCE_COMMIT=\"$source_commit\" STATUS_SHORT=\"$status_short\" JOB_ID=\"$job_id\" WORKSPACE=\"$workspace\" BRANCH=\"$branch\" node <<'NODE' > \"$status_file\"",
|
|
"console.log(JSON.stringify({ ok: process.env.STATUS_SHORT === '', status: 'succeeded', jobId: process.env.JOB_ID, workspace: process.env.WORKSPACE, branch: process.env.BRANCH, sourceCommit: process.env.SOURCE_COMMIT, workspaceClean: process.env.STATUS_SHORT === '', statusShort: process.env.STATUS_SHORT || null, removedServiceDeployJson: true, valuesPrinted: false }));",
|
|
"NODE",
|
|
" trap - EXIT",
|
|
") >\"$stdout_file\" 2>\"$stderr_file\" &",
|
|
"pid=$!",
|
|
"JOB_PID=\"$pid\" JOB_ID=\"$job_id\" STATUS_FILE=\"$status_file\" STDOUT_FILE=\"$stdout_file\" STDERR_FILE=\"$stderr_file\" node <<'NODE'",
|
|
"console.log(JSON.stringify({ ok: true, status: 'submitted', jobId: process.env.JOB_ID, pid: Number(process.env.JOB_PID), statusFile: process.env.STATUS_FILE, stdoutFile: process.env.STDOUT_FILE, stderrFile: process.env.STDERR_FILE, valuesPrinted: false }));",
|
|
"NODE",
|
|
].join("\n");
|
|
}
|
|
|
|
export async function waitForYamlLaneSourceBootstrap(config: UniDeskConfig, spec: AgentRunLaneSpec, jobId: string | null): Promise<Record<string, unknown> & { ok: boolean; payload: Record<string, unknown> }> {
|
|
if (jobId === null) return { ok: false, payload: { ok: false, degradedReason: "source-bootstrap-job-id-missing", valuesPrinted: false } };
|
|
const startedAt = Date.now();
|
|
const timeoutMs = spec.source.bootstrapTimeoutSeconds * 1000;
|
|
const pollMs = spec.source.bootstrapPollSeconds * 1000;
|
|
let lastPayload: Record<string, unknown> = {};
|
|
let polls = 0;
|
|
while (Date.now() - startedAt < timeoutMs) {
|
|
polls += 1;
|
|
const probe = await capture(config, spec.nodeRoute, ["sh", "--", yamlLaneSourceBootstrapStatusScript(spec, jobId)]);
|
|
const payload = captureJsonPayload(probe);
|
|
lastPayload = payload;
|
|
progressEvent("agentrun.yaml-lane.source-bootstrap.progress", {
|
|
node: spec.nodeId,
|
|
lane: spec.lane,
|
|
jobId,
|
|
polls,
|
|
status: stringOrNull(payload.status) ?? "unknown",
|
|
sourceCommit: stringOrNull(payload.sourceCommit),
|
|
elapsedMs: Date.now() - startedAt,
|
|
});
|
|
if (payload.status === "succeeded" && stringOrNull(payload.sourceCommit) !== null) return { ok: payload.ok === true, payload, polls, elapsedMs: Date.now() - startedAt };
|
|
if (payload.status === "failed") return { ok: false, payload, polls, elapsedMs: Date.now() - startedAt };
|
|
await sleep(pollMs);
|
|
}
|
|
return { ok: false, payload: { ...lastPayload, ok: false, status: "timeout", degradedReason: "source-bootstrap-timeout", valuesPrinted: false }, polls, elapsedMs: Date.now() - startedAt };
|
|
}
|
|
|
|
export function yamlLaneSourceBootstrapStatusScript(spec: AgentRunLaneSpec, jobId: string): string {
|
|
const stateDir = `/tmp/unidesk-agentrun-source-${spec.nodeId}-${spec.lane}`;
|
|
return [
|
|
"set +e",
|
|
`status_file=${shQuote(`${stateDir}/${jobId}.json`)}`,
|
|
"if [ -f \"$status_file\" ]; then cat \"$status_file\"; else printf '{\"ok\":false,\"status\":\"missing\",\"valuesPrinted\":false}\\n'; fi",
|
|
].join("\n");
|
|
}
|
|
|
|
export function yamlLaneSourceRestoreScript(spec: AgentRunLaneSpec): string {
|
|
return [
|
|
"set +e",
|
|
`workspace=${shQuote(spec.source.workspace)}`,
|
|
`branch=${shQuote(spec.source.branch)}`,
|
|
"tmp_dir=$(mktemp -d)",
|
|
"trap 'rm -rf \"$tmp_dir\"' EXIT",
|
|
"workspace_exists=false",
|
|
"if git -C \"$workspace\" rev-parse --git-dir >/dev/null 2>&1; then workspace_exists=true; fi",
|
|
"if [ \"$workspace_exists\" != true ]; then",
|
|
" WORKSPACE=\"$workspace\" BRANCH=\"$branch\" node <<'NODE'",
|
|
"console.log(JSON.stringify({ ok: false, status: 'skipped', failureKind: 'source-worktree-missing', workspace: process.env.WORKSPACE, branch: process.env.BRANCH, valuesPrinted: false }));",
|
|
"NODE",
|
|
" exit 0",
|
|
"fi",
|
|
"before_branch=$(git -C \"$workspace\" rev-parse --abbrev-ref HEAD 2>/dev/null || true)",
|
|
"before_head=$(git -C \"$workspace\" rev-parse HEAD 2>/dev/null || true)",
|
|
"status_short=$(git -C \"$workspace\" status --short 2>/dev/null || true)",
|
|
"if [ -n \"$status_short\" ]; then",
|
|
" WORKSPACE=\"$workspace\" BRANCH=\"$branch\" BEFORE_BRANCH=\"$before_branch\" BEFORE_HEAD=\"$before_head\" STATUS_SHORT=\"$status_short\" node <<'NODE'",
|
|
"console.log(JSON.stringify({ ok: false, status: 'skipped', failureKind: 'source-worktree-dirty', workspace: process.env.WORKSPACE, branch: process.env.BRANCH, before: { branch: process.env.BEFORE_BRANCH || null, head: process.env.BEFORE_HEAD || null, detached: process.env.BEFORE_BRANCH === 'HEAD' }, statusShort: process.env.STATUS_SHORT || null, valuesPrinted: false }));",
|
|
"NODE",
|
|
" exit 0",
|
|
"fi",
|
|
"git -C \"$workspace\" fetch origin \"$branch\" > \"$tmp_dir/fetch.out\" 2> \"$tmp_dir/fetch.err\"",
|
|
"fetch_exit=$?",
|
|
"remote_branch_commit=$(git -C \"$workspace\" rev-parse \"refs/remotes/origin/$branch\" 2>/dev/null || true)",
|
|
"checkout_exit=1",
|
|
"if [ \"$fetch_exit\" -eq 0 ] && [ -n \"$remote_branch_commit\" ]; then",
|
|
" git -C \"$workspace\" checkout -B \"$branch\" \"refs/remotes/origin/$branch\" > \"$tmp_dir/checkout.out\" 2> \"$tmp_dir/checkout.err\"",
|
|
" checkout_exit=$?",
|
|
"else",
|
|
" : > \"$tmp_dir/checkout.out\"",
|
|
" : > \"$tmp_dir/checkout.err\"",
|
|
"fi",
|
|
"after_branch=$(git -C \"$workspace\" rev-parse --abbrev-ref HEAD 2>/dev/null || true)",
|
|
"after_head=$(git -C \"$workspace\" rev-parse HEAD 2>/dev/null || true)",
|
|
"after_status_short=$(git -C \"$workspace\" status --short 2>/dev/null || true)",
|
|
"fetch_err=$(tail -n 20 \"$tmp_dir/fetch.err\" 2>/dev/null | tr '\\n' ' ' | cut -c1-1200)",
|
|
"checkout_err=$(tail -n 20 \"$tmp_dir/checkout.err\" 2>/dev/null | tr '\\n' ' ' | cut -c1-1200)",
|
|
"WORKSPACE=\"$workspace\" BRANCH=\"$branch\" BEFORE_BRANCH=\"$before_branch\" BEFORE_HEAD=\"$before_head\" FETCH_EXIT=\"$fetch_exit\" CHECKOUT_EXIT=\"$checkout_exit\" REMOTE_BRANCH_COMMIT=\"$remote_branch_commit\" AFTER_BRANCH=\"$after_branch\" AFTER_HEAD=\"$after_head\" AFTER_STATUS_SHORT=\"$after_status_short\" FETCH_ERR=\"$fetch_err\" CHECKOUT_ERR=\"$checkout_err\" node <<'NODE'",
|
|
"const fetchExit = Number(process.env.FETCH_EXIT || '1');",
|
|
"const checkoutExit = Number(process.env.CHECKOUT_EXIT || '1');",
|
|
"const afterClean = !process.env.AFTER_STATUS_SHORT;",
|
|
"const ok = fetchExit === 0 && checkoutExit === 0 && afterClean && process.env.AFTER_BRANCH === process.env.BRANCH;",
|
|
"let failureKind = null;",
|
|
"if (fetchExit !== 0) failureKind = 'source-branch-fetch-failed';",
|
|
"else if (!process.env.REMOTE_BRANCH_COMMIT) failureKind = 'source-branch-missing';",
|
|
"else if (checkoutExit !== 0) failureKind = 'source-branch-checkout-failed';",
|
|
"else if (!afterClean) failureKind = 'source-worktree-dirty-after-restore';",
|
|
"else if (process.env.AFTER_BRANCH !== process.env.BRANCH) failureKind = 'source-worktree-branch-mismatch';",
|
|
"console.log(JSON.stringify({",
|
|
" ok,",
|
|
" status: ok ? 'restored' : 'failed',",
|
|
" failureKind,",
|
|
" workspace: process.env.WORKSPACE,",
|
|
" branch: process.env.BRANCH,",
|
|
" before: { branch: process.env.BEFORE_BRANCH || null, head: process.env.BEFORE_HEAD || null, detached: process.env.BEFORE_BRANCH === 'HEAD' },",
|
|
" after: { branch: process.env.AFTER_BRANCH || null, head: process.env.AFTER_HEAD || null, clean: afterClean },",
|
|
" remoteBranchCommit: process.env.REMOTE_BRANCH_COMMIT || null,",
|
|
" fetch: { exitCode: fetchExit, stderrTail: process.env.FETCH_ERR || null },",
|
|
" checkout: { exitCode: checkoutExit, stderrTail: process.env.CHECKOUT_ERR || null },",
|
|
" valuesPrinted: false",
|
|
"}));",
|
|
"NODE",
|
|
].join("\n");
|
|
}
|
|
|
|
export function yamlLaneBuildImageSubmitScript(spec: AgentRunLaneSpec, sourceCommit: string): string {
|
|
const build = spec.deployment.manager.imageBuild;
|
|
const noProxy = build.noProxy.join(",");
|
|
const imageRepository = `${spec.ci.registryPrefix}/${build.repository}`;
|
|
const stateDir = `/tmp/unidesk-agentrun-build-${spec.nodeId}-${spec.lane}`;
|
|
const buildArgs = Object.entries(build.buildArgs)
|
|
.sort(([left], [right]) => left.localeCompare(right))
|
|
.map(([key, value]) => `${key}=${value}`);
|
|
const script = [
|
|
"set -eu",
|
|
`workspace=${shQuote(spec.source.workspace)}`,
|
|
`source_commit=${shQuote(sourceCommit)}`,
|
|
`state_dir=${shQuote(stateDir)}`,
|
|
`containerfile=${shQuote(build.containerfile)}`,
|
|
`context_dir=${shQuote(build.context)}`,
|
|
`image_repository=${shQuote(imageRepository)}`,
|
|
`network=${shQuote(build.network)}`,
|
|
`http_proxy_value=${build.httpProxy === null ? "''" : shQuote(build.httpProxy)}`,
|
|
`https_proxy_value=${build.httpsProxy === null ? "''" : shQuote(build.httpsProxy)}`,
|
|
`no_proxy_value=${shQuote(noProxy)}`,
|
|
`env_identity_files=${shQuote(JSON.stringify(build.envIdentityFiles))}`,
|
|
`build_args_json=${shQuote(JSON.stringify(buildArgs))}`,
|
|
"mkdir -p \"$state_dir\"",
|
|
"cd \"$workspace\"",
|
|
"git checkout \"$source_commit\"",
|
|
"env_identity=$(ENV_IDENTITY_FILES=\"$env_identity_files\" BUILD_ARGS_JSON=\"$build_args_json\" node <<'NODE'",
|
|
"const { createHash } = require('node:crypto');",
|
|
"const { readFileSync, existsSync, lstatSync, readdirSync } = require('node:fs');",
|
|
"const { join } = require('node:path');",
|
|
"const files = JSON.parse(process.env.ENV_IDENTITY_FILES || '[]');",
|
|
"const buildArgs = JSON.parse(process.env.BUILD_ARGS_JSON || '[]');",
|
|
"const hash = createHash('sha256');",
|
|
"const skipDirNames = new Set(['.git', '.worktree', '.state', 'node_modules', 'coverage', 'tmp', '.tmp']);",
|
|
"function collectIdentityFiles(input) {",
|
|
" if (!existsSync(input)) return [{ path: input, missing: true }];",
|
|
" const stat = lstatSync(input);",
|
|
" if (stat.isFile()) return [{ path: input, missing: false }];",
|
|
" if (!stat.isDirectory()) return [{ path: input, missing: true }];",
|
|
" const out = [];",
|
|
" const stack = [input];",
|
|
" while (stack.length > 0) {",
|
|
" const dir = stack.pop();",
|
|
" const entries = readdirSync(dir, { withFileTypes: true }).sort((left, right) => left.name.localeCompare(right.name));",
|
|
" for (const entry of entries) {",
|
|
" if (entry.isDirectory() && skipDirNames.has(entry.name)) continue;",
|
|
" const child = join(dir, entry.name);",
|
|
" if (entry.isDirectory()) stack.push(child);",
|
|
" else if (entry.isFile()) out.push({ path: child, missing: false });",
|
|
" }",
|
|
" }",
|
|
" return out.sort((left, right) => left.path.localeCompare(right.path));",
|
|
"}",
|
|
"for (const item of buildArgs) { hash.update('build-arg'); hash.update('\\0'); hash.update(item); hash.update('\\0'); }",
|
|
"for (const file of files) {",
|
|
" for (const entry of collectIdentityFiles(file)) {",
|
|
" hash.update(entry.path); hash.update('\\0');",
|
|
" if (!entry.missing) hash.update(readFileSync(entry.path));",
|
|
" hash.update('\\0');",
|
|
" }",
|
|
"}",
|
|
"process.stdout.write(hash.digest('hex').slice(0, 24));",
|
|
"NODE",
|
|
")",
|
|
"source_short=$(printf '%s' \"$source_commit\" | cut -c1-12)",
|
|
"job_id=\"$source_short-$env_identity\"",
|
|
"status_file=\"$state_dir/$job_id.json\"",
|
|
"stdout_file=\"$state_dir/$job_id.stdout.log\"",
|
|
"stderr_file=\"$state_dir/$job_id.stderr.log\"",
|
|
"cat > \"$status_file\" <<EOF",
|
|
"{\"ok\":false,\"status\":\"running\",\"jobId\":\"$job_id\",\"sourceCommit\":\"$source_commit\",\"envIdentity\":\"$env_identity\",\"image\":\"$image_repository:$env_identity\",\"valuesPrinted\":false}",
|
|
"EOF",
|
|
"(",
|
|
" set -eu",
|
|
" write_failed_status() { code=$?; if [ \"$code\" -ne 0 ]; then tail_text=$(tail -n 40 \"$stderr_file\" 2>/dev/null | sed 's/\"/\\\\\"/g' | tr '\\n' ' ' | cut -c1-2000); CODE=\"$code\" ERROR_TAIL=\"$tail_text\" JOB_ID=\"$job_id\" SOURCE_COMMIT=\"$source_commit\" ENV_IDENTITY=\"$env_identity\" IMAGE_REPOSITORY=\"$image_repository\" node <<'NODE' > \"$status_file\"",
|
|
"const code = Number(process.env.CODE || 1);",
|
|
"console.log(JSON.stringify({ ok: false, status: 'failed', exitCode: code, jobId: process.env.JOB_ID, sourceCommit: process.env.SOURCE_COMMIT, envIdentity: process.env.ENV_IDENTITY, image: `${process.env.IMAGE_REPOSITORY}:${process.env.ENV_IDENTITY}`, errorTail: process.env.ERROR_TAIL || null, valuesPrinted: false }));",
|
|
"NODE",
|
|
" fi; exit \"$code\"; }",
|
|
" trap write_failed_status EXIT",
|
|
" cd \"$workspace\"",
|
|
" image=\"$image_repository:$env_identity\"",
|
|
" args=\"--network $network\"",
|
|
" if [ -n \"$http_proxy_value\" ]; then args=\"$args --build-arg HTTP_PROXY=$http_proxy_value --build-arg http_proxy=$http_proxy_value\"; fi",
|
|
" if [ -n \"$https_proxy_value\" ]; then args=\"$args --build-arg HTTPS_PROXY=$https_proxy_value --build-arg https_proxy=$https_proxy_value\"; fi",
|
|
" if [ -n \"$no_proxy_value\" ]; then args=\"$args --build-arg NO_PROXY=$no_proxy_value --build-arg no_proxy=$no_proxy_value\"; fi",
|
|
" build_arg_values=$(BUILD_ARGS_JSON=\"$build_args_json\" node <<'NODE'",
|
|
"const values = JSON.parse(process.env.BUILD_ARGS_JSON || '[]');",
|
|
"for (const value of values) console.log(value);",
|
|
"NODE",
|
|
" )",
|
|
" while IFS= read -r build_arg_value; do [ -n \"$build_arg_value\" ] && args=\"$args --build-arg $build_arg_value\"; done <<EOF",
|
|
"$build_arg_values",
|
|
"EOF",
|
|
" if docker image inspect \"$image\" >/dev/null 2>&1; then build_status=reused; else docker build $args -f \"$containerfile\" -t \"$image\" \"$context_dir\"; build_status=built; fi",
|
|
" manifest_accept='application/vnd.oci.image.index.v1+json, application/vnd.oci.image.manifest.v1+json, application/vnd.docker.distribution.manifest.v2+json'",
|
|
" fetch_manifest_digest() {",
|
|
" ref=\"$1\"",
|
|
" attempt=1",
|
|
" max_attempts=30",
|
|
" while [ \"$attempt\" -le \"$max_attempts\" ]; do",
|
|
" headers=$(mktemp)",
|
|
" if curl -fsSI -H \"Accept: $manifest_accept\" \"http://127.0.0.1:5000/v2/${image_repository#127.0.0.1:5000/}/manifests/$ref\" >\"$headers\"; then",
|
|
" found_digest=$(awk -F': ' 'tolower($1)==\"docker-content-digest\"{print $2}' \"$headers\" | tr -d '\\r' | head -n 1)",
|
|
" rm -f \"$headers\"",
|
|
" if [ -n \"$found_digest\" ]; then printf '%s' \"$found_digest\"; return 0; fi",
|
|
" else",
|
|
" rm -f \"$headers\"",
|
|
" fi",
|
|
" printf '{\"event\":\"agentrun-registry-manifest-probe\",\"status\":\"retrying\",\"ref\":\"%s\",\"attempt\":\"%s/%s\"}\\n' \"$ref\" \"$attempt\" \"$max_attempts\" >&2",
|
|
" sleep 1",
|
|
" attempt=$((attempt + 1))",
|
|
" done",
|
|
" printf '{\"event\":\"agentrun-registry-manifest-probe\",\"status\":\"exhausted\",\"ref\":\"%s\",\"attempt\":\"%s/%s\"}\\n' \"$ref\" \"$max_attempts\" \"$max_attempts\" >&2",
|
|
" return 1",
|
|
" }",
|
|
" docker push \"$image\"",
|
|
" digest=$(fetch_manifest_digest \"$env_identity\" || true)",
|
|
" if [ -n \"$digest\" ]; then verified_digest=$(fetch_manifest_digest \"$digest\" || true); if [ \"$verified_digest\" != \"$digest\" ]; then digest=; fi; fi",
|
|
" STATUS=\"$build_status\" DIGEST=\"$digest\" JOB_ID=\"$job_id\" SOURCE_COMMIT=\"$source_commit\" ENV_IDENTITY=\"$env_identity\" IMAGE_REPOSITORY=\"$image_repository\" node <<'NODE' > \"$status_file\"",
|
|
"const digest = process.env.DIGEST || null;",
|
|
"console.log(JSON.stringify({ ok: Boolean(digest), status: process.env.STATUS || 'built', jobId: process.env.JOB_ID, sourceCommit: process.env.SOURCE_COMMIT, envIdentity: process.env.ENV_IDENTITY, image: `${process.env.IMAGE_REPOSITORY}:${process.env.ENV_IDENTITY}`, digest, repositoryDigest: digest ? `${process.env.IMAGE_REPOSITORY}@${digest}` : null, valuesPrinted: false }));",
|
|
"NODE",
|
|
" trap - EXIT",
|
|
") >\"$stdout_file\" 2>\"$stderr_file\" &",
|
|
"pid=$!",
|
|
"JOB_PID=\"$pid\" JOB_ID=\"$job_id\" STATUS_FILE=\"$status_file\" STDOUT_FILE=\"$stdout_file\" STDERR_FILE=\"$stderr_file\" node <<'NODE'",
|
|
"console.log(JSON.stringify({ ok: true, status: 'submitted', jobId: process.env.JOB_ID, pid: Number(process.env.JOB_PID), statusFile: process.env.STATUS_FILE, stdoutFile: process.env.STDOUT_FILE, stderrFile: process.env.STDERR_FILE, valuesPrinted: false }));",
|
|
"NODE",
|
|
].join("\n");
|
|
return script;
|
|
}
|
|
|
|
export async function waitForYamlLaneBuildImage(config: UniDeskConfig, spec: AgentRunLaneSpec, sourceCommit: string, jobId: string | null): Promise<Record<string, unknown> & { ok: boolean; payload: Record<string, unknown> }> {
|
|
if (jobId === null) return { ok: false, payload: { ok: false, degradedReason: "build-job-id-missing", valuesPrinted: false } };
|
|
const startedAt = Date.now();
|
|
const timeoutMs = Math.max(60, spec.deployment.manager.imageBuild.timeoutSeconds) * 1000;
|
|
let lastPayload: Record<string, unknown> = {};
|
|
let polls = 0;
|
|
while (Date.now() - startedAt < timeoutMs) {
|
|
polls += 1;
|
|
const probe = await capture(config, spec.nodeRoute, ["sh", "--", yamlLaneBuildImageStatusScript(spec, jobId)]);
|
|
const payload = captureJsonPayload(probe);
|
|
lastPayload = payload;
|
|
progressEvent("agentrun.yaml-lane.image-build.progress", {
|
|
node: spec.nodeId,
|
|
lane: spec.lane,
|
|
sourceCommit,
|
|
jobId,
|
|
polls,
|
|
status: stringOrNull(payload.status) ?? "unknown",
|
|
elapsedMs: Date.now() - startedAt,
|
|
});
|
|
if (payload.ok === true && stringOrNull(payload.digest) !== null) return { ok: true, payload, polls, elapsedMs: Date.now() - startedAt };
|
|
if (payload.status === "failed") return { ok: false, payload, polls, elapsedMs: Date.now() - startedAt };
|
|
await sleep(spec.deployment.manager.imageBuild.pollSeconds * 1000);
|
|
}
|
|
return { ok: false, payload: { ...lastPayload, ok: false, status: "timeout", degradedReason: "image-build-timeout", valuesPrinted: false }, polls, elapsedMs: Date.now() - startedAt };
|
|
}
|
|
|
|
export function yamlLaneBuildImageStatusScript(spec: AgentRunLaneSpec, jobId: string): string {
|
|
const stateDir = `/tmp/unidesk-agentrun-build-${spec.nodeId}-${spec.lane}`;
|
|
return [
|
|
"set +e",
|
|
`status_file=${shQuote(`${stateDir}/${jobId}.json`)}`,
|
|
`stdout_file=${shQuote(`${stateDir}/${jobId}.stdout.log`)}`,
|
|
`stderr_file=${shQuote(`${stateDir}/${jobId}.stderr.log`)}`,
|
|
"if [ -f \"$status_file\" ]; then cat \"$status_file\"; else printf '{\"ok\":false,\"status\":\"missing\",\"valuesPrinted\":false}\\n'; fi",
|
|
"if [ -f \"$stderr_file\" ] && tail -n 20 \"$stderr_file\" | grep -Eq 'ERROR|error|failed|denied'; then :; fi",
|
|
].join("\n");
|
|
}
|
|
|
|
export async function runYamlLaneGitopsPublishJob(config: UniDeskConfig, spec: AgentRunLaneSpec, sourceCommit: string, files: readonly { path: string; content: string }[]): Promise<Record<string, unknown> & { ok: boolean; payload: Record<string, unknown> }> {
|
|
const jobName = `gitops-publish-${spec.nodeId.toLowerCase()}-${spec.lane}-${Date.now().toString(36)}`.slice(0, 63);
|
|
const manifest = yamlLaneGitopsPublishJobManifest(spec, files, jobName);
|
|
const created = await capture(config, spec.nodeKubeRoute, ["sh", "--", createYamlLaneJobScript(spec.gitMirror.namespace, jobName, manifest)]);
|
|
if (created.exitCode !== 0) {
|
|
return {
|
|
ok: false,
|
|
payload: { ok: false, status: "create-failed", jobName, degradedReason: "gitops-publish-job-create-failed", valuesPrinted: false },
|
|
jobName,
|
|
sourceCommit,
|
|
phase: "create-job",
|
|
capture: compactCapture(created, { full: true, stdoutTailChars: 5000, stderrTailChars: 5000 }),
|
|
valuesPrinted: false,
|
|
};
|
|
}
|
|
const startedAt = Date.now();
|
|
const timeoutMs = 300_000;
|
|
let polls = 0;
|
|
let lastPayload: Record<string, unknown> = {};
|
|
let lastProbe: SshCaptureResult | null = null;
|
|
while (Date.now() - startedAt < timeoutMs) {
|
|
polls += 1;
|
|
lastProbe = await capture(config, spec.nodeKubeRoute, ["sh", "--", yamlLaneJobProbeScript(spec.gitMirror.namespace, jobName)]);
|
|
const probePayload = captureJsonPayload(lastProbe);
|
|
const publishPayload = yamlLaneGitopsPublishPayloadFromProbe(probePayload);
|
|
if (Object.keys(publishPayload).length > 0) lastPayload = publishPayload;
|
|
progressEvent("agentrun.yaml-lane.gitops-publish.progress", {
|
|
node: spec.nodeId,
|
|
lane: spec.lane,
|
|
sourceCommit,
|
|
jobName,
|
|
polls,
|
|
status: stringOrNull(publishPayload.status) ?? (probePayload.succeeded === true ? "succeeded" : probePayload.failed === true ? "failed" : "running"),
|
|
gitopsCommit: stringOrNull(publishPayload.gitopsCommit),
|
|
elapsedMs: Date.now() - startedAt,
|
|
});
|
|
if (probePayload.succeeded === true) {
|
|
if (publishPayload.ok === true && stringOrNull(publishPayload.gitopsCommit) !== null) {
|
|
return { ok: true, payload: publishPayload, jobName, polls, elapsedMs: Date.now() - startedAt, probe: compactCapture(lastProbe), valuesPrinted: false };
|
|
}
|
|
return {
|
|
ok: false,
|
|
payload: { ...publishPayload, ok: false, status: stringOrNull(publishPayload.status) ?? "succeeded-without-result", degradedReason: "gitops-publish-result-missing", jobName, valuesPrinted: false },
|
|
jobName,
|
|
polls,
|
|
elapsedMs: Date.now() - startedAt,
|
|
probe: compactCapture(lastProbe, { full: true, stdoutTailChars: 5000, stderrTailChars: 5000 }),
|
|
valuesPrinted: false,
|
|
};
|
|
}
|
|
if (probePayload.failed === true) {
|
|
const payload = Object.keys(publishPayload).length > 0 ? publishPayload : { ok: false, status: "failed", degradedReason: "gitops-publish-job-failed", jobName, valuesPrinted: false };
|
|
return {
|
|
ok: false,
|
|
payload,
|
|
jobName,
|
|
polls,
|
|
elapsedMs: Date.now() - startedAt,
|
|
probe: compactCapture(lastProbe, { full: true, stdoutTailChars: 5000, stderrTailChars: 5000 }),
|
|
valuesPrinted: false,
|
|
};
|
|
}
|
|
await sleep(5_000);
|
|
}
|
|
return {
|
|
ok: false,
|
|
payload: { ...lastPayload, ok: false, status: "timeout", degradedReason: "gitops-publish-job-timeout", jobName, valuesPrinted: false },
|
|
jobName,
|
|
polls,
|
|
elapsedMs: Date.now() - startedAt,
|
|
probe: lastProbe === null ? null : compactCapture(lastProbe, { full: true, stdoutTailChars: 5000, stderrTailChars: 5000 }),
|
|
valuesPrinted: false,
|
|
};
|
|
}
|