Merge pull request #1298 from pikasTech/fix/1248-sentinel-tekton-argo
fix: sentinel publish 改走 Tekton PipelineRun
This commit is contained in:
Lyon
@@ -22,7 +22,8 @@ bun scripts/cli.ts agentrun control-plane status
|
||||
## P0 边界
|
||||
|
||||
- CI/CD、GitOps、rollout、PipelineRun、Argo、git-mirror 和 AgentRun 部署必须走受控 CLI；不要用裸 `kubectl`、`argo`、`tkn`、`curl` 当正式控制入口。
|
||||
- CI/CD、rollout、publish、image build 和部署链路禁止新引入 Docker 依赖；不得依赖 Docker socket、Docker daemon、host Docker、`docker build`、`docker push` 或等价 Docker-only 路径。新增或重构发布能力必须走纯 Kubernetes 运行面，例如 Tekton Task/Pipeline、Kubernetes Job、rootless/daemonless 构建器或其他不要求宿主 Docker 的 k8s 原生路径。
|
||||
- CI/CD、rollout、publish、image build 和部署链路禁止新引入 Docker 依赖；不得依赖 Docker socket、Docker daemon、host Docker、`docker build`、`docker push` 或等价 Docker-only 路径。
|
||||
- 正式 CI/CD、publish、image build 和 rollout 必须走 Tekton Task/Pipeline/PipelineRun 承担 CI，并通过 GitOps/Argo 承担部署收敛；普通 Kubernetes Job 只允许用于 bounded helper、source sync、diagnostic、cleanup 或 bootstrap，不得作为正式发布、镜像构建或 rollout 入口。
|
||||
- 触发或验收 rollout 时必须绑定 lane、source commit、PipelineRun/GitOps revision 和用户入口验证结果。
|
||||
- Secret 只通过 YAML sourceRef/targetKey 和受控 CLI 下发；输出只披露 presence/fingerprint。
|
||||
- 长命令用异步 job 或短轮询；不要长时间挂住 trans/ssh。
|
||||
|
||||
@@ -175,6 +175,7 @@ interface SentinelObservedExpectation {
|
||||
interface SentinelRemoteJobResult {
|
||||
readonly ok: boolean;
|
||||
readonly phase: string;
|
||||
readonly resourceKind?: "Job" | "PipelineRun";
|
||||
readonly jobName: string;
|
||||
readonly payload: Record<string, unknown>;
|
||||
readonly polls?: number;
|
||||
@@ -1563,16 +1564,16 @@ function sentinelSourceMirrorSshSetupShellLines(state: SentinelCicdState): strin
|
||||
}
|
||||
|
||||
function runSentinelPublishJob(state: SentinelCicdState, publishGitops: boolean, timeoutSeconds: number): SentinelRemoteJobResult {
|
||||
const jobName = `${stringAt(state.cicd, "builder.jobPrefix")}-${Date.now().toString(36)}`.replace(/[^a-z0-9-]/giu, "-").toLowerCase().slice(0, 63);
|
||||
const manifest = sentinelPublishJobManifest(state, jobName, publishGitops);
|
||||
const pipelineRunName = `${stringAt(state.cicd, "builder.jobPrefix")}-${Date.now().toString(36)}`.replace(/[^a-z0-9-]/giu, "-").toLowerCase().slice(0, 63);
|
||||
const manifest = sentinelPublishPipelineRunManifest(state, pipelineRunName, publishGitops);
|
||||
const namespace = stringAt(state.cicd, "builder.namespace");
|
||||
sentinelProgressEvent("sentinel.publish.progress", { phase: "create-job", status: "submitting", jobName, publishGitops, sourceCommit: state.sourceHead.commit, node: state.spec.nodeId, lane: state.spec.lane });
|
||||
const created = runCommand(["trans", stringAt(state.controlPlaneNode, "kubeRoute"), "sh", "--", createK8sJobScript(namespace, manifest)], repoRoot, { timeoutMs: Math.min(timeoutSeconds, 60) * 1000 });
|
||||
sentinelProgressEvent("sentinel.publish.progress", { phase: "create-pipelinerun", status: "submitting", pipelineRun: pipelineRunName, publishGitops, sourceCommit: state.sourceHead.commit, node: state.spec.nodeId, lane: state.spec.lane });
|
||||
const created = runCommand(["trans", stringAt(state.controlPlaneNode, "kubeRoute"), "sh", "--", createTektonPipelineRunScript(namespace, manifest)], repoRoot, { timeoutMs: Math.min(timeoutSeconds, 60) * 1000 });
|
||||
if (created.exitCode !== 0) {
|
||||
sentinelProgressEvent("sentinel.publish.progress", { phase: "create-job", status: "failed", jobName, publishGitops, node: state.spec.nodeId, lane: state.spec.lane });
|
||||
return withSentinelRemoteJobDiagnostics(state, { ok: false, phase: "create-job", jobName, payload: { ok: false, status: "create-failed", valuesRedacted: true }, create: compactCommand(created), valuesRedacted: true }, "publish");
|
||||
sentinelProgressEvent("sentinel.publish.progress", { phase: "create-pipelinerun", status: "failed", pipelineRun: pipelineRunName, publishGitops, node: state.spec.nodeId, lane: state.spec.lane });
|
||||
return withSentinelRemoteJobDiagnostics(state, { ok: false, phase: "create-pipelinerun", resourceKind: "PipelineRun", jobName: pipelineRunName, payload: { ok: false, status: "create-failed", valuesRedacted: true }, create: compactCommand(created), valuesRedacted: true }, "publish");
|
||||
}
|
||||
sentinelProgressEvent("sentinel.publish.progress", { phase: "create-job", status: "succeeded", jobName, publishGitops, node: state.spec.nodeId, lane: state.spec.lane });
|
||||
sentinelProgressEvent("sentinel.publish.progress", { phase: "create-pipelinerun", status: "succeeded", pipelineRun: pipelineRunName, publishGitops, node: state.spec.nodeId, lane: state.spec.lane });
|
||||
const startedAt = Date.now();
|
||||
const timeoutMs = Math.max(5_000, Math.min(timeoutSeconds * 1000, controlPlaneWaitWarningSeconds(state) * 1000));
|
||||
const warningBudgetMs = Math.max(1, Math.trunc(controlPlaneWaitWarningSeconds(state))) * 1000;
|
||||
@@ -1581,17 +1582,18 @@ function runSentinelPublishJob(state: SentinelCicdState, publishGitops: boolean,
|
||||
let lastProbe: Record<string, unknown> = {};
|
||||
while (Date.now() - startedAt < timeoutMs) {
|
||||
polls += 1;
|
||||
const probeCapture = runCommand(["trans", stringAt(state.controlPlaneNode, "kubeRoute"), "sh", "--", probeK8sJobScript(namespace, jobName)], repoRoot, { timeoutMs: Math.min(timeoutSeconds, 60) * 1000 });
|
||||
const probeCapture = runCommand(["trans", stringAt(state.controlPlaneNode, "kubeRoute"), "sh", "--", probeTektonPipelineRunScript(namespace, pipelineRunName)], repoRoot, { timeoutMs: Math.min(timeoutSeconds, 60) * 1000 });
|
||||
const probe = parseJsonObject(probeCapture.stdout) ?? {};
|
||||
lastProbe = { ...probe, capture: compactCommand(probeCapture) };
|
||||
const payload = sentinelPayloadFromLogs(String(probe.logsTail ?? ""));
|
||||
sentinelProgressEvent("sentinel.publish.progress", {
|
||||
phase: "remote-job",
|
||||
phase: "remote-pipelinerun",
|
||||
status: probe.succeeded === true ? "succeeded" : probe.failed === true ? "failed" : "running",
|
||||
jobName,
|
||||
pipelineRun: pipelineRunName,
|
||||
publishGitops,
|
||||
polls,
|
||||
elapsedMs: Date.now() - startedAt,
|
||||
taskRun: probe.taskRun ?? null,
|
||||
pod: probe.pod ?? null,
|
||||
sourceCommit: state.sourceHead.commit,
|
||||
node: state.spec.nodeId,
|
||||
@@ -1599,21 +1601,21 @@ function runSentinelPublishJob(state: SentinelCicdState, publishGitops: boolean,
|
||||
});
|
||||
if (probe.succeeded === true) {
|
||||
const ok = payload.ok === true;
|
||||
return withSentinelRemoteJobDiagnostics(state, { ok, phase: "job-succeeded", jobName, payload: Object.keys(payload).length === 0 ? { ok: false, status: "result-missing", valuesRedacted: true } : payload, polls, elapsedMs: Date.now() - startedAt, probe: lastProbe, valuesRedacted: true }, "publish");
|
||||
return withSentinelRemoteJobDiagnostics(state, { ok, phase: "pipelinerun-succeeded", resourceKind: "PipelineRun", jobName: pipelineRunName, payload: Object.keys(payload).length === 0 ? { ok: false, status: "result-missing", valuesRedacted: true } : payload, polls, elapsedMs: Date.now() - startedAt, probe: lastProbe, valuesRedacted: true }, "publish");
|
||||
}
|
||||
if (probe.failed === true) {
|
||||
return withSentinelRemoteJobDiagnostics(state, { ok: false, phase: "job-failed", jobName, payload: Object.keys(payload).length === 0 ? { ok: false, status: "failed", valuesRedacted: true } : payload, polls, elapsedMs: Date.now() - startedAt, probe: lastProbe, valuesRedacted: true }, "publish");
|
||||
return withSentinelRemoteJobDiagnostics(state, { ok: false, phase: "pipelinerun-failed", resourceKind: "PipelineRun", jobName: pipelineRunName, payload: Object.keys(payload).length === 0 ? { ok: false, status: "failed", valuesRedacted: true } : payload, polls, elapsedMs: Date.now() - startedAt, probe: lastProbe, valuesRedacted: true }, "publish");
|
||||
}
|
||||
if (!slowWarningSent && Date.now() - startedAt > warningBudgetMs) {
|
||||
slowWarningSent = true;
|
||||
sentinelProgressEvent("sentinel.publish.warning", { warning: `remote publish job exceeded configured ${Math.round(warningBudgetMs / 1000)}s timing budget; non-blocking timing alert`, jobName, elapsedMs: Date.now() - startedAt, node: state.spec.nodeId, lane: state.spec.lane });
|
||||
sentinelProgressEvent("sentinel.publish.warning", { warning: `remote publish PipelineRun exceeded configured ${Math.round(warningBudgetMs / 1000)}s timing budget; non-blocking timing alert`, pipelineRun: pipelineRunName, elapsedMs: Date.now() - startedAt, node: state.spec.nodeId, lane: state.spec.lane });
|
||||
}
|
||||
runCommand(["sleep", "2"], repoRoot, { timeoutMs: 3_000 });
|
||||
}
|
||||
return withSentinelRemoteJobDiagnostics(state, { ok: false, phase: "job-timeout", jobName, payload: { ok: false, status: "timeout", valuesRedacted: true }, polls, elapsedMs: Date.now() - startedAt, probe: lastProbe, valuesRedacted: true }, "publish");
|
||||
return withSentinelRemoteJobDiagnostics(state, { ok: false, phase: "pipelinerun-timeout", resourceKind: "PipelineRun", jobName: pipelineRunName, payload: { ok: false, status: "timeout", valuesRedacted: true }, polls, elapsedMs: Date.now() - startedAt, probe: lastProbe, valuesRedacted: true }, "publish");
|
||||
}
|
||||
|
||||
function sentinelPublishJobManifest(state: SentinelCicdState, jobName: string, publishGitops: boolean): Record<string, unknown> {
|
||||
function sentinelPublishPipelineRunManifest(state: SentinelCicdState, pipelineRunName: string, publishGitops: boolean): Record<string, unknown> {
|
||||
const namespace = stringAt(state.cicd, "builder.namespace");
|
||||
const buildkitImage = requireSentinelBuildkitImage(state);
|
||||
const proxyEnv = sentinelImageBuildProxyEnv(state);
|
||||
@@ -1623,76 +1625,93 @@ function sentinelPublishJobManifest(state: SentinelCicdState, jobName: string, p
|
||||
"unidesk.ai/spec-ref": "PJ2026-01060508",
|
||||
"unidesk.ai/node": state.spec.nodeId,
|
||||
"unidesk.ai/lane": state.spec.lane,
|
||||
"unidesk.ai/ci-system": "tekton",
|
||||
};
|
||||
return {
|
||||
apiVersion: "batch/v1",
|
||||
kind: "Job",
|
||||
metadata: { name: jobName, namespace, labels },
|
||||
apiVersion: "tekton.dev/v1",
|
||||
kind: "PipelineRun",
|
||||
metadata: {
|
||||
name: pipelineRunName,
|
||||
namespace,
|
||||
labels,
|
||||
annotations: {
|
||||
"unidesk.ai/source-commit": state.sourceHead.commit,
|
||||
"unidesk.ai/gitops-target-revision": stringAt(state.cicd, "argo.targetRevision"),
|
||||
"unidesk.ai/publish-gitops": publishGitops ? "true" : "false",
|
||||
},
|
||||
},
|
||||
spec: {
|
||||
backoffLimit: 0,
|
||||
activeDeadlineSeconds: numberAt(state.cicd, "builder.activeDeadlineSeconds"),
|
||||
ttlSecondsAfterFinished: numberAt(state.cicd, "builder.ttlSecondsAfterFinished"),
|
||||
template: {
|
||||
metadata: { labels },
|
||||
spec: {
|
||||
restartPolicy: "Never",
|
||||
timeouts: { pipeline: `${numberAt(state.cicd, "builder.activeDeadlineSeconds")}s` },
|
||||
taskRunTemplate: {
|
||||
podTemplate: {
|
||||
hostNetwork: true,
|
||||
dnsPolicy: "ClusterFirstWithHostNet",
|
||||
securityContext: { fsGroup: 1000 },
|
||||
volumes: [
|
||||
sentinelGitMirrorCacheVolume(state),
|
||||
{ name: "git-ssh", secret: { secretName: stringAt(state.cicd, "builder.gitSshSecretName"), defaultMode: 256 } },
|
||||
{ name: "workspace", emptyDir: { sizeLimit: "8Gi" } },
|
||||
{ name: "buildkit-state", emptyDir: { sizeLimit: "8Gi" } },
|
||||
{ name: "tmp", emptyDir: {} },
|
||||
],
|
||||
initContainers: [
|
||||
{
|
||||
name: "source",
|
||||
image: state.image.baseImage,
|
||||
imagePullPolicy: "IfNotPresent",
|
||||
env: proxyEnv,
|
||||
command: ["/bin/sh", "-ec", sentinelPublishSourceShell(state, jobName)],
|
||||
volumeMounts: [
|
||||
{ name: "workspace", mountPath: "/workspace" },
|
||||
{ name: "git-ssh", mountPath: "/git-ssh", readOnly: true },
|
||||
],
|
||||
},
|
||||
{
|
||||
name: "image-build",
|
||||
image: buildkitImage,
|
||||
imagePullPolicy: "IfNotPresent",
|
||||
env: [
|
||||
...proxyEnv,
|
||||
{ name: "BUILDKITD_FLAGS", value: "--oci-worker-no-process-sandbox --oci-worker-net=host --allow-insecure-entitlement network.host" },
|
||||
],
|
||||
command: ["/bin/sh", "-ec", sentinelPublishImageBuildShell(state, jobName)],
|
||||
securityContext: { privileged: true, runAsUser: 1000, runAsGroup: 1000 },
|
||||
volumeMounts: [
|
||||
{ name: "workspace", mountPath: "/workspace" },
|
||||
{ name: "buildkit-state", mountPath: "/home/user/.local/share/buildkit" },
|
||||
{ name: "tmp", mountPath: "/tmp" },
|
||||
],
|
||||
},
|
||||
],
|
||||
containers: [{
|
||||
name: "publish",
|
||||
image: state.image.baseImage,
|
||||
imagePullPolicy: "IfNotPresent",
|
||||
env: proxyEnv,
|
||||
command: ["/bin/sh", "-ec", sentinelPublishShell(state, jobName, publishGitops)],
|
||||
volumeMounts: [
|
||||
{ name: "workspace", mountPath: "/workspace" },
|
||||
{ name: "cache", mountPath: "/cache" },
|
||||
{ name: "git-ssh", mountPath: "/git-ssh", readOnly: true },
|
||||
],
|
||||
}],
|
||||
},
|
||||
},
|
||||
pipelineSpec: {
|
||||
tasks: [{
|
||||
name: "publish",
|
||||
taskSpec: {
|
||||
volumes: [
|
||||
sentinelGitMirrorCacheVolume(state),
|
||||
{ name: "git-ssh", secret: { secretName: stringAt(state.cicd, "builder.gitSshSecretName"), defaultMode: 256 } },
|
||||
{ name: "workspace", emptyDir: { sizeLimit: "8Gi" } },
|
||||
{ name: "buildkit-state", emptyDir: { sizeLimit: "8Gi" } },
|
||||
{ name: "tmp", emptyDir: {} },
|
||||
],
|
||||
steps: [
|
||||
{
|
||||
name: "source",
|
||||
image: state.image.baseImage,
|
||||
imagePullPolicy: "IfNotPresent",
|
||||
env: proxyEnv,
|
||||
script: tektonShellScript(sentinelPublishSourceShell(state, pipelineRunName)),
|
||||
volumeMounts: [
|
||||
{ name: "workspace", mountPath: "/workspace" },
|
||||
{ name: "git-ssh", mountPath: "/git-ssh", readOnly: true },
|
||||
],
|
||||
},
|
||||
{
|
||||
name: "image-build",
|
||||
image: buildkitImage,
|
||||
imagePullPolicy: "IfNotPresent",
|
||||
env: [
|
||||
...proxyEnv,
|
||||
{ name: "BUILDKITD_FLAGS", value: "--oci-worker-no-process-sandbox --oci-worker-net=host --allow-insecure-entitlement network.host" },
|
||||
],
|
||||
script: tektonShellScript(sentinelPublishImageBuildShell(state, pipelineRunName)),
|
||||
securityContext: { privileged: true, runAsUser: 1000, runAsGroup: 1000 },
|
||||
volumeMounts: [
|
||||
{ name: "workspace", mountPath: "/workspace" },
|
||||
{ name: "buildkit-state", mountPath: "/home/user/.local/share/buildkit" },
|
||||
{ name: "tmp", mountPath: "/tmp" },
|
||||
],
|
||||
},
|
||||
{
|
||||
name: "publish",
|
||||
image: state.image.baseImage,
|
||||
imagePullPolicy: "IfNotPresent",
|
||||
env: proxyEnv,
|
||||
script: tektonShellScript(sentinelPublishShell(state, pipelineRunName, publishGitops)),
|
||||
volumeMounts: [
|
||||
{ name: "workspace", mountPath: "/workspace" },
|
||||
{ name: "cache", mountPath: "/cache" },
|
||||
{ name: "git-ssh", mountPath: "/git-ssh", readOnly: true },
|
||||
],
|
||||
},
|
||||
],
|
||||
},
|
||||
}],
|
||||
},
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
function tektonShellScript(body: string): string {
|
||||
return `#!/bin/sh\n${body}`;
|
||||
}
|
||||
|
||||
function sentinelGitMirrorCacheVolume(state: SentinelCicdState): Record<string, unknown> {
|
||||
const hostPath = nonEmptyString(valueAtPath(state.controlPlaneTarget, "gitMirror.cacheHostPath"));
|
||||
if (hostPath !== null) return { name: "cache", hostPath: { path: hostPath, type: "DirectoryOrCreate" } };
|
||||
@@ -2041,6 +2060,22 @@ function createK8sJobScript(namespace: string, manifest: Record<string, unknown>
|
||||
].join("\n");
|
||||
}
|
||||
|
||||
function createTektonPipelineRunScript(namespace: string, manifest: Record<string, unknown>): string {
|
||||
const yaml = `${Bun.YAML.stringify(manifest).trim()}\n`;
|
||||
const pipelineRunName = stringAt(manifest, "metadata.name");
|
||||
return [
|
||||
"set -eu",
|
||||
`pipeline_run=${shellQuote(pipelineRunName)}`,
|
||||
`namespace=${shellQuote(namespace)}`,
|
||||
"kubectl -n \"$namespace\" delete pipelinerun \"$pipeline_run\" --ignore-not-found=true --wait=false >/dev/null 2>&1 || true",
|
||||
"kubectl -n \"$namespace\" delete taskrun -l tekton.dev/pipelineRun=\"$pipeline_run\" --ignore-not-found=true --wait=false >/dev/null 2>&1 || true",
|
||||
"kubectl -n \"$namespace\" delete pod -l tekton.dev/pipelineRun=\"$pipeline_run\" --ignore-not-found=true --wait=false >/dev/null 2>&1 || true",
|
||||
"tmp=$(mktemp)",
|
||||
`cat >"$tmp" <<'YAML'\n${yaml}YAML`,
|
||||
"kubectl apply -f \"$tmp\"",
|
||||
].join("\n");
|
||||
}
|
||||
|
||||
function probeK8sJobScript(namespace: string, jobName: string): string {
|
||||
return [
|
||||
"set +e",
|
||||
@@ -2061,6 +2096,29 @@ function probeK8sJobScript(namespace: string, jobName: string): string {
|
||||
].join("\n");
|
||||
}
|
||||
|
||||
function probeTektonPipelineRunScript(namespace: string, pipelineRunName: string): string {
|
||||
return [
|
||||
"set +e",
|
||||
`namespace=${shellQuote(namespace)}`,
|
||||
`pipeline_run=${shellQuote(pipelineRunName)}`,
|
||||
"condition_status=$(kubectl -n \"$namespace\" get pipelinerun \"$pipeline_run\" -o jsonpath='{.status.conditions[?(@.type==\"Succeeded\")].status}' 2>/dev/null || true)",
|
||||
"condition_reason=$(kubectl -n \"$namespace\" get pipelinerun \"$pipeline_run\" -o jsonpath='{.status.conditions[?(@.type==\"Succeeded\")].reason}' 2>/dev/null || true)",
|
||||
"condition_message_b64=$(kubectl -n \"$namespace\" get pipelinerun \"$pipeline_run\" -o jsonpath='{.status.conditions[?(@.type==\"Succeeded\")].message}' 2>/dev/null | head -c 1600 | base64 | tr -d '\\n' || true)",
|
||||
"task_run=$(kubectl -n \"$namespace\" get taskrun -l tekton.dev/pipelineRun=\"$pipeline_run\" -o jsonpath='{.items[0].metadata.name}' 2>/dev/null || true)",
|
||||
"pod=$(kubectl -n \"$namespace\" get pod -l tekton.dev/pipelineRun=\"$pipeline_run\" -o jsonpath='{.items[0].metadata.name}' 2>/dev/null || true)",
|
||||
"pod_phase=''",
|
||||
"if [ -n \"$pod\" ]; then pod_phase=$(kubectl -n \"$namespace\" get pod \"$pod\" -o jsonpath='{.status.phase}' 2>/dev/null || true); fi",
|
||||
"logs_tail=''",
|
||||
"if [ -n \"$pod\" ]; then logs_tail=$({ kubectl -n \"$namespace\" logs \"$pod\" --all-containers=true --tail=120 2>/dev/null || true; kubectl -n \"$namespace\" logs \"$pod\" -c step-publish --tail=180 2>/dev/null || true; } | tail -c 24000 | base64 | tr -d '\\n'); fi",
|
||||
"node - \"$condition_status\" \"$condition_reason\" \"$condition_message_b64\" \"$task_run\" \"$pod\" \"$pod_phase\" \"$logs_tail\" <<'NODE'",
|
||||
"const [conditionStatus, conditionReason, conditionMessageB64, taskRun, pod, podPhase, logsB64] = process.argv.slice(2);",
|
||||
"const message = Buffer.from(conditionMessageB64 || '', 'base64').toString('utf8');",
|
||||
"const active = conditionStatus === 'Unknown' || (!conditionStatus && (podPhase === 'Pending' || podPhase === 'Running'));",
|
||||
"console.log(JSON.stringify({ succeeded: conditionStatus === 'True', failed: conditionStatus === 'False', active, conditionStatus: conditionStatus || null, conditionReason: conditionReason || null, conditionMessage: message || null, taskRun: taskRun || null, pod: pod || null, podPhase: podPhase || null, logsTail: Buffer.from(logsB64 || '', 'base64').toString('utf8'), valuesRedacted: true }));",
|
||||
"NODE",
|
||||
].join("\n");
|
||||
}
|
||||
|
||||
function sentinelPayloadFromLogs(logsTail: string): Record<string, unknown> {
|
||||
const lines = logsTail.split(/\r?\n/u).map((line) => line.trim()).filter(Boolean);
|
||||
for (let index = lines.length - 1; index >= 0; index -= 1) {
|
||||
@@ -2084,16 +2142,21 @@ function sentinelRemoteJobDiagnostics(state: SentinelCicdState, result: Sentinel
|
||||
const envReuse = sentinelEnvReuseFromLogs(logsTail);
|
||||
const completedStages = sentinelCompletedStages(events, record(result.payload));
|
||||
const currentPhase = sentinelCurrentRemotePhase(result, events, domain);
|
||||
const isPipelineRun = result.resourceKind === "PipelineRun";
|
||||
const commands = {
|
||||
cliStatus: domain === "publish"
|
||||
? `bun scripts/cli.ts web-probe sentinel control-plane status --node ${state.spec.nodeId} --lane ${state.spec.lane}${sentinelCliSuffix(state)}`
|
||||
: `bun scripts/cli.ts web-probe sentinel image status --node ${state.spec.nodeId} --lane ${state.spec.lane}${sentinelCliSuffix(state)}`,
|
||||
logs: result.jobName === "-"
|
||||
? "-"
|
||||
: `trans ${stringAt(state.controlPlaneNode, "kubeRoute")} kubectl -n ${namespace} logs job/${result.jobName} --all-containers=true --tail=120`,
|
||||
: isPipelineRun
|
||||
? `trans ${stringAt(state.controlPlaneNode, "kubeRoute")} kubectl -n ${namespace} logs -l tekton.dev/pipelineRun=${result.jobName} --all-containers=true --tail=120`
|
||||
: `trans ${stringAt(state.controlPlaneNode, "kubeRoute")} kubectl -n ${namespace} logs job/${result.jobName} --all-containers=true --tail=120`,
|
||||
describe: result.jobName === "-"
|
||||
? "-"
|
||||
: `trans ${stringAt(state.controlPlaneNode, "kubeRoute")} kubectl -n ${namespace} describe job/${result.jobName}`,
|
||||
: isPipelineRun
|
||||
? `trans ${stringAt(state.controlPlaneNode, "kubeRoute")} kubectl -n ${namespace} describe pipelinerun/${result.jobName}`
|
||||
: `trans ${stringAt(state.controlPlaneNode, "kubeRoute")} kubectl -n ${namespace} describe job/${result.jobName}`,
|
||||
gitMirrorStatus: `bun scripts/cli.ts hwlab nodes git-mirror status --node ${state.spec.nodeId} --lane ${state.spec.lane}`,
|
||||
gitMirrorFlush: `bun scripts/cli.ts hwlab nodes git-mirror flush --node ${state.spec.nodeId} --lane ${state.spec.lane} --confirm --wait`,
|
||||
controlPlaneApply: `bun scripts/cli.ts web-probe sentinel control-plane apply --node ${state.spec.nodeId} --lane ${state.spec.lane}${sentinelCliSuffix(state)} --confirm --wait`,
|
||||
@@ -2101,12 +2164,17 @@ function sentinelRemoteJobDiagnostics(state: SentinelCicdState, result: Sentinel
|
||||
};
|
||||
return {
|
||||
domain,
|
||||
resourceKind: result.resourceKind ?? "Job",
|
||||
pipelineRun: isPipelineRun ? result.jobName : null,
|
||||
taskRun: probe.taskRun ?? null,
|
||||
currentPhase,
|
||||
completedStages,
|
||||
envReuse,
|
||||
pod: probe.pod ?? null,
|
||||
podPhase: probe.podPhase ?? null,
|
||||
active: probe.active ?? null,
|
||||
conditionStatus: probe.conditionStatus ?? null,
|
||||
conditionReason: probe.conditionReason ?? null,
|
||||
recentLogSummary: sentinelRecentLogSummary(logsTail),
|
||||
commands,
|
||||
valuesRedacted: true,
|
||||
@@ -2139,8 +2207,8 @@ function sentinelCompletedStages(events: readonly Record<string, unknown>[], pay
|
||||
}
|
||||
|
||||
function sentinelCurrentRemotePhase(result: SentinelRemoteJobResult, events: readonly Record<string, unknown>[], domain: "source-mirror" | "publish"): string {
|
||||
if (result.phase === "job-succeeded") return "completed";
|
||||
if (result.phase === "create-job") return "create-job";
|
||||
if (result.phase === "job-succeeded" || result.phase === "pipelinerun-succeeded") return "completed";
|
||||
if (result.phase === "create-job" || result.phase === "create-pipelinerun") return result.phase;
|
||||
const reversed = [...events].reverse();
|
||||
const failed = reversed.find((event) => event.status === "failed");
|
||||
if (failed !== undefined) return text(failed.stage);
|
||||
@@ -2164,7 +2232,7 @@ function sentinelRecentLogSummary(logsTail: string): string {
|
||||
|
||||
function sentinelRemoteJobTimeoutWarnings(job: unknown, subject: string): string[] {
|
||||
const remote = record(job);
|
||||
if (remote.phase !== "job-timeout") return [];
|
||||
if (remote.phase !== "job-timeout" && remote.phase !== "pipelinerun-timeout") return [];
|
||||
const diagnostics = record(remote.diagnostics);
|
||||
const commands = record(diagnostics.commands);
|
||||
return [`${subject} reached wait budget at phase=${text(diagnostics.currentPhase)} completed=${text(Array.isArray(diagnostics.completedStages) ? diagnostics.completedStages.join(",") : "")}; inspect logs with ${text(commands.logs)} and continue via ${text(commands.cliStatus)}.`];
|
||||
@@ -2250,13 +2318,13 @@ function sentinelProgressEvent(event: string, payload: Record<string, unknown>):
|
||||
|
||||
function confirmBlocked(action: string, state: SentinelCicdState): Record<string, unknown> {
|
||||
return {
|
||||
code: "sentinel-cicd-confirm-requires-remote-publish-job",
|
||||
code: "sentinel-cicd-confirm-requires-tekton-pipelinerun",
|
||||
action,
|
||||
reason: "P4 currently provides YAML-first render/status/trigger dry-run and refuses to report a deployment mutation before the remote publish job is wired to the node-local git mirror.",
|
||||
reason: "Confirmed publish uses a Tekton PipelineRun for image build and GitOps writeback; dry-run refuses to report a deployment mutation before that CI run is submitted.",
|
||||
sourceGitMirrorReadUrl: stringAt(state.cicd, "source.gitMirrorReadUrl"),
|
||||
requiredNextImplementation: [
|
||||
"clone source from source.gitMirrorReadUrl at selected commit",
|
||||
"build and push digest-pinned image on the selected node",
|
||||
"build and push digest-pinned image through Tekton PipelineRun on the selected node",
|
||||
"publish manifests to the HWLAB gitops branch/path through git-mirror",
|
||||
"flush/recheck git-mirror and let Argo reconcile the Application",
|
||||
],
|
||||
@@ -2798,9 +2866,10 @@ function renderPublishResult(publish: Record<string, unknown>): string {
|
||||
const timings = record(payload.stageTimings);
|
||||
const commands = record(diagnostics.commands);
|
||||
const proxySummary = [imageBuildProxy.httpProxyPresent, imageBuildProxy.httpsProxyPresent, imageBuildProxy.allProxyPresent].some((item) => item === true) ? "present" : "none";
|
||||
const runColumn = diagnostics.resourceKind === "PipelineRun" || publish.resourceKind === "PipelineRun" ? "PIPELINERUN" : "JOB";
|
||||
const lines = [
|
||||
"PUBLISH",
|
||||
table(["OK", "PHASE", "JOB", "ELAPSED", "POD", "CURRENT", "DIGEST", "GITOPS"], [[
|
||||
table(["OK", "PHASE", runColumn, "ELAPSED", "POD", "CURRENT", "DIGEST", "GITOPS"], [[
|
||||
publish.ok,
|
||||
publish.phase,
|
||||
publish.jobName,
|
||||
@@ -2850,9 +2919,11 @@ function renderPublishResult(publish: Record<string, unknown>): string {
|
||||
lines.push(
|
||||
"",
|
||||
"PUBLISH_DIAGNOSTICS",
|
||||
table(["POD_PHASE", "ACTIVE", "COMPLETED", "RECENT_LOG"], [[
|
||||
table(["TASKRUN", "POD_PHASE", "ACTIVE", "CONDITION", "COMPLETED", "RECENT_LOG"], [[
|
||||
diagnostics.taskRun ?? "-",
|
||||
diagnostics.podPhase ?? "-",
|
||||
diagnostics.active ?? "-",
|
||||
diagnostics.conditionReason ?? diagnostics.conditionStatus ?? "-",
|
||||
Array.isArray(diagnostics.completedStages) ? diagnostics.completedStages.join(",") : "-",
|
||||
diagnostics.recentLogSummary ?? "-",
|
||||
]]),
|
||||
|
||||
Reference in New Issue
Block a user