refactor: use git-controlled dev ci runner

This commit is contained in:
Codex
2026-05-18 08:38:17 +00:00
parent e2c8daede7
commit f86a75791b
22 changed files with 529 additions and 1402 deletions
+153 -95
View File
@@ -39,11 +39,13 @@ interface CiDevE2EOptions {
desiredRef: string;
deployCommit: string;
environment: "dev";
scriptRepo: string;
scriptPath: string;
scriptTimeoutMs: number;
services: Array<{ id: string; commitId: string; repo: string }>;
runId: string;
keepNamespace: boolean;
waitMs: number;
direct: boolean;
}
interface DispatchResult {
@@ -60,6 +62,11 @@ interface DeployDevManifestSummary {
deployCommit: string;
desiredRef: string;
environment: "dev";
ci: {
repo: string;
scriptPath: string;
timeoutMs: number;
};
services: Array<{ id: string; commitId: string; repo: string }>;
}
@@ -121,13 +128,26 @@ function coreBody(response: unknown): Record<string, unknown> | null {
return asRecord(asRecord(response)?.body);
}
function proxyBody(response: unknown): Record<string, unknown> | null {
const body = coreBody(response);
const nested = asRecord(body?.body);
return nested ?? body;
function positiveManifestNumber(value: unknown, fallback: number, path: string): number {
if (value === undefined || value === null) return fallback;
if (typeof value !== "number" || !Number.isInteger(value) || value <= 0) throw new Error(`${path} must be a positive integer`);
return value;
}
async function dispatchSsh(command: string, waitMs: number, remoteTimeoutMs: number): Promise<DispatchResult> {
function requireManifestString(value: unknown, path: string): string {
if (typeof value !== "string" || value.length === 0) throw new Error(`${path} must be a non-empty string`);
return value;
}
function requireCiScriptPath(value: unknown): string {
const scriptPath = requireManifestString(value, "environments.dev.ci.scriptPath");
if (!scriptPath.startsWith("scripts/ci/") || scriptPath.includes("..") || scriptPath.startsWith("/") || !scriptPath.endsWith(".sh")) {
throw new Error("environments.dev.ci.scriptPath must be a repo-relative scripts/ci/*.sh path");
}
return scriptPath;
}
async function dispatchSsh(command: string, waitMs: number, remoteTimeoutMs: number, pollCompletion = true): Promise<DispatchResult> {
const dispatchResponse = coreInternalFetch("/api/dispatch", {
method: "POST",
body: {
@@ -155,7 +175,18 @@ async function dispatchSsh(command: string, waitMs: number, remoteTimeoutMs: num
raw: dispatchResponse,
};
}
const deadline = Date.now() + Math.max(waitMs, remoteTimeoutMs + 10_000);
if (!pollCompletion) {
return {
ok: true,
taskId,
status: "submitted",
stdout: "",
stderr: "",
exitCode: null,
raw: dispatchBody,
};
}
const deadline = Date.now() + Math.max(waitMs, 1_000);
let latest: unknown = null;
while (Date.now() < deadline) {
latest = coreInternalFetch(`/api/tasks/${encodeURIComponent(taskId)}`, { maxResponseBytes: 3_000_000 });
@@ -183,7 +214,7 @@ async function dispatchSsh(command: string, waitMs: number, remoteTimeoutMs: num
taskId,
status: "timeout",
stdout: "",
stderr: `host.ssh task ${taskId} did not finish within ${Math.max(waitMs, remoteTimeoutMs + 10_000)}ms`,
stderr: `host.ssh task ${taskId} did not finish within ${Math.max(waitMs, 1_000)}ms`,
exitCode: null,
raw: latest,
};
@@ -439,44 +470,6 @@ spec:
`;
}
function devE2EPipelineRunManifest(options: CiDevE2EOptions): string {
const deployRevisionLabel = options.deployCommit.slice(0, 40);
return `apiVersion: tekton.dev/v1
kind: PipelineRun
metadata:
generateName: unidesk-dev-e2e-${options.runId}-
namespace: unidesk-ci
labels:
app.kubernetes.io/name: unidesk-dev-namespace-e2e
app.kubernetes.io/part-of: unidesk
unidesk.ai/ci-kind: dev-namespace-e2e
unidesk.ai/deploy-ref: master-deploy-json-dev
unidesk.ai/deploy-commit: ${JSON.stringify(deployRevisionLabel)}
spec:
pipelineRef:
name: unidesk-dev-namespace-e2e
taskRunTemplate:
serviceAccountName: unidesk-ci-runner
params:
- name: repo-url
value: ${JSON.stringify(options.repoUrl)}
- name: desired-ref
value: ${JSON.stringify(options.desiredRef)}
- name: deploy-commit
value: ${JSON.stringify(options.deployCommit)}
- name: environment
value: ${JSON.stringify(options.environment)}
- name: run-id
value: ${JSON.stringify(options.runId)}
- name: keep-namespace
value: ${JSON.stringify(options.keepNamespace ? "true" : "false")}
workspaces:
- name: shared-workspace
persistentVolumeClaim:
claimName: unidesk-ci-cache
`;
}
async function remoteCreatePipelineRun(manifest: string): Promise<string> {
const encoded = Buffer.from(manifest, "utf8").toString("base64");
const token = randomUUID().replace(/-/gu, "").slice(0, 12);
@@ -543,6 +536,68 @@ async function run(options: CiOptions): Promise<Record<string, unknown>> {
};
}
async function runRemoteDevE2ELauncher(options: CiDevE2EOptions): Promise<DispatchResult> {
const scriptTimeoutMs = Math.max(options.scriptTimeoutMs, options.waitMs, 60_000);
const remoteTimeoutMs = scriptTimeoutMs + 120_000;
const waitMs = options.waitMs > 0 ? options.waitMs + 30_000 : 0;
const command = [
"set -euo pipefail",
`run_id=${shellQuote(options.runId)}`,
`repo_url=${shellQuote(options.scriptRepo)}`,
`commit=${shellQuote(options.deployCommit)}`,
`script_path=${shellQuote(options.scriptPath)}`,
`desired_ref=${shellQuote(options.desiredRef)}`,
`environment=${shellQuote(options.environment)}`,
`keep_namespace=${shellQuote(options.keepNamespace ? "true" : "false")}`,
`timeout_ms=${shellQuote(String(scriptTimeoutMs))}`,
"work_dir=\"/tmp/unidesk-ci/$run_id\"",
"result_dir=\"/home/ubuntu/.unidesk/runs/$run_id\"",
"mkdir -p \"$work_dir\" \"$result_dir\"",
"launcher_log=\"$result_dir/launcher.log\"",
"exec > >(tee -a \"$launcher_log\") 2>&1",
"echo \"launcher_run_id=$run_id\"",
"echo \"launcher_repo=$repo_url\"",
"echo \"launcher_commit=$commit\"",
"echo \"launcher_script_path=$script_path\"",
"case \"$script_path\" in scripts/ci/*.sh) ;; *) echo \"invalid_script_path=$script_path\" >&2; exit 2 ;; esac",
"export DOCKER_CONFIG=/tmp/unidesk-ci-docker-config",
"mkdir -p \"$DOCKER_CONFIG\"",
"printf '{}\\n' > \"$DOCKER_CONFIG/config.json\"",
`build_proxy=${shellQuote(providerGatewayWsEgressProxyUrl)}`,
"export HTTP_PROXY=\"$build_proxy\" HTTPS_PROXY=\"$build_proxy\" ALL_PROXY=\"$build_proxy\"",
"export NO_PROXY=\"localhost,127.0.0.1,::1,host.docker.internal,.svc,.cluster.local,kubernetes.default.svc\"",
"if ! curl -fsSI --max-time 20 -x \"$build_proxy\" https://github.com >/dev/null; then",
" echo \"ci_provider_egress_proxy_unavailable=$build_proxy\" >&2",
" exit 1",
"fi",
"echo \"ci_provider_egress_proxy=provider-gateway-ws-egress:$build_proxy\"",
"repo_dir=\"$work_dir/repo\"",
"if [ ! -d \"$repo_dir/.git\" ]; then",
" git clone --no-checkout \"$repo_url\" \"$repo_dir\"",
"fi",
"git -C \"$repo_dir\" remote set-url origin \"$repo_url\"",
"git -C \"$repo_dir\" fetch --no-tags origin \"$commit\" || git -C \"$repo_dir\" fetch --no-tags origin '+refs/heads/*:refs/remotes/origin/*'",
"resolved=$(git -C \"$repo_dir\" rev-parse --verify \"$commit^{commit}\")",
"test \"$resolved\" = \"$commit\" || { echo \"resolved_commit_mismatch=$resolved expected=$commit\" >&2; exit 1; }",
"git -C \"$repo_dir\" cat-file -e \"$resolved:$script_path\"",
"git -C \"$repo_dir\" show \"$resolved:$script_path\" > \"$work_dir/runner.sh\"",
"chmod 700 \"$work_dir/runner.sh\"",
"echo \"runner_script_ready=$work_dir/runner.sh\"",
"runner_args=(",
" --run-id \"$run_id\"",
" --repo-url \"$repo_url\"",
" --desired-ref \"$desired_ref\"",
" --manifest-commit \"$commit\"",
" --environment \"$environment\"",
" --result-dir \"$result_dir\"",
" --timeout-ms \"$timeout_ms\"",
")",
"if [ \"$keep_namespace\" = \"true\" ]; then runner_args+=(--keep-namespace); fi",
"bash \"$work_dir/runner.sh\" \"${runner_args[@]}\"",
].join("\n");
return dispatchSsh(command, waitMs, remoteTimeoutMs, options.waitMs > 0);
}
function resolveDeployDevManifest(desiredRef: string): DeployDevManifestSummary {
const remoteRef = `refs/remotes/origin/${desiredRef}`;
const fetch = runCommand(["git", "fetch", "--quiet", "origin", `+refs/heads/${desiredRef}:${remoteRef}`], repoRoot);
@@ -556,6 +611,8 @@ function resolveDeployDevManifest(desiredRef: string): DeployDevManifestSummary
if (record?.schemaVersion !== 2) throw new Error(`origin/${desiredRef}:deploy.json must use schemaVersion=2`);
const environments = asRecord(record.environments);
const dev = asRecord(environments?.dev);
const ci = asRecord(dev?.ci);
if (ci === null) throw new Error(`origin/${desiredRef}:deploy.json must contain environments.dev.ci`);
const rawServices = Array.isArray(dev?.services) ? dev.services : [];
const services = rawServices.map((item) => {
const service = asRecord(item);
@@ -570,6 +627,11 @@ function resolveDeployDevManifest(desiredRef: string): DeployDevManifestSummary
deployCommit: deployCommitResult.stdout.trim(),
desiredRef,
environment: "dev",
ci: {
repo: requireManifestString(ci.repo, "environments.dev.ci.repo"),
scriptPath: requireCiScriptPath(ci.scriptPath),
timeoutMs: positiveManifestNumber(ci.timeoutMs, 1_800_000, "environments.dev.ci.timeoutMs"),
},
services,
};
}
@@ -580,68 +642,62 @@ function makeRunId(deployCommit: string): string {
}
async function runDevE2E(options: CiDevE2EOptions): Promise<Record<string, unknown>> {
if (!options.direct) {
const devopsResponse = coreInternalFetch("/api/microservices/devops/proxy/api/ci/dev-e2e/run", {
method: "POST",
body: {
repoUrl: options.repoUrl,
desiredRef: options.desiredRef,
environment: options.environment,
runId: options.runId,
keepNamespace: options.keepNamespace,
},
maxResponseBytes: 2_000_000,
});
const devopsBody = proxyBody(devopsResponse);
if (devopsBody?.ok === true) {
const pipelineRun = asString(devopsBody.pipelineRun);
const wait = pipelineRun.length > 0 ? await waitForPipelineRun(pipelineRun, options.waitMs) : null;
const waitSucceeded = wait === null || wait.exitCode === 0 || wait.stdout.trimStart().startsWith("True\tSucceeded\t");
return {
...devopsBody,
ok: waitSucceeded,
triggerMode: "devops-service",
wait: wait === null ? null : {
stdoutTail: wait.stdout.slice(-6000),
stderrTail: wait.stderr.slice(-6000),
},
};
}
return {
ok: false,
triggerMode: "devops-service",
error: "DevOps service trigger failed or did not return ok=true; use --direct only for CI bootstrap/recovery, never as a service deployment path.",
devopsResponse,
};
}
const name = await remoteCreatePipelineRun(devE2EPipelineRunManifest(options));
const wait = await waitForPipelineRun(name, options.waitMs);
const waitSucceeded = wait === null || wait.exitCode === 0 || wait.stdout.trimStart().startsWith("True\tSucceeded\t");
const result = await runRemoteDevE2ELauncher(options);
const ok = result.ok && (result.exitCode === null || result.exitCode === 0);
return {
ok: waitSucceeded,
pipelineRun: name,
ok,
runId: options.runId,
namespace: "unidesk-ci",
temporaryNamespace: `unidesk-ci-e2e-${options.runId}`,
repoUrl: options.repoUrl,
desiredRef: options.desiredRef,
deployCommit: options.deployCommit,
scriptRepo: options.scriptRepo,
scriptPath: options.scriptPath,
environment: options.environment,
services: options.services,
keepNamespace: options.keepNamespace,
triggerMode: "direct-maintenance",
wait: wait === null ? null : {
stdoutTail: wait.stdout.slice(-6000),
stderrTail: wait.stderr.slice(-6000),
triggerMode: "commit-pinned-ssh-launcher",
launcher: {
taskId: result.taskId,
status: result.status,
exitCode: result.exitCode,
stdoutTail: result.stdout.slice(-6000),
stderrTail: result.stderr.slice(-6000),
},
resultDir: `/home/ubuntu/.unidesk/runs/${options.runId}`,
next: [
`bun scripts/cli.ts ci logs ${name}`,
`bun scripts/cli.ts ci logs ${options.runId}`,
"bun scripts/cli.ts ci status",
],
};
}
async function logs(name: string): Promise<Record<string, unknown>> {
if (name.length === 0) throw new Error("ci logs requires PipelineRun name");
if (name.length === 0) throw new Error("ci logs requires run id or PipelineRun name");
if (/^[a-z0-9]([-a-z0-9]{0,46}[a-z0-9])?$/u.test(name)) {
const result = await dispatchSsh([
"set -euo pipefail",
`run_id=${shellQuote(name)}`,
"result_dir=\"/home/ubuntu/.unidesk/runs/$run_id\"",
"printf 'result_dir=%s\\n' \"$result_dir\"",
"found=0",
"if [ -f \"$result_dir/result.json\" ]; then found=1; echo '===== result.json'; cat \"$result_dir/result.json\"; fi",
"if [ -f \"$result_dir/launcher.log\" ]; then found=1; echo '===== launcher.log'; tail -n 160 \"$result_dir/launcher.log\"; fi",
"if [ -f \"$result_dir/runner.log\" ]; then found=1; echo '===== runner.log'; tail -n 240 \"$result_dir/runner.log\"; fi",
"if [ -f \"$result_dir/pods.log\" ]; then found=1; echo '===== pods.log'; tail -n 240 \"$result_dir/pods.log\"; fi",
"if [ \"$found\" = \"0\" ]; then echo \"no_run_files=$result_dir\" >&2; exit 42; fi",
].join("\n"), 60_000, 45_000);
if (result.ok || result.exitCode !== 42) {
return {
ok: result.ok,
runId: name,
output: result.stdout,
stderr: result.stderr,
exitCode: result.exitCode,
};
}
}
const result = await runRemoteKubectl([
"set -euo pipefail",
`kubectl get pipelinerun/${shellQuote(name)} -n unidesk-ci -o wide`,
@@ -664,7 +720,7 @@ function help(): Record<string, unknown> {
"bun scripts/cli.ts ci install",
"bun scripts/cli.ts ci run --revision <commit>",
"bun scripts/cli.ts ci run-dev-e2e --wait-ms 600000",
"bun scripts/cli.ts ci logs <pipelineRun>",
"bun scripts/cli.ts ci logs <runId>",
],
tekton: {
pipelineVersion: tektonPipelineVersion,
@@ -676,9 +732,9 @@ function help(): Record<string, unknown> {
},
},
runDevE2E: {
defaultTriggerMode: "devops-service",
directMaintenanceFlag: "--direct",
defaultTriggerMode: "commit-pinned-ssh-launcher",
desiredState: "origin/master:deploy.json#environments.dev",
scriptSource: "origin/master:deploy.json#environments.dev.ci",
},
};
}
@@ -712,11 +768,13 @@ export async function runCiCommand(_config: UniDeskConfig, args: string[]): Prom
desiredRef,
deployCommit: manifest.deployCommit,
environment: manifest.environment,
scriptRepo: manifest.ci.repo,
scriptPath: manifest.ci.scriptPath,
scriptTimeoutMs: manifest.ci.timeoutMs,
services: manifest.services,
runId,
keepNamespace: boolFlag(args, "--keep-namespace"),
waitMs,
direct: boolFlag(args, "--direct"),
});
}
if (action === "logs") return logs(nameArg ?? "");
+12 -29
View File
@@ -131,8 +131,8 @@ const nativeK3sInstallVersion = "v1.34.1+k3s1";
const nativeK3sImage = "rancher/k3s:v1.34.1-k3s1";
const nativeK3sCtrAddress = "/run/k3s/containerd/containerd.sock";
const unideskRepoUrl = "https://github.com/pikasTech/unidesk";
const d601MaintenanceDeployAllowedServiceIds = new Set(["devops"]);
const devApplySupportedServiceIds = d601MaintenanceDeployAllowedServiceIds;
const d601MaintenanceDeployAllowedServiceIds = new Set<string>();
const devApplySupportedServiceIds = new Set<string>();
const deployEnvironmentTargets: Record<DeployEnvironment, DeployEnvironmentTarget> = {
dev: {
environment: "dev",
@@ -195,7 +195,7 @@ function deployHelp(action: string | undefined = undefined): Record<string, unkn
},
options: [
{ name: "--file <path>", default: defaultDeployFile, description: "Desired-state manifest path relative to the repo root. JSON and ESM JS manifests are supported, for example deploy.json or develop.js." },
{ name: "--env <dev|prod>", description: "Read the named environment from origin/master:deploy.json. Direct D601 apply is enabled only for DevOps bootstrap/repair." },
{ name: "--env <dev|prod>", description: "Read the named environment from origin/master:deploy.json. Direct D601 service apply is disabled in the current CI-only phase." },
{ name: "--service <id>", description: "Limit reconcile to one service from the manifest." },
{ name: "--dry-run", description: "Prepare and validate without mutating the target service." },
{ name: "--force", description: "Redeploy even when the live commit appears up to date." },
@@ -665,20 +665,6 @@ function devK3sDeployService(id: string): UniDeskMicroserviceConfig | undefined
allowedMethods: ["GET", "HEAD", "POST", "PUT", "PATCH", "DELETE"],
allowedPathPrefixes: ["/", "/api/", "/logs"],
},
devops: {
name: "UniDesk DevOps Control",
description: "D601 k3s-managed DevOps control plane for normal CI trigger/status/log paths.",
dockerfile: "src/components/microservices/devops/Dockerfile",
composeFile: "src/components/microservices/k3sctl-adapter/k3s/devops.k3s.json",
composeService: "devops",
containerName: "k3s:devops",
nodeBaseUrl: "k3s://devops",
nodePort: 4286,
healthPath: "/health",
route: "/devops",
allowedMethods: ["GET", "HEAD", "POST"],
allowedPathPrefixes: ["/health", "/live", "/logs", "/api/"],
},
};
const spec = specs[id];
if (spec === undefined) return undefined;
@@ -697,9 +683,9 @@ function devK3sDeployService(id: string): UniDeskMicroserviceConfig | undefined
},
backend: {
nodeBaseUrl: spec.nodeBaseUrl,
nodeBindHost: `k3s://${id === "devops" ? "unidesk-ci" : "unidesk-dev"}/${spec.composeService}`,
nodeBindHost: `k3s://unidesk-dev/${spec.composeService}`,
nodePort: spec.nodePort,
proxyMode: id === "devops" ? "k3sctl-adapter-http" : "dev-k3s-direct",
proxyMode: "dev-k3s-direct",
frontendOnly: true,
public: false,
allowedMethods: spec.allowedMethods,
@@ -711,16 +697,14 @@ function devK3sDeployService(id: string): UniDeskMicroserviceConfig | undefined
mode: "k3sctl-managed",
adapterServiceId: "k3sctl-adapter",
k3sServiceId: spec.composeService,
namespace: id === "devops" ? "unidesk-ci" : "unidesk-dev",
namespace: "unidesk-dev",
expectedNodeIds: ["D601"],
activeNodeId: "D601",
},
development: {
providerId: "D601",
sshPassthrough: true,
worktreePath: id === "devops"
? "/home/ubuntu/.unidesk/devops-deploy"
: id === "code-queue"
worktreePath: id === "code-queue"
? "/home/ubuntu/unidesk-dev-code-queue-deploy/code-queue"
: `/home/ubuntu/unidesk-dev-core-deploy/${id}`,
},
@@ -767,7 +751,7 @@ function selectServices(config: UniDeskConfig, manifest: DeployManifest, service
if (manifest.environment === "dev") {
const service = devK3sDeployService(desired.id);
if (service === undefined) {
throw new Error(`deploy --env dev service ${desired.id} is not enabled in this executor yet; currently supported: ${[...devApplySupportedServiceIds].join(", ")}`);
throw new Error(`deploy --env dev service ${desired.id} is not enabled for direct rollout in the current CI-only phase`);
}
return { desired, config: service };
}
@@ -1021,7 +1005,6 @@ function dockerBuildTimeoutMs(service: UniDeskMicroserviceConfig, options: Deplo
function devK3sPrepullImages(service: UniDeskMicroserviceConfig): string[] {
if (!isDevK3sDeployService(service)) return [];
if (service.id === "devops") return ["golang:1.23-bookworm", "debian:bookworm-slim"];
return ["oven/bun:1-alpine"];
}
@@ -2186,7 +2169,7 @@ async function applyOneService(config: UniDeskConfig, service: UniDeskMicroservi
ok: false,
serviceId: service.id,
skipped: true,
reason: `D601 maintenance-channel direct deployment is allowed only for ${[...d601MaintenanceDeployAllowedServiceIds].join(", ")} bootstrap/repair. Deploy ${service.id} through the DevOps control plane instead.`,
reason: `D601 maintenance-channel direct deployment is disabled for ${service.id}. Current dev automation is CI-only; use ci run-dev-e2e for the temporary namespace smoke runner.`,
steps,
};
}
@@ -2361,7 +2344,7 @@ function blockedD601MaintenanceDeployServices(config: UniDeskConfig, manifest: D
}
function d601MaintenanceDeployBlockMessage(blocked: string[]): string {
return `D601 maintenance-channel direct deployment is allowed only for ${[...d601MaintenanceDeployAllowedServiceIds].join(", ")} bootstrap/repair; blocked services: ${blocked.join(", ")}. Use the DevOps control plane for other direct/managed microservices.`;
return `D601 maintenance-channel direct deployment is disabled for direct/managed services in the current CI-only phase; blocked services: ${blocked.join(", ")}. Use ci run-dev-e2e for dev smoke verification.`;
}
async function runApplyNow(config: UniDeskConfig, manifest: DeployManifest, options: DeployOptions): Promise<Record<string, unknown>> {
@@ -2413,7 +2396,7 @@ export async function runDeployCommand(config: UniDeskConfig | null, args: strin
if (options.environment !== "dev") throw new Error("deploy apply --env prod is not enabled yet");
const unsupported = unsupportedDevApplyServices(manifest, options.serviceId);
if (unsupported.length > 0) {
throw new Error(`deploy apply --env dev currently supports only ${[...devApplySupportedServiceIds].join(", ")}; unsupported selected services: ${unsupported.join(", ")}`);
throw new Error(`deploy apply --env dev is disabled for direct service rollout in the current CI-only phase; unsupported selected services: ${unsupported.join(", ")}. Use ci run-dev-e2e for dev smoke verification.`);
}
if (config === null) throw new Error("deploy apply --env dev requires config.json");
if (!options.dryRun) {
@@ -2438,5 +2421,5 @@ export async function runCodeQueueDeployCompatCommand(_config: UniDeskConfig, ar
if (args.includes("--skip-build")) throw new Error("codex deploy is disabled; --skip-build is not supported");
const providerId = optionValue(args, ["--provider-id", "--provider"]) ?? "D601";
if (providerId !== "D601") throw new Error(`codex deploy compatibility path only supports D601; got ${providerId}`);
throw new Error("codex deploy is disabled because D601 maintenance-channel direct deployment is now reserved for DevOps bootstrap/repair. Use the DevOps control plane for Code Queue deployment.");
throw new Error("codex deploy is disabled because D601 maintenance-channel direct deployment must not deploy Code Queue. Current dev automation is CI-only; use ci run-dev-e2e for dev smoke verification.");
}