1import * as _ from 'lodash-es'
2import { DateTime } from 'luxon'
3import { type V1Volume, type V1VolumeMount, type V1Container, type V1Pod, type V1Job, type V1ConfigMap } from '@kubernetes/client-node'
4import { genContext } from '../../../serverF/genContextF.mts'
5import { getKlusterCtx, type KubeResource } from '../ctxF/klusterCtxF.mts'
6import { appCfgCtx, getAppCfg} from '../ctxF/appCfgCtxF.mts'
7import { getAction } from '../../ctxF/actionCtxF.mts'
8import { read2Resource } from '../read2ResourceF.mts'
9import { secretFileTemplate } from '../k8sSecF.mts'
10import { get1ClusterPods } from '../getClusterPodsF.mts'
11import { regcredSecTempl, regCredSecName } from '../../dockerF/dockRegF.mts'
12import { genericPvcTmplA } from '../genericPvcF.mts'
13import { objectApiCall, delete2Resource, inspect2KubeRes } from '../kubeClientF.mts'
14import { getKubeApis } from '../getApisF.mts'
15import { liveRepoExec } from '../../libF/gitF/gitRepoF.mts'
16import { resourcesActionCore } from '../resourcesActionCoreF.mts'
17import { pvcVolumeMounts } from '../pvcVolumeMountsF.mts'
18import { sleep } from '../../../sharedF/sleepF.mts'
19import { assertDefined } from '../../../sharedF/assertsF/assertDefinedF.mts'
20import { throwDebugH } from '../../../serverF/throwErrorF/throwDebugHF.mts'
21import { assertTruthy } from '../../../sharedF/assertsF/assertTruthyAI.mts'
22import { calcShortHash } from '../../../serverF/calc1ShortHashF.mts'
23import { awaitJobPod, defaultJobTimeoutMs } from '../awaitJobPodAI.mts'
24import { nodeJsDockerImage } from '../../dockerF/nodeJsDockerImageF.mts'
25import { liveSpawn } from '../../../serverF/libFs/live1SpawnF.mts'
26import { eptKubeCli, noOutCmdCtx } from '../cliF/eptKubeCliF.mts'
27import { chalkCyan } from '../../../serverF/libChalkF.mts'
28import { autoPushOnDeploy } from '../../../configF.mts'
29import { isOcWeb } from '../../../sharedF/isDevPcF.mts'
30import { logElapsed } from '../../cliF/cliTimerCtxF.mts'
31import { cliFlag } from '../../../serverF/cliArgAI.mts'
32import { prebuildTarsPtPath } from '../../dockerF/prebuildTarsAI.mts'
33import { genImportTreeCtxTar, uploadCtxToPod } from './localCtxUploadAI.mts'
34import { mkProgressLogger } from '../../cliF/cliProgressAI.mts'
35import { getDfBuildLabel } from '../../dockerF/dockerUtilsF.mts'
36import { getPendingReason, getInitContainerStatus, pendingTimeoutMs, pvcPendingTimeoutMs } from '../podPendingStatusAI.mts'
37import { chalkRed } from '../../../serverF/libChalkF.mts'
38import { parseKanikoFailure } from '../../dockerF/dockRegErrorsAI.mts'
39import { serializeKanikoBuilds } from './serializeKanikoBuildsAI.mts'
40import { getOomKilledHint } from './kanikoOomHintAI.mts'
41import { type secretNameType } from '../../../serverF/secretsF/getPlainSecAI.mts'
42import { genPlainSecretIfMissing } from '../../secretF/genSecretAI.mts'
43import { isApplyishAction } from '../applyishActionF.mts'
44
45type TarSpec = {
46  tarPath: string
47  filesA: string[]
48  patchedFilesH?: Record<string, string>
49  wpManReplaceTokH?: Record<string, string>
50}
51
52type KanikoCtxStore = {
53  appliedShared: Set<string>
54  gitChecked: Set<string>
55  localCtxTarPath?: string
56  cluster_name?: string
57  dockreg_host?: string
58  git_sha?: string
59}
60
61const tarspecsMaxBytes = 1024 * 1024 // 1MB ConfigMap limit
62
63const kanikoCtx = genContext<KanikoCtxStore>()
64const getKanikoCtx = () => {
65  if (!kanikoCtx.getStore()) kanikoCtx.enterWith({appliedShared: new Set(), gitChecked: new Set()}) // ctx:clear
66  return kanikoCtx.getStore()!
67}
68
69const kanikoWorkspaceName = 'kanwork', kanikoWorkspacePath = `/${kanikoWorkspaceName}`, kanCacheDir = '/kanikocache'
70const kanikoWorkspaceVol: V1Volume = {emptyDir: {}, name: kanikoWorkspaceName}
71const kanikoWorkspaceMount: V1VolumeMount = {mountPath: kanikoWorkspacePath, name: kanikoWorkspaceName}
72
73const getGitInitCmd = ({gitRemoteUrl, git_sha}: {gitRemoteUrl: string, git_sha?: string}) => {
74  const fetchCmd = git_sha ? `git fetch origin ${git_sha}` : 'git fetch origin'
75  return `git init && git config remote.origin.url >&- || git remote add origin ${gitRemoteUrl} && ${fetchCmd}`
76}
77
78type CiContainersSshProps = {
79  git_sha?: string
80  gitsshHost: string
81  gitsshRepoPath: string
82  sshKeySecretName?: string
83}
84
85type CiContainersResult = {initContainers: V1Container[], volumes: V1Volume[], volumeMounts: V1VolumeMount[]}
86
87const ciContainersSsh = ({git_sha, gitsshHost, gitsshRepoPath, sshKeySecretName = 'codeserv-ssh'}: CiContainersSshProps): CiContainersResult => {
88  const sshKeyVolName = 'ssh-key'
89  const volumes = [
90    {name: sshKeyVolName, secret: {secretName: sshKeySecretName, defaultMode: 0o600}},
91    kanikoWorkspaceVol,
92  ]
93  const sshSetup = [
94    'mkdir -p ~/.ssh',
95    'cp /ssh-key/id_ed25519 ~/.ssh/id_ed25519',
96    'chmod 600 ~/.ssh/id_ed25519',
97    `ssh-keyscan -p 22 "${gitsshHost}" >> ~/.ssh/known_hosts 2>/dev/null || true`,
98  ].join(' && ')
99  const gitUrl = `ssh://git@${gitsshHost}${gitsshRepoPath}`
100  let cmd = `${sshSetup} && cd ${kanikoWorkspacePath} && ${getGitInitCmd({gitRemoteUrl: gitUrl, git_sha})}`
101  if (git_sha) cmd += ` && git reset --hard ${git_sha}`
102  const initContainers = [{
103    command: ['/bin/sh', '-c'], args: [cmd], image: 'alpine/git:latest', imagePullPolicy: 'Always', name: 'gitclone',
104    volumeMounts: [kanikoWorkspaceMount, {mountPath: '/ssh-key', name: sshKeyVolName, readOnly: true}]
105  }]
106  return {initContainers, volumes, volumeMounts: [kanikoWorkspaceMount]}
107}
108
109type CiContainersHttpsProps = {
110  git_sha?: string
111  gitsshHost: string
112  gitsshRepoPath: string
113}
114
115const ciContainersHttps = ({git_sha, gitsshHost, gitsshRepoPath}: CiContainersHttpsProps): CiContainersResult => {
116  const repoName = gitsshRepoPath.replace(/^\/home\/git\/repos\//, '').replace(/\.git$/, '')
117  const tokenVolName = 'http-token'
118  const volumes = [
119    {name: tokenVolName, secret: {secretName: `${gitsshHost}-httptoken`}},
120    kanikoWorkspaceVol,
121  ]
122  const gitUrl = `http://git:$(cat /token/token)@${gitsshHost}-http/git/${repoName}.git`
123  let cmd = `cd ${kanikoWorkspacePath} && ${getGitInitCmd({gitRemoteUrl: gitUrl, git_sha})}`
124  if (git_sha) cmd += ` && git reset --hard ${git_sha}`
125  const initContainers = [{
126    command: ['/bin/sh', '-c'], args: [cmd], image: 'alpine/git:latest', imagePullPolicy: 'Always', name: 'gitclone',
127    volumeMounts: [kanikoWorkspaceMount, {mountPath: '/token', name: tokenVolName, readOnly: true}]
128  }]
129  return {initContainers, volumes, volumeMounts: [kanikoWorkspaceMount]}
130}
131
132type CiContainersProps = {
133  git_sha?: string
134  gitRemoteSecretName: string
135  gitRemoteKubeSecretName: string
136}
137
138const ciContainers = ({git_sha, gitRemoteSecretName, gitRemoteKubeSecretName}: CiContainersProps): CiContainersResult => {
139  const volumes = [
140    {name: gitRemoteKubeSecretName, secret: {secretName: gitRemoteKubeSecretName}},
141    kanikoWorkspaceVol,
142  ]
143  let cmd = `cd ${kanikoWorkspacePath} && ${getGitInitCmd({gitRemoteUrl: `$(cat /secrets2/${gitRemoteSecretName})`, git_sha})}`
144  if (git_sha) cmd += ` && git reset --hard ${git_sha}`
145  const initContainers = [{
146    command: ['/bin/sh', '-c'], args: [cmd], image: 'alpine/git:latest', imagePullPolicy: 'Always', name: 'gitclone',
147    volumeMounts: [kanikoWorkspaceMount, {mountPath: '/secrets2', name: gitRemoteKubeSecretName}]
148  }]
149  return {initContainers, volumes, volumeMounts: [kanikoWorkspaceMount]}
150}
151
152const ciContainersLocalCtx = (): CiContainersResult => {
153  const waitctx: V1Container = {name: 'waitctx', image: 'busybox:latest', command: ['/bin/sh', '-c'], args: [`while [ ! -f ${kanikoWorkspacePath}/.ctxready ]; do sleep 2; done`], volumeMounts: [kanikoWorkspaceMount]}
154  const extractctx: V1Container = {name: 'extractctx', image: 'busybox:latest', command: ['/bin/sh', '-c'], args: [`cd ${kanikoWorkspacePath} && tar -xzf localctx.tar.gz && rm -f localctx.tar.gz`], volumeMounts: [kanikoWorkspaceMount]}
155  return {initContainers: [waitctx, extractctx], volumes: [kanikoWorkspaceVol], volumeMounts: [kanikoWorkspaceMount]}
156}
157
158type KanikoBuildJobTmplProps = {
159  name: string
160  repo_name_tag: string
161  gitRemoteKubeSecretName?: string
162  gitRemoteSecretName?: string
163  df_sha: string
164  isStandalone?: boolean
165  pvcName: string
166  dockerfileCfgMap: V1ConfigMap
167  tarspecsCfgMapName?: string
168  gitsshHost?: string
169  gitsshRepoPath?: string
170  gitsshHttpsHostname?: string
171}
172
173const kanikoBuildJobTmpl = ({name, repo_name_tag, gitRemoteKubeSecretName, gitRemoteSecretName, df_sha, isStandalone, pvcName, dockerfileCfgMap, tarspecsCfgMapName, gitsshHost, gitsshRepoPath, gitsshHttpsHostname}: KanikoBuildJobTmplProps): V1Job => {
174  const {cluster_name, dockreg_host, git_sha} = getKanikoCtx()
175  assertDefined(cluster_name), assertDefined(dockreg_host), assertDefined(git_sha)
176  const sshKeySecretName = isOcWeb ? process.env.GITSSH_SSH_SECRET : undefined
177  const jobName = `kaniko2-${name}`
178  const jobBase = {apiVersion: 'batch/v1', kind: 'Job', metadata: {name: jobName, namespace: 'default', labels: {cluster_name}}}
179  const noCloneCi: CiContainersResult = {initContainers: [] as V1Container[], volumes: [kanikoWorkspaceVol], volumeMounts: [kanikoWorkspaceMount]}
180  const kanikoLocalContext = getAppCfg()?.kanikoLocalContext ?? true
181  const ci = kanikoLocalContext ? (tarspecsCfgMapName ? ciContainersLocalCtx() : noCloneCi) : isStandalone ? noCloneCi : (gitsshHttpsHostname ? ciContainersHttps({git_sha, gitsshHost: gitsshHost!, gitsshRepoPath: gitsshRepoPath!}) : gitsshHost ? ciContainersSsh({git_sha, gitsshHost, gitsshRepoPath: gitsshRepoPath!, ...(sshKeySecretName && {sshKeySecretName})}) : ciContainers({git_sha, gitRemoteSecretName: gitRemoteSecretName!, gitRemoteKubeSecretName: gitRemoteKubeSecretName!}))
182  const pvc = pvcVolumeMounts({name: pvcName, mountPath: kanCacheDir})
183  let {initContainers} = ci
184  const volumes = [...ci.volumes, ...pvc.volumes]
185  const volumeMounts = [...ci.volumeMounts, ...pvc.volumeMounts]
186
187  const appPath = getAppCfg()?.appPath
188  const nextBuildHeapMb = getAppCfg()?.nextBuildHeapMb || 2048
189  const kanikoCpuLimit = getAppCfg()?.kanikoCpuLimit || '2'
190  const nextCachePath = appPath ? `/approot/${appPath}/.next/cache` : undefined
191  const nextCacheSubPath = appPath ? `next-cache-${appPath.replace(/\//g, '-')}` : undefined
192
193  if (tarspecsCfgMapName) {
194    const prebuildCmd = `cd ${kanikoWorkspacePath} && node ${prebuildTarsPtPath}`
195    initContainers = [
196      ...initContainers,
197      {
198        name: 'prebuild',
199        image: nodeJsDockerImage(),
200        command: ['/bin/sh', '-c'],
201        args: [prebuildCmd],
202        resources: {requests: {memory: '2Gi', cpu: '500m'}, limits: {memory: '4Gi', cpu: '2'}},
203        env: [{name: 'TAR_SPECS_PATH', value: '/tarspecs/tarspecs.json'}, {name: 'PNPM_HOME', value: '/pnpm'}, {name: 'HOME', value: '/pnpm'}, {name: 'PATH', value: '/pnpm:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin'}, ...((getAppCfg() as Record<string, unknown>)?.forceIpv4Dns ? [{name: 'NODE_OPTIONS', value: '--dns-result-order=ipv4first'}] : [])],
204        volumeMounts: [kanikoWorkspaceMount, {name: pvcName, mountPath: '/pnpm/store', subPath: 'pnpm-store'}, {mountPath: '/tarspecs', name: 'tarspecs-cfg', readOnly: true}],
205      },
206    ]
207    volumes.push({name: 'tarspecs-cfg', configMap: {name: tarspecsCfgMapName}})
208  }
209
210  return _.merge(jobBase, {
211    metadata: {labels: {cluster_name, git_sha, df_sha}},
212    spec: {
213      ttlSecondsAfterFinished: 60 * 60 * 2,
214      activeDeadlineSeconds: Math.floor(defaultJobTimeoutMs / 1000) - 60, // <client wait so k8s DeadlineExceeded surfaces; auto-fails stuck jobs for TTL cleanup
215      backoffLimit: 0,
216      template: {
217        metadata: {labels: {name, git_sha}},
218        spec: {
219          restartPolicy: 'Never',
220          initContainers,
221          containers: [{
222            // image: 'gcr.io/kaniko-project/executor:v1.10.0',
223            image: 'ghcr.io/kaniko-build/dist/chainguard-forks-kaniko/executor:v1.25.11-debug',
224            args: _.compact([
225              `--dockerfile=/dockerfilecfg/Dockerfile`, `--context=dir://${kanikoWorkspacePath}`, `--destination=${repo_name_tag}`,
226              `--cache=true`, `--cache-dir=${kanCacheDir}`, `--snapshot-mode=redo`, `--use-new-run`, `--ignore-path=/pnpm/store`,
227              nextCachePath && `--ignore-path=${nextCachePath}`, cluster_name === 'minik' && '--force'
228            ]),
229            name: 'kaniko',
230            resources: {requests: {memory: `${nextBuildHeapMb}Mi`, cpu: '500m'}, limits: {memory: `${nextBuildHeapMb + 2048}Mi`, cpu: kanikoCpuLimit}},
231            volumeMounts: [...volumeMounts, {name: pvcName, mountPath: '/pnpm/store', subPath: 'pnpm-store'},
232              ...(nextCachePath ? [{name: pvcName, mountPath: nextCachePath, subPath: nextCacheSubPath!}] : []),
233              {name: 'dockerfilecfg', mountPath: '/dockerfilecfg'}, {mountPath: '/kaniko/.docker', name: 'dockerconfigvol'}]
234          }],
235          volumes: [
236            ...volumes,
237            {name: 'dockerfilecfg', configMap: {name: dockerfileCfgMap.metadata?.name}},
238            {name: 'dockerconfigvol', secret: {items: [{key: '.dockerconfigjson', path: 'config.json'}], secretName: regCredSecName({dockreg_host})}}
239          ]
240        }
241      }
242    }
243  })
244}
245
246const getGitRepoVars = ({git_repo_name}: {git_repo_name: string}) => {
247  const gitRemoteSecretName = `git_remote_${git_repo_name}` as secretNameType
248  const gitRemoteSecret = secretFileTemplate({name: gitRemoteSecretName})
249  const {metadata: {name: gitRemoteKubeSecretName}} = gitRemoteSecret
250  return {gitRemoteSecretName, gitRemoteKubeSecretName, gitRemoteSecret}
251}
252
253const findMatchingPods = async (cluster_name: string, cb: (pod: V1Pod) => boolean) => (await get1ClusterPods({cluster_name}) as V1Pod[]).filter(cb)
254
255const gitshaPath = ['metadata', 'labels', 'git_sha']
256
257const getLatestPod = async ({cluster_name, name, git_sha}: {cluster_name: string, name: string, git_sha: string}) => {
258  let jobPods = await findMatchingPods(cluster_name, (podH) => {
259    const podName = podH.metadata?.name
260    if (!podName?.includes(name)) return false
261    return git_sha == _.get(podH, gitshaPath)
262  })
263  jobPods = _.chain(jobPods).sortBy((pod) => {
264    const ts = pod.metadata?.creationTimestamp
265    return DateTime.fromISO(typeof ts === 'string' ? ts : ts?.toISOString() || '')
266  }).reverse().value()
267  return jobPods[0]
268}
269
270const getKanikoPod = async ({newJob, cluster_name}: {newJob: V1Job, cluster_name: string}) => {
271  const pods = await findMatchingPods(cluster_name, (podH) => {
272    const podName = podH.metadata?.name
273    if (!podName?.includes(newJob.metadata?.name || '')) return false
274    if (_.get(newJob, gitshaPath) != _.get(podH, gitshaPath)) return false
275    const containerStatuses = podH.status?.containerStatuses
276    return !!containerStatuses?.find(({name, state}) => name === 'kaniko' && (state?.running || state?.terminated))
277  })
278  return pods[0]
279}
280
281const getFailedContainerLogs = async ({cluster_name, podName, pod}: {cluster_name: string, podName: string, pod: V1Pod}): Promise<{logs: string, oomKilledHint?: string}> => {
282  const allStatuses = [...(pod.status?.initContainerStatuses || []), ...(pod.status?.containerStatuses || [])]
283  const failedContainers = allStatuses.filter(s => s.state?.terminated?.exitCode !== 0 && s.state?.terminated)
284  let allLogs = '', oomKilledHint: string | undefined
285  for (const container of failedContainers) {
286    const containerName = container.name!
287    const oomHint = getOomKilledHint({terminatedReason: container.state?.terminated?.reason, containerName})
288    if (oomHint) {
289      oomKilledHint = oomHint
290      console.error(chalkRed(`\n>>> ${oomHint}`))
291    }
292    console.log(`\n--- ${containerName} logs (${podName}) ---`)
293    const result = await noOutCmdCtx.run({}, () => eptKubeCli([cluster_name, 'logs', podName, '-c', containerName, '--tail=150']))
294    const logs = (result as {stdout?: string})?.stdout || '(empty)'
295    console.log(logs)
296    allLogs += logs + '\n'
297
298    const parsed = parseKanikoFailure(logs)
299    if (parsed) console.log(chalkRed(`\n>>> ${parsed.hint}`))
300  }
301  return {logs: allLogs, oomKilledHint}
302}
303
304const guardJobFailureStatus = async ({jobH, cluster_name}: {jobH: V1Job, cluster_name: string}) => {
305  const {succeeded, conditions} = jobH.status || {}
306  if (succeeded) return {succeeded: !!succeeded}
307  const failureStatus = conditions?.find(({type}) => type == 'Failed')
308  if (failureStatus) {
309    console.log(inspect2KubeRes({resource: jobH as KubeResource}), {failureStatus})
310    const jobName = jobH.metadata?.name
311    const pods = await findMatchingPods(cluster_name, (p) => !!p.metadata?.name?.includes(jobName || ''))
312    const failedPod = pods.find((p) => p.status?.phase === 'Failed') || pods[0]
313    let logs = '', oomKilledHint: string | undefined
314    if (failedPod?.metadata?.name) ({logs, oomKilledHint} = await getFailedContainerLogs({cluster_name, podName: failedPod.metadata.name, pod: failedPod}))
315    throwDebugH({jobName, failureStatus, oomKilledHint, logs})
316  }
317  return {}
318}
319
320const checkKanikoStatus = async ({newJob}: {newJob: V1Job}) => {
321  const {cluster_name} = getKanikoCtx()
322  assertDefined(cluster_name)
323  return guardJobFailureStatus({jobH: await read2Resource({resource: newJob as KubeResource, cluster_name}) as V1Job, cluster_name})
324}
325
326type GetKanikoResourcesProps = {
327  name: string
328  dockerfileContent: string
329  tarSpecsA?: TarSpec[]
330  df_sha: string
331  isStandalone?: boolean
332  repo_name_tag: string
333  action?: string
334  gitsshHost?: string
335  gitsshRepoPath?: string
336  gitsshHttpsHostname?: string
337}
338
339export const getKanikoResources = async ({name, dockerfileContent, tarSpecsA, df_sha, isStandalone, repo_name_tag, action, gitsshHost, gitsshRepoPath, gitsshHttpsHostname}: GetKanikoResourcesProps) => {
340  const {cluster_name} = getKanikoCtx()
341  assertDefined(cluster_name)
342  const {git_repo_name} = getAppCfg()
343  const kanikoLocalContext = getAppCfg()?.kanikoLocalContext ?? true
344  const dockerfileCfgMap: V1ConfigMap = {apiVersion: 'v1', kind: 'ConfigMap', metadata: {name, labels: {cluster_name}}, data: {Dockerfile: dockerfileContent}}
345  const useSsh = !!gitsshHost, useGitsshHttps = !!gitsshHttpsHostname
346  let gitRemoteKubeSecretName: string | undefined, gitRemoteSecretName: secretNameType | undefined, gitRemoteSecret: any
347  if (!useSsh && !useGitsshHttps && !isStandalone) {
348    ;({gitRemoteKubeSecretName, gitRemoteSecretName, gitRemoteSecret} = getGitRepoVars({git_repo_name: git_repo_name!}))
349  }
350  const pvcName = 'kanikocache-' + (kanikoLocalContext ? 'localctx' : isStandalone ? 'standalone' : useGitsshHttps ? 'gitssh-https' : useSsh ? 'gitssh' : git_repo_name)
351
352  let tarspecsCfgMapName: string | undefined
353  const resources: any[] = [dockerfileCfgMap]
354  if (tarSpecsA?.length) {
355    tarspecsCfgMapName = `${name}-tarspecs`
356    const tarspecsJson = JSON.stringify({tarSpecsA})
357    if (tarspecsJson.length > tarspecsMaxBytes) throwDebugH({tarspecsBytes: tarspecsJson.length, tarspecsMaxBytes})
358    const tarspecsCfgMap = {apiVersion: 'v1', kind: 'ConfigMap', metadata: {name: tarspecsCfgMapName, labels: {cluster_name}}, data: {'tarspecs.json': tarspecsJson}}
359    resources.push(tarspecsCfgMap)
360  }
361
362  const newJob = kanikoBuildJobTmpl({name, repo_name_tag, gitRemoteKubeSecretName, gitRemoteSecretName: gitRemoteSecretName as string | undefined, df_sha, isStandalone, dockerfileCfgMap, tarspecsCfgMapName, pvcName, gitsshHost, gitsshRepoPath, gitsshHttpsHostname})
363  resources.push(...await genericPvcTmplA({cluster_name, name: pvcName, sizeGb: 200, useM2: true, action, accessModes: ['ReadWriteMany']}))
364  return {newJob, gitRemoteSecret, resources, useSsh}
365}
366
367const getPodPhase = (pod: any) => pod?.status?.phase || 'Unknown'
368
369const kanikoRetryable = async ({newJob, dfLabel, repo_name_tag}: {newJob: V1Job, dfLabel: string, repo_name_tag: string}) => {
370  assertDefined(newJob.metadata?.name)
371  const name = newJob.metadata.name, git_sha = _.get(newJob, gitshaPath)
372  const {localCtxTarPath, cluster_name} = getKanikoCtx()
373  assertDefined(cluster_name)
374  const maxRetries = 120, pollMs = 10_000
375  const buildLabel = getDfBuildLabel({dfLabel, name, repo_name_tag})
376  const logStatus = mkProgressLogger(`${buildLabel} kanikobuild`, defaultJobTimeoutMs)
377  let pendingStartTime: number | null = null, lastPendingLog: string | null = null
378  let ctxUploaded = false
379  await serializeKanikoBuilds({newJob, cluster_name, git_sha, logStatus})
380  
381  const throwKanikoBuildFailed = async ({failReason, delStaleJobHint}: {failReason?: string, delStaleJobHint?: boolean}): Promise<never> => {
382    console.log(`\n--- kaniko build failed ---`)
383    const pods = await findMatchingPods(cluster_name, p => !!p.metadata?.name?.includes(name))
384    const failedPod = pods.find(p => p.status?.phase === 'Failed') || pods[0]
385    let logs = '', oomKilledHint: string | undefined
386    if (failedPod?.metadata?.name) ({logs, oomKilledHint} = await getFailedContainerLogs({cluster_name, podName: failedPod.metadata.name, pod: failedPod}))
387    if (!oomKilledHint && delStaleJobHint) console.error(chalkRed(`\n>>> kaniko job failed (${failReason || 'unknown'}). Re-run with --delStaleJob to delete and retry.`))
388    throwDebugH({name, reason: oomKilledHint ? 'OOMKilled during kaniko build' : 'kaniko build failed', failReason, oomKilledHint, logs})
389  }
390
391  for (let i = 0; i < maxRetries; i++) {
392   try {
393    const existingJob = await read2Resource({resource: newJob as KubeResource, cluster_name}) as V1Job | undefined
394    if (existingJob) {
395      const dfShaMismatch = _.get(existingJob, 'metadata.labels.df_sha') !== _.get(newJob, 'metadata.labels.df_sha')
396      const isFailed = existingJob.status?.failed
397      if (isFailed) {
398        if (cliFlag('--delStaleJob')) {
399          await delete2Resource({resource: existingJob as KubeResource, cluster_name})
400          logStatus('deleted stale job'); await sleep(15_000); continue
401        }
402        const failCondition = existingJob.status?.conditions?.find(c => c.type === 'Failed')
403        await throwKanikoBuildFailed({failReason: failCondition?.reason, delStaleJobHint: true})
404      }
405      if (dfShaMismatch) {
406        await delete2Resource({resource: existingJob as KubeResource, cluster_name})
407        logStatus('expired'); await sleep(15_000); continue
408      }
409    }
410    
411    if (!existingJob) {
412      const resp = await objectApiCall({resource: newJob as KubeResource, objectApiMethod: 'create', cluster_name})
413      if (resp?.kind === 'Job') logStatus('created')
414      else if (resp?.kind === 'Status') { console.error(resp); throwDebugH({resp, context: 'kaniko job create'}) }
415      else if (resp) console.error('kaniko create error', resp)
416      await sleep(pollMs); continue
417    }
418    
419    const {succeeded} = await guardJobFailureStatus({jobH: existingJob, cluster_name})
420    if (succeeded) { console.log(' done'); return }
421    
422    const latestPod = await getLatestPod({cluster_name, git_sha, name})
423    const phase = getPodPhase(latestPod)
424    if (phase === 'Failed') { logStatus('Failed'); throwDebugH({podName: latestPod?.metadata?.name}) }
425    
426    if (localCtxTarPath && !ctxUploaded) {
427      const waitctxPods = await findMatchingPods(cluster_name, p => {
428        if (!p.metadata?.name?.includes(name)) return false
429        return !!p.status?.initContainerStatuses?.find(s => s.name === 'waitctx' && s.state?.running)
430      })
431      if (waitctxPods[0]?.metadata?.name) {
432        logStatus('uploading ctx')
433        await uploadCtxToPod({cluster_name, podName: waitctxPods[0].metadata.name, containerName: 'waitctx', ctxTarPath: localCtxTarPath})
434        ctxUploaded = true
435      }
436    }
437    
438    const runningPod = await getKanikoPod({newJob, cluster_name})
439    if (!runningPod) {
440      const initInfo = getInitContainerStatus(latestPod)
441      const pendingInfo = getPendingReason(latestPod)
442      const statusStr = initInfo ? `${phase} (${initInfo.name}${initInfo.detail ? ': ' + initInfo.detail : ' ' + initInfo.status})` : phase
443      logStatus(statusStr)
444      // Only timeout if truly stuck in Pending (unschedulable) - not during normal init container execution
445      const isInitRunning = initInfo && (initInfo.status === 'running' || initInfo.detail === 'PodInitializing')
446      if (phase === 'Pending' && pendingInfo && !isInitRunning) {
447        pendingStartTime ||= Date.now()
448        const timeout = pendingInfo.isPvcPending ? pvcPendingTimeoutMs : pendingTimeoutMs
449        if (Date.now() - pendingStartTime > timeout) {
450          console.error(chalkRed(`\n${pendingInfo.message || 'Pod stuck in Pending'}`))
451          throwDebugH({podName: latestPod?.metadata?.name, reason: pendingInfo.reason, message: pendingInfo.message})
452        }
453        if (pendingInfo.message && pendingInfo.message !== lastPendingLog) { console.error(chalkRed(`\n${pendingInfo.message}`)); lastPendingLog = pendingInfo.message }
454      } else if (phase === 'Pending' && !pendingInfo && !isInitRunning) {
455        pendingStartTime ||= Date.now()
456        if (Date.now() - pendingStartTime > pvcPendingTimeoutMs * 5) {
457          throwDebugH({podName: latestPod?.metadata?.name, reason: 'volume attach timeout'})
458        }
459      } else if (initInfo?.status === 'failed') {
460        console.error(chalkRed(`\ninit container ${initInfo.name} failed: ${initInfo.detail || 'unknown'}`))
461        throwDebugH({podName: latestPod?.metadata?.name, initContainer: initInfo.name, reason: initInfo.detail})
462      } else { pendingStartTime = null }
463      await sleep(pollMs); continue
464    }
465    
466    const podAge = runningPod.metadata?.creationTimestamp
467      ? Math.round((Date.now() - new Date(runningPod.metadata.creationTimestamp as unknown as string).getTime()) / 1000)
468      : 0
469    logStatus(podAge > 60 ? `streaming (running ${Math.round(podAge / 60)}m)` : 'building')
470    console.log()
471    assertDefined(runningPod.metadata?.name)
472    const awaitRes = await awaitJobPod({job: newJob as KubeResource, podName: runningPod.metadata.name, containerName: 'kaniko', cluster_name, skipHeader: true})
473    if (awaitRes?.failed) await throwKanikoBuildFailed({failReason: awaitRes.failReason})
474    return
475   } catch (err: any) {
476     const msg = err?.message || String(err)
477     if (/abort|ETIMEDOUT|ECONNRESET|ECONNREFUSED|FetchError/i.test(msg)) {
478       logStatus('transient API error, retrying')
479       await sleep(pollMs)
480       continue
481     }
482     throw err
483   }
484  }
485  const finalJob = await read2Resource({resource: newJob as KubeResource, cluster_name}) as any
486  const {succeeded} = finalJob?.status || {}
487  if (succeeded) { console.log(' done'); return }
488  throwDebugH({name, reason: 'too many retries'})
489}
490
491type KubeJobKanikoProps = {
492  name: string
493  dockerfileContent: string
494  tarSpecsA?: TarSpec[]
495  repo_name_tag: string
496  dfLabel: string
497  cluster_name?: string
498  dockreg_host?: string
499  dockLanHost?: string
500  action?: string
501  git_sha?: string
502}
503
504export const kubeJobKaniko = async ({name, dockerfileContent, tarSpecsA, repo_name_tag, dfLabel, cluster_name: cluster_nameArg, dockreg_host: dockreg_hostArg, dockLanHost: dockLanHostArg, action: actionArg, git_sha: git_shaArg}: KubeJobKanikoProps) => {
505  const ctx = getKlusterCtx()
506  const cluster_name = cluster_nameArg || ctx.cluster_name, dockreg_host = dockreg_hostArg || ctx.dockreg_host, dockLanHost = dockLanHostArg || ctx.dockLanHost, action = actionArg || getAction()
507  const {git_sha: git_shaCfg, git_repo_name, gitsshHost: gitsshHostCfg, gitsshRepoPath: gitsshRepoPathCfg, gitsshHttpsHostname: gitsshHttpsHostnameCfg} = getAppCfg()
508  const kanikoLocalContext = getAppCfg()?.kanikoLocalContext ?? true
509  const git_sha = git_shaArg || git_shaCfg
510  const gitsshHost = (isOcWeb && process.env.GITSSH_HOST) || gitsshHostCfg || process.env.GITSSH_HOST
511  const gitsshRepoPath = (isOcWeb && process.env.GITSSH_REPO) || gitsshRepoPathCfg || process.env.GITSSH_REPO
512  const gitsshHttpsHostname = gitsshHttpsHostnameCfg
513  assertDefined(git_sha)
514  assertDefined(cluster_name)
515  assertDefined(dockreg_host)
516  const useSsh = !!gitsshHost, useGitsshHttps = !!gitsshHttpsHostname
517  const df_sha = calcShortHash(dockerfileContent)
518  const isStandalone = !tarSpecsA?.length && !/^\s*(COPY|ADD)\b/im.test(dockerfileContent)
519  if (!isStandalone && !kanikoLocalContext) {
520    if (useGitsshHttps || useSsh) assertDefined(gitsshRepoPath)
521    else assertDefined(git_repo_name)
522  }
523
524  const kanikoStore = getKanikoCtx()
525  kanikoStore.cluster_name = cluster_name
526  kanikoStore.dockreg_host = dockreg_host
527  kanikoStore.git_sha = git_sha
528  const {appliedShared, gitChecked} = kanikoStore
529  const gitCheckKey = useGitsshHttps ? `https:${gitsshHttpsHostname}:${gitsshRepoPath}:${git_sha}` : useSsh ? `${gitsshHost}:${gitsshRepoPath}:${git_sha}` : `${git_repo_name}:${git_sha}`
530  if (!isStandalone && !kanikoLocalContext && !gitChecked.has(gitCheckKey)) {
531    if (useGitsshHttps) {
532      assertDefined(gitsshRepoPath)
533      const repoName = gitsshRepoPath.replace(/^\/home\/git\/repos\//, '').replace(/\.git$/, '')
534      const httpsToken = await genPlainSecretIfMissing({secretName: `${gitsshHost}-httptoken`})
535      const tokenMissingHint = `run gitssh apply to generate ${gitsshHost}-httptoken`
536      assertDefined(httpsToken, {gitsshHost, tokenMissingHint})
537      const httpsUrl = `https://git:${httpsToken}@${gitsshHttpsHostname}/git/${repoName}.git`
538      logElapsed(`commitexists ${git_sha} (on ${gitsshHttpsHostname})`)
539      let {stdout, isSuccess} = await liveSpawn({cmd: `git ls-remote ${httpsUrl}`, isQuiet: true, noOutCmd: true, timeoutAfterSec: 15})
540      if (!isSuccess || !String(stdout).includes(git_sha)) {
541        if (autoPushOnDeploy) {
542          logElapsed(`auto-pushing to ${gitsshHttpsHostname}`)
543          await liveSpawn({cmd: `git push ${httpsUrl} HEAD:main`, noOutCmd: true, timeoutAfterSec: 60})
544          ;({stdout, isSuccess} = await liveSpawn({cmd: `git ls-remote ${httpsUrl}`, isQuiet: true, noOutCmd: true, timeoutAfterSec: 15}))
545        }
546        const gitHttpsPushHint = 'git push to gitssh https'
547        assertTruthy(isSuccess && String(stdout).includes(git_sha), {git_sha, gitHttpsPushHint})
548      }
549    } else if (useSsh) {
550      const gitsshUrl = `ssh://git@${gitsshHost}${gitsshRepoPath}`
551      assertDefined(gitsshRepoPath)
552      const repoName = gitsshRepoPath.replace(/^\/home\/git\/repos\//, '').replace(/\.git$/, '')
553      const checkCmd = `ssh git@${gitsshHost} commitexists ${repoName} ${git_sha}`
554      logElapsed(`commitexists ${git_sha} (on ${gitsshHost})`)
555      let {isSuccess} = await liveSpawn({cmd: checkCmd, isQuiet: true, noOutCmd: true, timeoutAfterSec: 10})
556      if (!isSuccess) {
557        if (autoPushOnDeploy) {
558          logElapsed(`auto-pushing to ${gitsshHost}`)
559          await liveSpawn({cmd: `git push ${gitsshUrl} HEAD:main`, timeoutAfterSec: 30})
560          const {isSuccess: isSuccessRetry} = await liveSpawn({cmd: checkCmd, isQuiet: true, noOutCmd: true, timeoutAfterSec: 10})
561          if (isSuccessRetry) isSuccess = true
562        }
563        const gitPushHint = 'git push to gitssh'
564        assertTruthy(isSuccess, {git_sha, gitsshUrl, hint: gitPushHint})
565      }
566    } else {
567      const result = await liveRepoExec({cmd: `git branch -a --contains ${git_sha}`, git_repo_name})
568      const gitPushHint = 'git push so kaniko can pull'
569      assertTruthy(result.includes('remotes/origin'), {git_sha, hint: gitPushHint})
570    }
571    gitChecked.add(gitCheckKey)
572  }
573
574  if (kanikoLocalContext && tarSpecsA?.length) getKanikoCtx().localCtxTarPath = await genImportTreeCtxTar({tarSpecsA})
575  const kanikoProps = {name, dockerfileContent, tarSpecsA, df_sha, isStandalone, repo_name_tag, action, gitsshHost, gitsshRepoPath, gitsshHttpsHostname}
576  const {newJob, gitRemoteSecret, resources} = await getKanikoResources(kanikoProps)
577
578  const sharedKey = `${cluster_name}:${dockreg_host}`
579  const isSharedApplied = appliedShared.has(sharedKey)
580  const perBuildResources = resources.filter(r => r.kind === 'ConfigMap')
581  const sharedResources = resources.filter(r => r.kind !== 'ConfigMap')
582  if (action != 'delete') sharedResources.push(regcredSecTempl({dockreg_host, dockLanHost, cluster_name}))
583
584  await Promise.all(perBuildResources.map((resource) => resourcesActionCore({resources: [resource], action, cluster_name})))
585  if (!isSharedApplied) {
586    await Promise.all(sharedResources.map((resource) => resourcesActionCore({resources: [resource], action, cluster_name})))
587    appliedShared.add(sharedKey)
588  }
589
590  if (isApplyishAction(action)) {
591    if (!useSsh && gitRemoteSecret && !isSharedApplied) await objectApiCall({resource: gitRemoteSecret, objectApiMethod: 'create', cluster_name})
592    await kanikoRetryable({newJob, dfLabel, repo_name_tag})
593  } else {
594    await delete2Resource({resource: newJob as KubeResource, cluster_name})
595  }
596  if (isApplyishAction(action)) await checkKanikoStatus({newJob})
597}
598