1import * as _ from 'lodash-es'
2import fs from 'fs'
3import { genericPvcTmplA } from '../k8sF/genericPvcF.mts'
4import { secretFileTemplate, secretTemplate, cleanupResOnDelete } from '../k8sF/k8sSecF.mts'
5import { jwtPath } from './jwtPathF.mts'
6import { getEth2Vals } from './getExCliResourcesF.mts'
7import { getKlusterCtx } from '../k8sF/ctxF/klusterCtxF.mts'
8import { getAction } from '../ctxF/actionCtxF.mts'
9import { kubeSvcTmpl } from '../k8sF/kubeSvcTmplF.mts'
10import { dualNodePortTmpl } from '../k8sF/dualNodePortTmplF.mts'
11import { guardOnlyRecreateDeploymentsResourcesAction } from '../k8sF/guardOnlyRecreateDeploymentsResourcesActionF.mts'
12import { getFixpermsContainer } from '../k8sF/getFixPermsContF.mts'
13import type { GuardValidatorFn } from './guardValidatorAI.mts'
14import { assertDefined } from '../../sharedF/assertsF/assertDefinedF.mts'
15import { assertTruthy } from '../../sharedF/assertsF/assertTruthyAI.mts'
16import { getPlainNoMappedSec, type secretNameType } from '../../serverF/secretsF/getPlainSecAI.mts'
17import { deployLabel } from '../k8sF/kubeLabelsF.mts'
18import { withClaimCheck } from './claimCheckContainerF.mts'
19
20const nimbusDeploymentTmpl = ({
21  name, dataPvcName, httpWeb3Prov, nimbusNodePort, execEngProv, ethNetwork, ethRewardAddr,
22  p2pHostIp, cpSyncApiUrl, trustedBlockRoot, trustedStateRoot, nimbusV, enableValidator,
23  validatorsSecName, secretsSecName, validatorPubkeys, podResources,
24}: {
25  name: string, dataPvcName: string, httpWeb3Prov: string, nimbusNodePort: number,
26  execEngProv: string, ethNetwork: string, ethRewardAddr: string,
27  p2pHostIp: string, cpSyncApiUrl: string | null, trustedBlockRoot: string | null,
28  trustedStateRoot: string | null, nimbusV: string, enableValidator: boolean,
29  validatorsSecName: string, secretsSecName: string, validatorPubkeys: string[],
30  podResources?: object,
31}) => {
32  const jwtSecretName = `${ethNetwork}-jwt`
33  const image = `statusim/nimbus-eth2:multiarch-${nimbusV}`
34
35  const validatorsDir = '/validators'
36  const secretsDir = '/secrets'
37  const keystoresSrcDir = '/keystores-src'
38  const secretsSrcDir = '/secrets-src'
39
40  let args = [
41    '--data-dir=/data',
42    `--network=${ethNetwork}`,
43    `--web3-url=${execEngProv}`,
44    `--jwt-secret=${jwtPath}`,
45    `--tcp-port=${nimbusNodePort}`,
46    `--udp-port=${nimbusNodePort}`,
47    `--nat=extip:${p2pHostIp}`,
48    '--rest',
49    '--rest-address=0.0.0.0',
50    '--rest-port=5052',
51    `--suggested-fee-recipient=${ethRewardAddr}`,
52    '--enr-auto-update=false',
53  ]
54  if (cpSyncApiUrl) {
55    args = [
56      ...args,
57      `--external-beacon-api-url=${cpSyncApiUrl}`,
58    ]
59  }
60  if (trustedBlockRoot) {
61    args = [
62      ...args,
63      `--trusted-block-root=${trustedBlockRoot}`,
64    ]
65  }
66  if (trustedStateRoot) {
67    args = [
68      ...args,
69      `--trusted-state-root=${trustedStateRoot}`,
70    ]
71  }
72  if (enableValidator) {
73    args = [
74      ...args,
75      `--validators-dir=${validatorsDir}`,
76      `--secrets-dir=${secretsDir}`,
77    ]
78  }
79
80  let volumeMounts = [
81    {
82      mountPath: '/data',
83      name: dataPvcName
84    },
85    {
86      mountPath: jwtPath,
87      name: jwtSecretName,
88      subPath: jwtSecretName,
89    }
90  ]
91
92  let volumes: Record<string, unknown>[] = [
93    {
94      name: dataPvcName,
95      persistentVolumeClaim: {
96        claimName: dataPvcName
97      }
98    },
99    {
100      name: jwtSecretName,
101      secret: {
102        secretName: jwtSecretName
103      }
104    }
105  ]
106
107  let initContainers: Record<string, unknown>[] = [
108    getFixpermsContainer({volumeMounts}),
109  ]
110
111  if (enableValidator) {
112    const validatorVolName = 'validator-data'
113    const secretsVolName = 'secrets-data'
114    
115    volumeMounts = [
116      ...volumeMounts,
117      {
118        name: validatorVolName,
119        mountPath: validatorsDir,
120      },
121      {
122        name: secretsVolName,
123        mountPath: secretsDir,
124      }
125    ]
126    volumes = [
127      ...volumes,
128      {
129        name: validatorsSecName,
130        secret: {
131          secretName: validatorsSecName
132        }
133      },
134      {
135        name: secretsSecName,
136        secret: {
137          secretName: secretsSecName
138        }
139      },
140      {
141        name: validatorVolName,
142        emptyDir: {}
143      },
144      {
145        name: secretsVolName,
146        emptyDir: {}
147      }
148    ]
149    
150    // Build shell script to organize keystores into Nimbus directory structure
151    const setupCmds = validatorPubkeys.map(pubkey => [
152      `mkdir -p ${validatorsDir}/0x${pubkey}`,
153      `cp ${keystoresSrcDir}/keystore-${pubkey}.json ${validatorsDir}/0x${pubkey}/keystore.json`,
154      `cp ${secretsSrcDir}/secret-${pubkey} ${secretsDir}/0x${pubkey}`,
155    ].join(' && ')).join(' && ')
156    
157    initContainers = [
158      ...initContainers,
159      {
160        name: 'setup-validators',
161        image: 'busybox:1.36',
162        command: ['sh', '-c', setupCmds],
163        volumeMounts: [
164          {name: validatorsSecName, mountPath: keystoresSrcDir},
165          {name: secretsSecName, mountPath: secretsSrcDir},
166          {name: validatorVolName, mountPath: validatorsDir},
167          {name: secretsVolName, mountPath: secretsDir},
168        ]
169      }
170    ]
171  }
172
173  return {
174    apiVersion: 'apps/v1',
175    kind: 'Deployment',
176    metadata: {
177      name
178    },
179    spec: {
180      replicas: 1,
181      strategy: {
182        type: 'Recreate'
183      },
184      selector: {
185        matchLabels: {
186          name
187        }
188      },
189      template: {
190        metadata: {
191          labels: {
192            name,
193            [deployLabel]: name,
194          }
195        },
196        spec: {
197          securityContext: {
198            runAsUser: 1000,
199            fsGroup: 1000,
200          },
201          initContainers,
202          containers: [
203            {
204              args, image, command: [],
205              name,
206              volumeMounts
207            }
208          ],
209          volumes
210        }
211      }
212    }
213  }
214}
215
216export const doSyncNimbus = async ({
217  eth1ClientName, ethNetwork, ethRewardAddr, p2pHostIp,
218  nimbusNodePort, nimbusV, cpSyncApiUrl, trustedBlockRoot, trustedStateRoot,
219  enableValidator, guardValidatorFn, localValSecretsPath, ethValidatorNums,
220}: {
221  eth1ClientName: string, ethNetwork: string, ethRewardAddr: string, p2pHostIp: string,
222  nimbusNodePort: number, nimbusV: string, cpSyncApiUrl: string | null,
223  trustedBlockRoot?: string | null, trustedStateRoot?: string | null,
224  enableValidator: boolean, guardValidatorFn?: GuardValidatorFn, localValSecretsPath?: string, ethValidatorNums?: number[],
225}) => {
226  const {cluster_name, cfApiKeySecretName, domainNames} = getKlusterCtx()
227  const action = getAction()
228  assertDefined(p2pHostIp)
229
230  const name = `nimbus-${ethNetwork}`
231  const dataPvcName = name
232
233  // Note: trustedBlockRoot requires LC bootstrap support which most checkpoint servers don't have.
234  // If trustedBlockRoot is not provided, Nimbus will sync from cpSyncApiUrl without LC validation.
235
236  let {httpWeb3Prov, execEngProv, jwtSecretName} = getEth2Vals({ethNetwork, eth1ClientName})
237
238  const validatorsSecName = `${name}-validators`
239  const secretsSecName = `${name}-secrets`
240
241  let validatorPubkeys: string[] = []
242  const validatorsSecretsH: Record<string, string> = {}
243  const secretsSecretsH: Record<string, string> = {}
244
245  if (enableValidator === true) {
246    assertDefined(guardValidatorFn, {enableValidator})
247    await guardValidatorFn({ethNetwork, name, cluster_name, action})
248
249    const {walletPassSecName} = getEth2Vals({ethNetwork, eth1ClientName})
250    
251    if (action != 'delete') {
252      assertDefined(localValSecretsPath, {enableValidator})
253      const walletPassS = getPlainNoMappedSec(walletPassSecName as secretNameType)
254      const keystoreFiles = _.filter(fs.readdirSync(localValSecretsPath), (path) => {
255        return _.endsWith(path, '.json') && _.startsWith(path, 'keystore-')
256      })
257      
258      _.each(keystoreFiles, (filename) => {
259        const keystoreVal = fs.readFileSync([localValSecretsPath, filename].join('/'), 'utf8')
260        const keystoreJson = JSON.parse(keystoreVal)
261        const pubkey = keystoreJson.pubkey
262        assertTruthy(pubkey && pubkey.length === 96, {filename, pubkeyLen: pubkey?.length})
263        validatorPubkeys.push(pubkey)
264        
265        validatorsSecretsH[`keystore-${pubkey}.json`] = keystoreVal
266        secretsSecretsH[`secret-${pubkey}`] = walletPassS
267      })
268    }
269  }
270
271  const nimbusDeployment = nimbusDeploymentTmpl({
272    name, dataPvcName, httpWeb3Prov, nimbusNodePort,
273    nimbusV, execEngProv, ethNetwork, ethRewardAddr,
274    p2pHostIp, cpSyncApiUrl, trustedBlockRoot: trustedBlockRoot ?? null, trustedStateRoot: trustedStateRoot ?? null,
275    enableValidator, validatorsSecName, secretsSecName, validatorPubkeys,
276  })
277  const claimCheckConfigMap = await withClaimCheck({
278    deployment: nimbusDeployment, enabled: enableValidator === true && action != 'delete',
279    ethValidatorNum: ethValidatorNums?.[0], cluster_name, domainName: domainNames?.[0], cfApiKeySecretName,
280  })
281
282  let resources = [
283    ...(claimCheckConfigMap ? [claimCheckConfigMap] : []),
284    dualNodePortTmpl({selfName: name + '-p2p', name, nodePort: nimbusNodePort}),
285    kubeSvcTmpl({name, portNo: 5052}),
286    nimbusDeployment,
287    secretFileTemplate({name: jwtSecretName as any, kubeName: jwtSecretName}),
288    ...await genericPvcTmplA({
289      name: dataPvcName,
290      sizeGb: 400,
291      useM2: true, action
292    }),
293  ]
294
295  if (enableValidator === true) {
296    resources = _.union(resources, [
297      secretTemplate({name: validatorsSecName, secretsH: validatorsSecretsH}),
298      secretTemplate({name: secretsSecName, secretsH: secretsSecretsH}),
299    ])
300  }
301
302  resources = _.union(resources, cleanupResOnDelete({action, enabled: enableValidator, secNames: [validatorsSecName, secretsSecName]}))
303
304  await guardOnlyRecreateDeploymentsResourcesAction({resources, action, cluster_name})
305}
306