1/**
2 * Integration test for multi-replica scaling with pod deletion.
3 * 
4 * Tests:
5 * 1. Two games run in parallel on potentially different pods
6 * 2. Pod deletion happens mid-test
7 * 3. Both games complete successfully despite disruption
8 */
9
10import { createLiveHarness } from './liveHarnessAI.mts'
11import { createAgentTeam, runAgentsUntilDone } from './agentsAI.mts'
12import { sleep } from '../../sharedF/sleepF.mts'
13import { testEmailDomain } from '../../sharedF/testEmailDomainAI.mts'
14import { chalkGray } from '../../serverF/libChalkF.mts'
15import { sql } from 'kysely'
16import { getGamesDb } from '../serverF/getGamesDbF.mts'
17import { eptKubeCli, noOutCmdCtx } from '../../deployF/k8sF/cliF/eptKubeCliF.mts'
18import { assertTruthy } from '../../sharedF/assertsF/assertTruthyAI.mts'
19import { throwDebugCtx } from '../../sharedF/throwErrorF/throwDebugCtxAI.mts'
20
21type ScalingCtxOpts = { cluster_name: string, name: string }
22type K8sPod = { metadata: { name: string }, status: { containerStatuses?: Array<{ ready: boolean }> } }
23
24// Helper context passed through to avoid module-level mutable state
25const createScalingCtx = ({ cluster_name, name }: ScalingCtxOpts) => {
26  const logVerbose = (msg: string) => { console.log(chalkGray(msg)) }
27  
28  const kubeCli = async (argsStr: string) => {
29    const args = argsStr.split(' ').filter(Boolean)
30    return noOutCmdCtx.run({}, () => eptKubeCli([cluster_name, ...args]))
31  }
32  
33  const deleteOnePod = async () => {
34    const result = await kubeCli(`get pods -l name=${name} -o jsonpath={.items[0].metadata.name}`)
35    const podName = result?.stdout?.trim()
36    assertTruthy(podName, {name, result})
37    logVerbose(`  Deleting pod: ${podName}`)
38    await kubeCli(`delete pod ${podName}`)
39    return podName
40  }
41  
42  const getReadyPods = async (): Promise<K8sPod[]> => {
43    const result = await kubeCli(`get pods -l name=${name} --field-selector=status.phase=Running -o json`)
44    const pods = JSON.parse(result?.stdout || '{"items":[]}') as { items: K8sPod[] }
45    return pods.items.filter((p: K8sPod) => p.status.containerStatuses?.every((c) => c.ready))
46  }
47  
48  const waitForPodCount = async (count: number, timeoutMs = 90_000) => {
49    const start = Date.now()
50    while (Date.now() - start < timeoutMs) {
51      const readyPods = await getReadyPods()
52      if (readyPods.length === count) {
53        return readyPods
54      }
55      await sleep(2000)
56    }
57    throwDebugCtx({expectedCount: count, timeoutMs})
58  }
59  
60  return { logVerbose, kubeCli, deleteOnePod, getReadyPods, waitForPodCount }
61}
62
63const getOrCreateTestOwner = async (suffix: string) => {
64  const db = getGamesDb()
65  const testEmail = `test-room-owner-${suffix}@${testEmailDomain}`
66  const existing = await db.selectFrom('users')
67    .where('email', '=', testEmail)
68    .select('id')
69    .executeTakeFirst()
70  if (existing) return existing.id
71  
72  const inserted = await db.insertInto('users')
73    .values({
74      email: testEmail, updated_at: sql`now()`,
75      data: {} as never, faucet_ips: {} as never,
76    })
77    .returning('id')
78    .executeTakeFirst()
79  return inserted!.id
80}
81
82const ensureTestRoomExists = async (roomLabel: string) => {
83  const db = getGamesDb()
84  const existing = await db.selectFrom('gamerooms')
85    .where('room_label', '=', roomLabel)
86    .select('id')
87    .executeTakeFirst()
88  if (existing) return
89  
90  // Use different owners for each room to avoid constraint violations
91  const ownerUserId = await getOrCreateTestOwner(roomLabel)
92  // Deactivate any existing active rooms for this test owner (unique constraint)
93  await db.updateTable('gamerooms')
94    .set({is_gameactive: false})
95    .where('owner_userid', '=', ownerUserId)
96    .where('is_gameactive', '=', true)
97    .execute()
98  await db.insertInto('gamerooms')
99    .values({room_label: roomLabel, owner_userid: ownerUserId, jdata: {}, updated_at: sql`now()`})
100    .execute()
101}
102
103type RunGameOpts = {
104  baseUrl: string
105  gameRoom: string
106  playerCount: number
107  onProgress?: (info: { gameRoom: string, round: number }) => void
108}
109
110// eslint-disable-next-line @typescript-eslint/no-explicit-any
111type AnyHarness = any
112
113const runGameToCompletion = async ({ baseUrl, gameRoom, playerCount, onProgress }: RunGameOpts) => {
114  await ensureTestRoomExists(gameRoom)
115  const harness: AnyHarness = await createLiveHarness({ baseUrl, playerCount, gameRoom })
116  const agents = createAgentTeam({ count: playerCount, seed: gameRoom, type: 'random' })
117  
118  const result = await runAgentsUntilDone({ 
119    harness, 
120    agents, 
121    maxRounds: 30, // 4-player game finishes in ~8-10 rounds
122    onRound: ({ round }: { round: number }) => onProgress?.({ gameRoom, round })
123  })
124  
125  await harness.cleanup()
126  return { gameRoom, ...result }
127}
128
129type ScalingIntegrationOpts = { baseUrl?: string, cluster_name?: string, name?: string }
130
131export const scalingIntegration = async ({ baseUrl, cluster_name, name }: ScalingIntegrationOpts = {}) => {
132  assertTruthy(baseUrl, {cluster_name, name})
133  assertTruthy(cluster_name, {cluster_name, name})
134  assertTruthy(name, {cluster_name, name})
135  
136  // Create context with helpers (avoids module-level mutable state)
137  const ctx = createScalingCtx({ cluster_name, name })
138  const { logVerbose, deleteOnePod, getReadyPods, waitForPodCount } = ctx
139  
140  const timestamp = Date.now()
141  const gameA = `scale-a-${timestamp}`
142  const gameB = `scale-b-${timestamp}`
143  const playerCount = 4
144  
145  // Check if we have enough replicas to run the test
146  const initialPods = await getReadyPods()
147  if (initialPods.length < 2) {
148    return { 
149      passed: true, 
150      skipped: true,
151      message: `Skipped: only ${initialPods.length} replica(s) running (need 2)` 
152    }
153  }
154  
155  logVerbose(`\n  Games: ${gameA}, ${gameB}`)
156  logVerbose(`  Players per game: ${playerCount}`)
157  
158  // Verify we have 2 replicas
159  const pods = await waitForPodCount(2)
160  logVerbose(`  Pods ready: ${pods?.map((p: K8sPod) => p.metadata.name).join(', ')}`)
161  
162  // Track progress for both games
163  const progress = { a: 0, b: 0 }
164  const updateProgress = ({ gameRoom, round }: { gameRoom: string, round: number }) => {
165    if (gameRoom.includes('-a-')) progress.a = round
166    else progress.b = round
167    process.stdout.write(`\r  Progress: A=${progress.a} B=${progress.b}`)
168  }
169  
170  // Schedule pod deletion after 5 seconds (should be mid-game for at least one)
171  const deletionPromise = (async () => {
172    await sleep(5000)
173    logVerbose('\n  Deleting one pod mid-test...')
174    const deleted = await deleteOnePod()
175    logVerbose(`  Waiting for replacement...`)
176    await waitForPodCount(2)
177    logVerbose(`  Replacement ready, continuing...`)
178    return deleted
179  })()
180  
181  // Run both games to completion in parallel
182  const [resultA, resultB, deletedPod] = await Promise.all([
183    runGameToCompletion({ baseUrl, gameRoom: gameA, playerCount, onProgress: updateProgress }),
184    runGameToCompletion({ baseUrl, gameRoom: gameB, playerCount, onProgress: updateProgress }),
185    deletionPromise,
186  ])
187  
188  logVerbose('')
189  logVerbose(`  Game A: ${resultA.isGameFin ? 'finished' : 'NOT finished'} in ${resultA.rounds} rounds`)
190  logVerbose(`  Game B: ${resultB.isGameFin ? 'finished' : 'NOT finished'} in ${resultB.rounds} rounds`)
191  logVerbose(`  Pod deleted: ${deletedPod}`)
192  
193  // Verify results
194  const errors: string[] = []
195  if (!resultA.isGameFin) errors.push(`Game A did not finish (rounds: ${resultA.rounds})`)
196  if (!resultB.isGameFin) errors.push(`Game B did not finish (rounds: ${resultB.rounds})`)
197  
198  if (errors.length > 0) {
199    return { passed: false, message: errors.join('; ') }
200  }
201  
202  return { 
203    passed: true, 
204    message: `Both games completed (A: ${resultA.rounds}, B: ${resultB.rounds} rounds) with pod deletion mid-test` 
205  }
206}
207