1import fs from 'fs/promises' 2import { existsSync } from 'fs' 12export const ossReviewModelDefault = 'anthropic/claude-haiku-4-5' 14export const ossOpsecReviewPrompt = `You are performing an OCD opsec review of an open-source software export before it is published. Scrutinize the provided source files for ANY of the following that could link the creator or maintainer of this repository to existing websites, services, or real-world identity on the public internet: 15- personally identifiable information (names, emails, phone numbers, physical addresses) 16- hostnames, domain names, subdomains (especially internal/private ones) 17- IP addresses (public or private LAN ranges) 18- wallet addresses, payment identifiers 19- MAC addresses, hardware identifiers 20- API keys, tokens, secrets, passwords, private keys 21- internal tool names, project codenames, internal abbreviations 22- cloud-provider account IDs, cluster names, node names 23For each finding, output exactly one line in the form: <relative-file-path>:<line-number>: <what is sensitive and why it could deanonymize the author> 24If absolutely nothing sensitive is found, output exactly: CLEAN 25Do not output headings, explanations, markdown, or any other text.` 33 chunkCharBudget?: number, 34 maxFileChars?: number, 37type ChunkSpec = { i: number, rels: string[], chars: number } 38type SkippedFile = { rel: string, chars: number } 40const reviewRootDir = () => path.join(ptDiskDir, 'ossreview') 41const chunkCacheDir = () => path.join(reviewRootDir(), 'chunkcache') 43const getOssReviewApiKey = async () => { 44 const secName = await ensureOpnRtrKey({name: 'ossreview', limit: 10, limitReset: 'daily'}) 48const buildChunkPlan = async (dir: string, chunkCharBudget: number, maxFileChars: number) => { 50 .filter(f => !f.includes(`${path.sep}.git${path.sep}`) && !f.endsWith(`${path.sep}.git`)) 53 const chunks: {rels: string[], content: string, fileHashes: [string, string][]}[] = [] 54 const skippedOversized: SkippedFile[] = [] 55 let cur = {rels: [] as string[], content: '', fileHashes: [] as [string, string][]} 56 for (const abs of absFiles) { 57 const rel = path.relative(dir, abs) 58 const content = await fs.readFile(abs, 'utf8').catch(() => '') 59 const segment = `=== FILE: ${rel} ===\n${content}\n` 60 if (content.length > maxFileChars) { 61 skippedOversized.push({rel, chars: content.length}) 64 if (cur.content.length + segment.length > chunkCharBudget && cur.rels.length) { 66 cur = {rels: [], content: '', fileHashes: []} 69 cur.content += segment 72 if (cur.rels.length) chunks.push(cur) 73 return {chunks, skippedOversized} 76export const llmReviewDir = async ({dir, shortSha, reviewName, promptStr = ossOpsecReviewPrompt, model = ossReviewModelDefault, chunkCharBudget = 100_000, maxFileChars = 200_000}: ReviewOpts) => { 77 const reviewDir = path.join(reviewRootDir(), `${reviewName}-${shortSha}`) 78 await fs.mkdir(reviewDir, {recursive: true}) 79 await fs.mkdir(chunkCacheDir(), {recursive: true}) 80 const summaryPath = path.join(reviewDir, 'summary.json') 81 if (existsSync(summaryPath)) { 82 console.log(`llmReview: cached ${reviewName}-${shortSha} (${reviewDir})`) 83 return {reviewDir, reviewMdPath: path.join(reviewDir, 'review.md'), summary: JSON.parse(await fs.readFile(summaryPath, 'utf8')), cached: true} 86 const apiKey = await getOssReviewApiKey() 88 const {chunks, skippedOversized} = await buildChunkPlan(dir, chunkCharBudget, maxFileChars) 89 const totChars = chunks.reduce((a, c) => a + c.content.length, 0) 90 const planPath = path.join(reviewDir, 'plan.json') 91 await fs.writeFile(planPath, JSON.stringify({model, reviewName, shortSha, chunkCharBudget, maxFileChars, totChars, chunks: chunks.map((c, i): ChunkSpec => ({i, rels: c.rels, chars: c.content.length})), skippedOversized}, null, 2)) 92 if (skippedOversized.length) console.log(`llmReview: skipping ${skippedOversized.length} oversized file(s) (>${maxFileChars.toLocaleString()} chars): ${skippedOversized.map(s => s.rel).join(', ')}`) 94 console.log(`llmReview: ${reviewName}-${shortSha} | ${chunks.length} chunk(s) | ${totChars.toLocaleString()} chars | model ${model}`) 95 const findingsA: string[] = [] 96 const usageA: {prompt_tokens: number, completion_tokens: number, costUsd: number | null}[] = [] 97 const allUsageA: {prompt_tokens: number, completion_tokens: number, costUsd: number | null}[] = [] 98 let cachedChunkCount = 0, failedChunkCount = 0, consecutiveFails = 0 99 let abortedEarly = false, firstError: any = null 101 for (let i = 0; i < chunks.length; i++) { 102 const chunkPath = path.join(reviewDir, `chunk-${i}.json`) 103 let chunkRes: any, freshCall = false 104 if (existsSync(chunkPath)) { 105 chunkRes = JSON.parse(await fs.readFile(chunkPath, 'utf8')) 107 const chunkCacheKey = calcHash(`${model}:${promptHash}:${JSON.stringify(chunks[i].fileHashes)}`) 108 const cachePath = path.join(chunkCacheDir(), `${chunkCacheKey}.json`) 109 if (existsSync(cachePath)) { 110 chunkRes = JSON.parse(await fs.readFile(cachePath, 'utf8')) 112 process.stdout.write(`llmReview: chunk ${i + 1}/${chunks.length} (${chunks[i].rels.length} files) ... cached\n`) 116 process.stdout.write(`llmReview: chunk ${i + 1}/${chunks.length} (${chunks[i].rels.length} files, ${chunks[i].content.length.toLocaleString()} chars) ... `) 118 const fullPrompt = `${promptStr}\n\n--- SOURCE FILES (chunk ${i + 1} of ${chunks.length}) ---\n${chunks[i].content}` 119 const respH: any = await openRouterPrompt({promptStr: fullPrompt, model, apiKey, max_tokens: 4096}) 120 const findings = (respH.choices?.[0]?.message?.content || '').trim() 121 chunkRes = {findings, usage: {prompt_tokens: respH.usage?.prompt_tokens, completion_tokens: respH.usage?.completion_tokens, costUsd: respH.costUsd}} 123 console.log(findings === 'CLEAN' || !findings ? 'clean' : `${findings.split('\n').length} finding(s)`) 124 await fs.writeFile(cachePath, JSON.stringify(chunkRes, null, 2)) 126 const dbg = err?.uniqDebugH?.error 127 const errorCode = dbg?.code ?? (err?.status ? String(err.status) : undefined) 128 const errorDetail = dbg?.message || String(err?.message || err) 129 chunkRes = {error: errorDetail, errorCode, findings: ''} 130 if (!firstError) firstError = {code: errorCode, detail: errorDetail} 133 console.log(`FAILED: ${errorCode || ''} ${errorDetail}`) 134 if (errorCode === '402') { abortedEarly = true; console.log('llmReview: credits exhausted — aborting'); await fs.writeFile(chunkPath, JSON.stringify(chunkRes, null, 2)); break } 135 if (consecutiveFails >= 3) { abortedEarly = true; console.log(`llmReview: ${consecutiveFails} consecutive failures — aborting`); await fs.writeFile(chunkPath, JSON.stringify(chunkRes, null, 2)); break } 138 if (!existsSync(chunkPath)) await fs.writeFile(chunkPath, JSON.stringify(chunkRes, null, 2)) 140 if (chunkRes.findings && chunkRes.findings !== 'CLEAN') findingsA.push(chunkRes.findings) 141 if (chunkRes.usage) { 142 allUsageA.push(chunkRes.usage) 143 if (freshCall) usageA.push(chunkRes.usage) 145 if (chunkRes.error) { failedChunkCount++; if (!firstError) firstError = {code: chunkRes.errorCode, detail: chunkRes.error} } 148 const totPromptTokens = usageA.reduce((a, u) => a + (u.prompt_tokens || 0), 0) 149 const totCompletionTokens = usageA.reduce((a, u) => a + (u.completion_tokens || 0), 0) 150 const totCostUsd = usageA.reduce((a, u) => a + (u.costUsd || 0), 0) 151 const fullTotCostUsd = allUsageA.reduce((a, u) => a + (u.costUsd || 0), 0) 152 const chunksWithFindings = findingsA.length 153 const summary = {reviewName, shortSha, model, chunkCount: chunks.length, totChars, totPromptTokens, totCompletionTokens, totCostUsd, fullTotCostUsd, chunksWithFindings, cachedChunkCount, failedChunkCount, abortedEarly, firstError, skippedOversizedCount: skippedOversized.length, skippedOversized, ts: new Date().toISOString()} 154 await fs.writeFile(summaryPath, JSON.stringify(summary, null, 2)) 155 const reviewMdPath = path.join(reviewDir, 'review.md') 156 const skippedNote = skippedOversized.length ? `\n\n## skipped oversized files (>${maxFileChars.toLocaleString()} chars, not reviewed)\n${skippedOversized.map(s => `- ${s.rel} (${s.chars.toLocaleString()} chars)`).join('\n')}\n` : '' 157 await fs.writeFile(reviewMdPath, (findingsA.length ? findingsA.join('\n\n---\n\n') : 'CLEAN - no findings\n') + skippedNote) 158 const status = abortedEarly ? 'ABORTED' : `${chunksWithFindings}/${chunks.length} chunk(s) w/ findings` 159 console.log(`llmReview: done ${reviewName}-${shortSha} | ${status} | ${cachedChunkCount} cached | $${totCostUsd.toFixed(4)} marginal${fullTotCostUsd > totCostUsd ? ` ($${fullTotCostUsd.toFixed(4)} if all fresh)` : ''} | ${reviewDir}`) 160 return {reviewDir, reviewMdPath, summary, cached: false} 163export type OssAnchor = { 167 tgtShaPrefix: string, 170 reviewTotCostUsd?: number, 171 reviewChunksWithFindings?: number, 175export const appendOssAnchor = async (anchor: OssAnchor) => { 176 await fs.mkdir(reviewRootDir(), {recursive: true}) 177 const anchorsPath = path.join(reviewRootDir(), 'anchors.jsonl') 178 await fs.appendFile(anchorsPath, JSON.stringify(anchor) + '\n') 181export const getOssReviewSummary = async (reviewName: string, shortSha: string) => { 182 const summaryPath = path.join(reviewRootDir(), `${reviewName}-${shortSha}`, 'summary.json') 183 const content = await fs.readFile(summaryPath, 'utf8').catch(() => null) 184 return content ? JSON.parse(content) : undefined 187export const getLastOssAnchor = async (name: string): Promise<OssAnchor | undefined> => { 188 const anchorsPath = path.join(reviewRootDir(), 'anchors.jsonl') 189 const content = await fs.readFile(anchorsPath, 'utf8').catch(() => '') 190 const entries = content.split('\n').filter(Boolean) 191 for (let i = entries.length - 1; i >= 0; i--) { 193 const anchor = JSON.parse(entries[i]) as OssAnchor 194 if (anchor.name === name) return anchor 200export const assertOssBakeable = async ({name, wipCfgVar}: {name: string, wipCfgVar: string}) => { 201 const existing = await getLastOssAnchor(name) 202 if (!existing) return 203 throw new Error(`${name} already baked (srcHead ${existing.srcHeadSha}, repoHead ${existing.repoHeadSha.slice(0, 12)}, ${existing.ts}). Release names are immutable — bump ${wipCfgVar} in pt0/sharedF/ossRepoConfigF.mts to def a new WIP.`)