1import { create } from 'kubo-rpc-client'
2import fs from 'fs/promises'
3import path from 'path'
4import * as _ from 'lodash-es'
5import { betLog } from '../../../sharedF/betterConsLogF.mts'
6import { runMemoTempfile } from '../../../serverF/ptDiskF/memoTmpfileF.mts'
7import { calcHash } from '../../../serverF/calcHashF.mts'
8import { humanBytes } from '../../../sharedF/strF/humanBytesF.mts'
9import { chainPromiseCalls } from '../../../sharedF/moDashF/promF/moPromisesF.mts'
10import { getPlainNoMappedSec, type secretNameType } from '../../../serverF/secretsF/getPlainSecAI.mts'
11
12const chunkerType = 'size-1048576'
13
14const getFilesRecursive = async (dir: string): Promise<string[]> => {
15  const entries = await fs.readdir(dir, { withFileTypes: true })
16  return _.flatten(await Promise.all(entries.map(async (entry) => {
17    const fullPath = path.join(dir, entry.name)
18    return entry.isDirectory() ? getFilesRecursive(fullPath) : fullPath
19  })))
20}
21
22const addWithProgress = async ({filebase, content, fileSize, prefix, label, onlyHash = false}: {filebase: any, content: Buffer, fileSize: number, prefix: string, label: string, onlyHash?: boolean}) => {
23  return await filebase.add(content, {
24    chunker: chunkerType,
25    onlyHash,
26    progress: (bytes: number) => {
27      const percent = ((bytes / fileSize) * 100).toFixed(0)
28      const progressText = `\r${prefix}: ${label} ${percent}%`
29      process.stdout.write(progressText.padEnd(80))
30    }
31  })
32}
33
34const uploadFiles = async ({apiUrl, gatewayUrl, inputPath}: {apiUrl: string, gatewayUrl: string, inputPath: string}) => {
35  const filebaseToken = apiUrl.includes('filebase') ? getPlainNoMappedSec('ipfs-filebase' as secretNameType) : null
36  const clientConfig = filebaseToken
37    ? { url: apiUrl, headers: { Authorization: `Bearer ${filebaseToken}` } }
38    : { url: apiUrl }
39  
40  const filebase = create(clientConfig)
41  
42  const stats = await fs.stat(inputPath)
43  const allFiles = stats.isDirectory() ? await getFilesRecursive(inputPath) : [inputPath]
44
45  const files = _.filter(allFiles, filePath => path.basename(filePath) !== '.DS_Store')
46
47  const results: {path: string, hash: string}[] = []
48  for (let index = 0; index < files.length; index++) {
49    const filePath = files[index]
50    const basePath = path.basename(filePath)
51    const content = await fs.readFile(filePath)
52    const fileSize = content.length
53    const prefix = `[${index + 1}/${files.length}] ${basePath} (${humanBytes(fileSize)})`
54
55    const contentHash = calcHash(content.toString(), 'sha256')
56    const cachedCid = await runMemoTempfile({
57      cacheKeyA: ['ipfs1UploadedCid', chunkerType, contentHash]
58    }, async () => {
59      process.stdout.write(`\r${prefix}: uploading...`.padEnd(80))
60      const { cid } = await addWithProgress({filebase, content, fileSize, prefix, label: 'uploading'})
61      console.log(`\r${prefix}: uploaded ✓`.padEnd(80))
62      return cid.toString()
63    })
64
65    if (cachedCid) {
66      console.log(`\r${prefix}: exists ✓`.padEnd(80))
67    }
68
69    results.push({ path: filePath, hash: cachedCid })
70  }
71
72  betLog({ 
73    files: _.fromPairs(results.map((r) => {
74      return [r.path, `${gatewayUrl}/${r.hash}`]
75    })),
76  })
77}
78
79import { assertDefined } from '../../../sharedF/assertsF/assertDefinedF.mts'
80
81export const ipfsUpload = async ({apiUrl, gatewayUrl, inputPath}: {apiUrl: string, gatewayUrl: string, inputPath: string}) => {
82  assertDefined(inputPath)
83  await uploadFiles({apiUrl, gatewayUrl, inputPath})
84}
85