99 lines
3.2 KiB
JavaScript
99 lines
3.2 KiB
JavaScript
#!/usr/bin/env bun
|
|
// s3-sync.js — Ensure all objects in the source bucket exist in the dest bucket
|
|
// Usage: bun s3-sync.js
|
|
// Lists all objects in SOURCE and DEST, then copies only the objects that
|
|
// are missing from DEST. Direction is one-way: SOURCE -> DEST.
|
|
|
|
// ============================================================
|
|
// CREDENTIALS — replace with your actual values
|
|
// ============================================================
|
|
const SOURCE = {
|
|
accessKeyId: '8fab47503efb9547b50e4fb317e35cc7',
|
|
secretAccessKey: '47c2eb5636843cf568dda7ad0959a3e42071303f26dbdff94bd45a3c33dcd950',
|
|
region: 'auto', // 'us-east-1' for AWS, 'auto' for R2
|
|
endpoint: 'https://da9b1aa7c1951c23e2c0c3246ba68a58.r2.cloudflarestorage.com', // S3 or R2 endpoint
|
|
bucket: 'meatfarmer',
|
|
}
|
|
|
|
const DEST = {
|
|
accessKeyId: '8fab47503efb9547b50e4fb317e35cc7',
|
|
secretAccessKey: '47c2eb5636843cf568dda7ad0959a3e42071303f26dbdff94bd45a3c33dcd950',
|
|
region: 'auto',
|
|
endpoint: 'https://da9b1aa7c1951c23e2c0c3246ba68a58.r2.cloudflarestorage.com',
|
|
bucket: 'meatfarmer-dev',
|
|
}
|
|
// ============================================================
|
|
|
|
const CONCURRENCY = 20
|
|
|
|
if (!SOURCE.endpoint || !SOURCE.bucket || !DEST.endpoint || !DEST.bucket) {
|
|
console.error('Fill in the SOURCE and DEST credentials at the top of the script first.')
|
|
process.exit(1)
|
|
}
|
|
|
|
const source = new Bun.S3Client(SOURCE)
|
|
const dest = new Bun.S3Client(DEST)
|
|
|
|
async function listAllObjects(s3) {
|
|
const allKeys = []
|
|
let continuationToken
|
|
|
|
do {
|
|
const options = { maxKeys: 1000 }
|
|
if (continuationToken) options.continuationToken = continuationToken
|
|
|
|
const result = await s3.list(options)
|
|
for (const obj of result.contents) {
|
|
allKeys.push(obj.key)
|
|
}
|
|
continuationToken = result.nextContinuationToken
|
|
|
|
process.stdout.write(`\r Listed ${allKeys.length} objects...`)
|
|
} while (continuationToken)
|
|
|
|
console.log('')
|
|
return allKeys
|
|
}
|
|
|
|
// ============================================================
|
|
// MAIN
|
|
// ============================================================
|
|
try {
|
|
console.log(`🔁 S3 Sync — ${SOURCE.bucket} -> ${DEST.bucket}`)
|
|
|
|
console.log('\n📋 Listing SOURCE...')
|
|
const sourceKeys = await listAllObjects(source)
|
|
|
|
console.log('\n📋 Listing DEST...')
|
|
const destKeys = await listAllObjects(dest)
|
|
|
|
const destSet = new Set(destKeys)
|
|
const toCopy = sourceKeys.filter((key) => !destSet.has(key))
|
|
|
|
console.log(`\n📊 SOURCE objects: ${sourceKeys.length}`)
|
|
console.log(` DEST objects: ${destKeys.length}`)
|
|
console.log(` To copy: ${toCopy.length} (present in SOURCE, missing in DEST)`)
|
|
|
|
if (toCopy.length === 0) {
|
|
console.log('\n✅ DEST already has everything from SOURCE.')
|
|
process.exit(0)
|
|
}
|
|
|
|
console.log('\n🚚 Copying missing objects...')
|
|
let copied = 0
|
|
for (let i = 0; i < toCopy.length; i += CONCURRENCY) {
|
|
const batch = toCopy.slice(i, i + CONCURRENCY)
|
|
await Promise.all(
|
|
batch.map(async (key) => {
|
|
await dest.write(key, source.file(key))
|
|
copied++
|
|
})
|
|
)
|
|
process.stdout.write(`\r Copied ${copied}/${toCopy.length}...`)
|
|
}
|
|
|
|
console.log(`\n✅ Done — copied ${copied} objects to ${DEST.bucket}.`)
|
|
} catch (error) {
|
|
console.error(`\n❌ Error: ${error.message}`)
|
|
process.exit(1)
|
|
}
|