enh
This commit is contained in:
parent
0eab909eb1
commit
27c1515c86
9 changed files with 178 additions and 22 deletions
|
|
@ -19,7 +19,7 @@
|
|||
"@react-navigation/elements": "^2.3.8",
|
||||
"@react-navigation/material-top-tabs": "^7.4.11",
|
||||
"@react-navigation/native": "^7.1.6",
|
||||
"@tanstack/react-query": "^5.85.9",
|
||||
"@tanstack/react-query": "^5.100.0",
|
||||
"@trpc/client": "^11.6.0",
|
||||
"@trpc/react-query": "^11.6.0",
|
||||
"axios": "^1.11.0",
|
||||
|
|
|
|||
|
|
@ -3,8 +3,14 @@ set -euo pipefail
|
|||
|
||||
ROOT_DIR="$(cd "$(dirname "$0")/.." && pwd)"
|
||||
DUMP_FILE="$ROOT_DIR/dumps/latest.sql"
|
||||
WRANGLER_CONFIG="$ROOT_DIR/wrangler.dev.toml"
|
||||
DB_NAME="freshyo-dev"
|
||||
|
||||
if [ "${1:-}" = "--dev" ]; then
|
||||
WRANGLER_CONFIG="$ROOT_DIR/wrangler.dev.toml"
|
||||
DB_NAME="freshyo-backend-dev"
|
||||
else
|
||||
WRANGLER_CONFIG="$ROOT_DIR/wrangler.prod.toml"
|
||||
DB_NAME="freshyo-dev"
|
||||
fi
|
||||
|
||||
if [ ! -f "$DUMP_FILE" ]; then
|
||||
echo "Dump file not found: $DUMP_FILE"
|
||||
|
|
|
|||
|
|
@ -8,3 +8,51 @@
|
|||
wrangler d1 execute freshyo-dev \
|
||||
--config wrangler.dev.toml \
|
||||
--file ../../packages/db_helper_sqlite/drizzle/0002_sku_split.sql
|
||||
|
||||
# ============================================================================
|
||||
# Importing `wrangler d1 export` dumps locally (--local)
|
||||
# ============================================================================
|
||||
|
||||
## Why it can fail with `no such table: main.<table>`
|
||||
|
||||
`wrangler d1 export` writes tables in `sqlite_master` rowid order, which is the
|
||||
order tables were *historically created*. Any migration that **drops & re-creates**
|
||||
a table (e.g. the SKU split does `DROP TABLE product_info` + rename) pushes that
|
||||
parent table to the END of the dump.
|
||||
|
||||
So the dump can create + populate a child table (e.g. `product_skus`, which has
|
||||
`FOREIGN KEY (product_id) REFERENCES product_info(id)`) BEFORE its parent
|
||||
(`product_info`) exists. Loading into an empty DB then fails with:
|
||||
|
||||
no such table: main.product_info
|
||||
|
||||
## Why the usual fixes DON'T work
|
||||
|
||||
- `PRAGMA foreign_keys=OFF` — is a **no-op inside a transaction**, and
|
||||
`wrangler d1 execute --local` runs the whole file as ONE transaction.
|
||||
- `PRAGMA defer_foreign_keys=TRUE` (already at the top of every export) — only
|
||||
defers *row-level* FK checks, not the "parent table doesn't exist" error.
|
||||
|
||||
The only real fix is **ordering** the tables.
|
||||
|
||||
## How to check after every export
|
||||
|
||||
grep -nE '^CREATE TABLE|REFERENCES' dumps/<dump>.sql
|
||||
|
||||
For every `REFERENCES x(...)`, confirm the `CREATE TABLE "x"` line appears
|
||||
BEFORE the referencing table's `CREATE TABLE`.
|
||||
|
||||
## How to hand-fix
|
||||
|
||||
Cut the parent table's block (`CREATE TABLE ...` + all its `INSERT ...` lines)
|
||||
and paste it ABOVE the child table's block. Then verify it loads cleanly:
|
||||
|
||||
sqlite3 :memory: "PRAGMA foreign_keys=ON; BEGIN; .read dumps/<dump>.sql; COMMIT;"
|
||||
|
||||
This should exit 0 with no error. (Example already applied to `latest_1.sql`:
|
||||
`product_info` was moved above `product_skus`.)
|
||||
|
||||
## When to re-check
|
||||
|
||||
After ANY new `wrangler d1 export`, especially once a migration that re-creates
|
||||
tables has been applied. This is a general trap, not specific to one dump.
|
||||
|
|
|
|||
|
|
@ -8,10 +8,10 @@ routes = [
|
|||
|
||||
[[d1_databases]]
|
||||
binding = "DB"
|
||||
#database_name = "freshyo-backend-dev"
|
||||
#database_id = "6b93ddc9-9b24-4cfc-9320-e81aec38887a"
|
||||
database_name = "freshyo-dev"
|
||||
database_id = "3cad440f-dc14-4baa-ab05-04eb08e206de"
|
||||
database_name = "freshyo-backend-dev"
|
||||
database_id = "0814d709-5278-4311-8978-c36c0f05875d"
|
||||
#database_name = "freshyo-dev"
|
||||
#database_id = "3cad440f-dc14-4baa-ab05-04eb08e206de"
|
||||
migrations_dir="../../packages/db_helper_sqlite/drizzle"
|
||||
migrations_pattern="migration.sql"
|
||||
[durable_objects]
|
||||
|
|
|
|||
|
|
@ -14,6 +14,7 @@
|
|||
"autoIncrement": true
|
||||
},
|
||||
"dev": {
|
||||
"distribution": "internal",
|
||||
"channel": "dev",
|
||||
"autoIncrement": true
|
||||
},
|
||||
|
|
|
|||
|
|
@ -19,7 +19,7 @@
|
|||
"@react-navigation/drawer": "^7.3.9",
|
||||
"@react-navigation/elements": "^2.3.8",
|
||||
"@react-navigation/native": "^7.1.6",
|
||||
"@tanstack/react-query": "^5.85.9",
|
||||
"@tanstack/react-query": "^5.100.0",
|
||||
"@trpc/client": "^11.6.0",
|
||||
"@trpc/react-query": "^11.6.0",
|
||||
"axios": "^1.11.0",
|
||||
|
|
|
|||
|
|
@ -12,7 +12,7 @@
|
|||
"@react-navigation/drawer": "^7.5.8",
|
||||
"@react-navigation/elements": "^2.3.8",
|
||||
"@react-navigation/native": "^7.1.6",
|
||||
"@tanstack/react-query": "^5.85.9",
|
||||
"@tanstack/react-query": "^5.100.0",
|
||||
"axios": "^1.11.0",
|
||||
"buffer": "^6.0.3",
|
||||
"dayjs": "^1.11.18",
|
||||
|
|
|
|||
|
|
@ -7,11 +7,11 @@
|
|||
// ============================================================
|
||||
// CREDENTIALS — replace with your actual values
|
||||
// ============================================================
|
||||
const S3_ACCESS_KEY_ID = 'YOUR_ACCESS_KEY_ID'
|
||||
const S3_SECRET_ACCESS_KEY = 'YOUR_SECRET_ACCESS_KEY'
|
||||
const S3_REGION = 'auto' // 'us-east-1' for AWS, 'auto' for R2
|
||||
const S3_ENDPOINT = 'https://your-account.r2.cloudflarestorage.com' // S3 or R2 endpoint
|
||||
const S3_BUCKET = 'your-bucket-name'
|
||||
const S3_ACCESS_KEY_ID = '8fab47503efb9547b50e4fb317e35cc7'
|
||||
const S3_SECRET_ACCESS_KEY = '47c2eb5636843cf568dda7ad0959a3e42071303f26dbdff94bd45a3c33dcd950'
|
||||
const S3_REGION = 'apac' // 'us-east-1' for AWS, 'auto' for R2
|
||||
const S3_ENDPOINT = 'https://da9b1aa7c1951c23e2c0c3246ba68a58.r2.cloudflarestorage.com' // S3 or R2 endpoint
|
||||
const S3_BUCKET = 'meatfarmer'
|
||||
const API_CACHE_KEY = 'api-cache' // matches API_CACHE_KEY env var in backend
|
||||
|
||||
// ============================================================
|
||||
|
|
@ -62,14 +62,16 @@ async function listAllObjects(prefix) {
|
|||
|
||||
async function deleteObjects(keys) {
|
||||
let deleted = 0
|
||||
const batchSize = 1000
|
||||
const concurrency = 20
|
||||
|
||||
for (let i = 0; i < keys.length; i += batchSize) {
|
||||
const batch = keys.slice(i, i + batchSize)
|
||||
const objects = batch.map((key) => ({ key }))
|
||||
|
||||
const result = await s3.deleteObjects({ objects })
|
||||
deleted += result.deleted?.length ?? 0
|
||||
for (let i = 0; i < keys.length; i += concurrency) {
|
||||
const batch = keys.slice(i, i + concurrency)
|
||||
await Promise.all(
|
||||
batch.map(async (key) => {
|
||||
await s3.delete(key)
|
||||
deleted++
|
||||
})
|
||||
)
|
||||
|
||||
process.stdout.write(`\r Deleted ${deleted}/${keys.length}...`)
|
||||
}
|
||||
|
|
|
|||
99
scripts/s3-sync.js
Normal file
99
scripts/s3-sync.js
Normal file
|
|
@ -0,0 +1,99 @@
|
|||
#!/usr/bin/env bun
|
||||
// s3-sync.js — Ensure all objects in the source bucket exist in the dest bucket
|
||||
// Usage: bun s3-sync.js
|
||||
// Lists all objects in SOURCE and DEST, then copies only the objects that
|
||||
// are missing from DEST. Direction is one-way: SOURCE -> DEST.
|
||||
|
||||
// ============================================================
|
||||
// CREDENTIALS — replace with your actual values
|
||||
// ============================================================
|
||||
const SOURCE = {
|
||||
accessKeyId: '8fab47503efb9547b50e4fb317e35cc7',
|
||||
secretAccessKey: '47c2eb5636843cf568dda7ad0959a3e42071303f26dbdff94bd45a3c33dcd950',
|
||||
region: 'auto', // 'us-east-1' for AWS, 'auto' for R2
|
||||
endpoint: 'https://da9b1aa7c1951c23e2c0c3246ba68a58.r2.cloudflarestorage.com', // S3 or R2 endpoint
|
||||
bucket: 'meatfarmer',
|
||||
}
|
||||
|
||||
const DEST = {
|
||||
accessKeyId: '8fab47503efb9547b50e4fb317e35cc7',
|
||||
secretAccessKey: '47c2eb5636843cf568dda7ad0959a3e42071303f26dbdff94bd45a3c33dcd950',
|
||||
region: 'auto',
|
||||
endpoint: 'https://da9b1aa7c1951c23e2c0c3246ba68a58.r2.cloudflarestorage.com',
|
||||
bucket: 'meatfarmer-dev',
|
||||
}
|
||||
// ============================================================
|
||||
|
||||
const CONCURRENCY = 20
|
||||
|
||||
if (!SOURCE.endpoint || !SOURCE.bucket || !DEST.endpoint || !DEST.bucket) {
|
||||
console.error('Fill in the SOURCE and DEST credentials at the top of the script first.')
|
||||
process.exit(1)
|
||||
}
|
||||
|
||||
const source = new Bun.S3Client(SOURCE)
|
||||
const dest = new Bun.S3Client(DEST)
|
||||
|
||||
async function listAllObjects(s3) {
|
||||
const allKeys = []
|
||||
let continuationToken
|
||||
|
||||
do {
|
||||
const options = { maxKeys: 1000 }
|
||||
if (continuationToken) options.continuationToken = continuationToken
|
||||
|
||||
const result = await s3.list(options)
|
||||
for (const obj of result.contents) {
|
||||
allKeys.push(obj.key)
|
||||
}
|
||||
continuationToken = result.nextContinuationToken
|
||||
|
||||
process.stdout.write(`\r Listed ${allKeys.length} objects...`)
|
||||
} while (continuationToken)
|
||||
|
||||
console.log('')
|
||||
return allKeys
|
||||
}
|
||||
|
||||
// ============================================================
|
||||
// MAIN
|
||||
// ============================================================
|
||||
try {
|
||||
console.log(`🔁 S3 Sync — ${SOURCE.bucket} -> ${DEST.bucket}`)
|
||||
|
||||
console.log('\n📋 Listing SOURCE...')
|
||||
const sourceKeys = await listAllObjects(source)
|
||||
|
||||
console.log('\n📋 Listing DEST...')
|
||||
const destKeys = await listAllObjects(dest)
|
||||
|
||||
const destSet = new Set(destKeys)
|
||||
const toCopy = sourceKeys.filter((key) => !destSet.has(key))
|
||||
|
||||
console.log(`\n📊 SOURCE objects: ${sourceKeys.length}`)
|
||||
console.log(` DEST objects: ${destKeys.length}`)
|
||||
console.log(` To copy: ${toCopy.length} (present in SOURCE, missing in DEST)`)
|
||||
|
||||
if (toCopy.length === 0) {
|
||||
console.log('\n✅ DEST already has everything from SOURCE.')
|
||||
process.exit(0)
|
||||
}
|
||||
|
||||
console.log('\n🚚 Copying missing objects...')
|
||||
let copied = 0
|
||||
for (let i = 0; i < toCopy.length; i += CONCURRENCY) {
|
||||
const batch = toCopy.slice(i, i + CONCURRENCY)
|
||||
await Promise.all(
|
||||
batch.map(async (key) => {
|
||||
await dest.write(key, source.file(key))
|
||||
copied++
|
||||
})
|
||||
)
|
||||
process.stdout.write(`\r Copied ${copied}/${toCopy.length}...`)
|
||||
}
|
||||
|
||||
console.log(`\n✅ Done — copied ${copied} objects to ${DEST.bucket}.`)
|
||||
} catch (error) {
|
||||
console.error(`\n❌ Error: ${error.message}`)
|
||||
process.exit(1)
|
||||
}
|
||||
Loading…
Add table
Reference in a new issue