From 27c1515c86c77c7f61c9738e790bb98859498a6b Mon Sep 17 00:00:00 2001 From: shafi54 <108669266+shafi-aviz@users.noreply.github.com> Date: Mon, 3 Aug 2026 20:39:40 +0530 Subject: [PATCH] enh --- apps/admin-ui/package.json | 2 +- apps/backend/scripts/populate_localdb.sh | 10 ++- apps/backend/wrangler-commands.md | 50 +++++++++++- apps/backend/wrangler.dev.toml | 8 +- apps/user-ui/eas.json | 1 + apps/user-ui/package.json | 2 +- packages/ui/package.json | 2 +- scripts/s3-cleaner.js | 26 ++++--- scripts/s3-sync.js | 99 ++++++++++++++++++++++++ 9 files changed, 178 insertions(+), 22 deletions(-) create mode 100644 scripts/s3-sync.js diff --git a/apps/admin-ui/package.json b/apps/admin-ui/package.json index e1691b7..a991b79 100644 --- a/apps/admin-ui/package.json +++ b/apps/admin-ui/package.json @@ -19,7 +19,7 @@ "@react-navigation/elements": "^2.3.8", "@react-navigation/material-top-tabs": "^7.4.11", "@react-navigation/native": "^7.1.6", - "@tanstack/react-query": "^5.85.9", + "@tanstack/react-query": "^5.100.0", "@trpc/client": "^11.6.0", "@trpc/react-query": "^11.6.0", "axios": "^1.11.0", diff --git a/apps/backend/scripts/populate_localdb.sh b/apps/backend/scripts/populate_localdb.sh index 101c70b..e0c2761 100755 --- a/apps/backend/scripts/populate_localdb.sh +++ b/apps/backend/scripts/populate_localdb.sh @@ -3,8 +3,14 @@ set -euo pipefail ROOT_DIR="$(cd "$(dirname "$0")/.." && pwd)" DUMP_FILE="$ROOT_DIR/dumps/latest.sql" -WRANGLER_CONFIG="$ROOT_DIR/wrangler.dev.toml" -DB_NAME="freshyo-dev" + +if [ "${1:-}" = "--dev" ]; then + WRANGLER_CONFIG="$ROOT_DIR/wrangler.dev.toml" + DB_NAME="freshyo-backend-dev" +else + WRANGLER_CONFIG="$ROOT_DIR/wrangler.prod.toml" + DB_NAME="freshyo-dev" +fi if [ ! -f "$DUMP_FILE" ]; then echo "Dump file not found: $DUMP_FILE" diff --git a/apps/backend/wrangler-commands.md b/apps/backend/wrangler-commands.md index 40425d1..5844c56 100644 --- a/apps/backend/wrangler-commands.md +++ b/apps/backend/wrangler-commands.md @@ -7,4 +7,52 @@ wrangler d1 execute freshyo-dev \ --config wrangler.dev.toml \ - --file ../../packages/db_helper_sqlite/drizzle/0002_sku_split.sql \ No newline at end of file + --file ../../packages/db_helper_sqlite/drizzle/0002_sku_split.sql + +# ============================================================================ +# Importing `wrangler d1 export` dumps locally (--local) +# ============================================================================ + +## Why it can fail with `no such table: main.` + +`wrangler d1 export` writes tables in `sqlite_master` rowid order, which is the +order tables were *historically created*. Any migration that **drops & re-creates** +a table (e.g. the SKU split does `DROP TABLE product_info` + rename) pushes that +parent table to the END of the dump. + +So the dump can create + populate a child table (e.g. `product_skus`, which has +`FOREIGN KEY (product_id) REFERENCES product_info(id)`) BEFORE its parent +(`product_info`) exists. Loading into an empty DB then fails with: + + no such table: main.product_info + +## Why the usual fixes DON'T work + +- `PRAGMA foreign_keys=OFF` — is a **no-op inside a transaction**, and + `wrangler d1 execute --local` runs the whole file as ONE transaction. +- `PRAGMA defer_foreign_keys=TRUE` (already at the top of every export) — only + defers *row-level* FK checks, not the "parent table doesn't exist" error. + +The only real fix is **ordering** the tables. + +## How to check after every export + + grep -nE '^CREATE TABLE|REFERENCES' dumps/.sql + +For every `REFERENCES x(...)`, confirm the `CREATE TABLE "x"` line appears +BEFORE the referencing table's `CREATE TABLE`. + +## How to hand-fix + +Cut the parent table's block (`CREATE TABLE ...` + all its `INSERT ...` lines) +and paste it ABOVE the child table's block. Then verify it loads cleanly: + + sqlite3 :memory: "PRAGMA foreign_keys=ON; BEGIN; .read dumps/.sql; COMMIT;" + +This should exit 0 with no error. (Example already applied to `latest_1.sql`: +`product_info` was moved above `product_skus`.) + +## When to re-check + +After ANY new `wrangler d1 export`, especially once a migration that re-creates +tables has been applied. This is a general trap, not specific to one dump. diff --git a/apps/backend/wrangler.dev.toml b/apps/backend/wrangler.dev.toml index 99b12c3..bd1f08b 100644 --- a/apps/backend/wrangler.dev.toml +++ b/apps/backend/wrangler.dev.toml @@ -8,10 +8,10 @@ routes = [ [[d1_databases]] binding = "DB" -#database_name = "freshyo-backend-dev" -#database_id = "6b93ddc9-9b24-4cfc-9320-e81aec38887a" -database_name = "freshyo-dev" -database_id = "3cad440f-dc14-4baa-ab05-04eb08e206de" +database_name = "freshyo-backend-dev" +database_id = "0814d709-5278-4311-8978-c36c0f05875d" +#database_name = "freshyo-dev" +#database_id = "3cad440f-dc14-4baa-ab05-04eb08e206de" migrations_dir="../../packages/db_helper_sqlite/drizzle" migrations_pattern="migration.sql" [durable_objects] diff --git a/apps/user-ui/eas.json b/apps/user-ui/eas.json index 5cbedc9..77a244f 100755 --- a/apps/user-ui/eas.json +++ b/apps/user-ui/eas.json @@ -14,6 +14,7 @@ "autoIncrement": true }, "dev": { + "distribution": "internal", "channel": "dev", "autoIncrement": true }, diff --git a/apps/user-ui/package.json b/apps/user-ui/package.json index b28bfb2..a809d30 100644 --- a/apps/user-ui/package.json +++ b/apps/user-ui/package.json @@ -19,7 +19,7 @@ "@react-navigation/drawer": "^7.3.9", "@react-navigation/elements": "^2.3.8", "@react-navigation/native": "^7.1.6", - "@tanstack/react-query": "^5.85.9", + "@tanstack/react-query": "^5.100.0", "@trpc/client": "^11.6.0", "@trpc/react-query": "^11.6.0", "axios": "^1.11.0", diff --git a/packages/ui/package.json b/packages/ui/package.json index 3d5e06f..9aed9e2 100755 --- a/packages/ui/package.json +++ b/packages/ui/package.json @@ -12,7 +12,7 @@ "@react-navigation/drawer": "^7.5.8", "@react-navigation/elements": "^2.3.8", "@react-navigation/native": "^7.1.6", - "@tanstack/react-query": "^5.85.9", + "@tanstack/react-query": "^5.100.0", "axios": "^1.11.0", "buffer": "^6.0.3", "dayjs": "^1.11.18", diff --git a/scripts/s3-cleaner.js b/scripts/s3-cleaner.js index d60e8f2..30893b3 100644 --- a/scripts/s3-cleaner.js +++ b/scripts/s3-cleaner.js @@ -7,11 +7,11 @@ // ============================================================ // CREDENTIALS — replace with your actual values // ============================================================ -const S3_ACCESS_KEY_ID = 'YOUR_ACCESS_KEY_ID' -const S3_SECRET_ACCESS_KEY = 'YOUR_SECRET_ACCESS_KEY' -const S3_REGION = 'auto' // 'us-east-1' for AWS, 'auto' for R2 -const S3_ENDPOINT = 'https://your-account.r2.cloudflarestorage.com' // S3 or R2 endpoint -const S3_BUCKET = 'your-bucket-name' +const S3_ACCESS_KEY_ID = '8fab47503efb9547b50e4fb317e35cc7' +const S3_SECRET_ACCESS_KEY = '47c2eb5636843cf568dda7ad0959a3e42071303f26dbdff94bd45a3c33dcd950' +const S3_REGION = 'apac' // 'us-east-1' for AWS, 'auto' for R2 +const S3_ENDPOINT = 'https://da9b1aa7c1951c23e2c0c3246ba68a58.r2.cloudflarestorage.com' // S3 or R2 endpoint +const S3_BUCKET = 'meatfarmer' const API_CACHE_KEY = 'api-cache' // matches API_CACHE_KEY env var in backend // ============================================================ @@ -62,14 +62,16 @@ async function listAllObjects(prefix) { async function deleteObjects(keys) { let deleted = 0 - const batchSize = 1000 + const concurrency = 20 - for (let i = 0; i < keys.length; i += batchSize) { - const batch = keys.slice(i, i + batchSize) - const objects = batch.map((key) => ({ key })) - - const result = await s3.deleteObjects({ objects }) - deleted += result.deleted?.length ?? 0 + for (let i = 0; i < keys.length; i += concurrency) { + const batch = keys.slice(i, i + concurrency) + await Promise.all( + batch.map(async (key) => { + await s3.delete(key) + deleted++ + }) + ) process.stdout.write(`\r Deleted ${deleted}/${keys.length}...`) } diff --git a/scripts/s3-sync.js b/scripts/s3-sync.js new file mode 100644 index 0000000..cf137d9 --- /dev/null +++ b/scripts/s3-sync.js @@ -0,0 +1,99 @@ +#!/usr/bin/env bun +// s3-sync.js — Ensure all objects in the source bucket exist in the dest bucket +// Usage: bun s3-sync.js +// Lists all objects in SOURCE and DEST, then copies only the objects that +// are missing from DEST. Direction is one-way: SOURCE -> DEST. + +// ============================================================ +// CREDENTIALS — replace with your actual values +// ============================================================ +const SOURCE = { + accessKeyId: '8fab47503efb9547b50e4fb317e35cc7', + secretAccessKey: '47c2eb5636843cf568dda7ad0959a3e42071303f26dbdff94bd45a3c33dcd950', + region: 'auto', // 'us-east-1' for AWS, 'auto' for R2 + endpoint: 'https://da9b1aa7c1951c23e2c0c3246ba68a58.r2.cloudflarestorage.com', // S3 or R2 endpoint + bucket: 'meatfarmer', +} + +const DEST = { + accessKeyId: '8fab47503efb9547b50e4fb317e35cc7', + secretAccessKey: '47c2eb5636843cf568dda7ad0959a3e42071303f26dbdff94bd45a3c33dcd950', + region: 'auto', + endpoint: 'https://da9b1aa7c1951c23e2c0c3246ba68a58.r2.cloudflarestorage.com', + bucket: 'meatfarmer-dev', +} +// ============================================================ + +const CONCURRENCY = 20 + +if (!SOURCE.endpoint || !SOURCE.bucket || !DEST.endpoint || !DEST.bucket) { + console.error('Fill in the SOURCE and DEST credentials at the top of the script first.') + process.exit(1) +} + +const source = new Bun.S3Client(SOURCE) +const dest = new Bun.S3Client(DEST) + +async function listAllObjects(s3) { + const allKeys = [] + let continuationToken + + do { + const options = { maxKeys: 1000 } + if (continuationToken) options.continuationToken = continuationToken + + const result = await s3.list(options) + for (const obj of result.contents) { + allKeys.push(obj.key) + } + continuationToken = result.nextContinuationToken + + process.stdout.write(`\r Listed ${allKeys.length} objects...`) + } while (continuationToken) + + console.log('') + return allKeys +} + +// ============================================================ +// MAIN +// ============================================================ +try { + console.log(`🔁 S3 Sync — ${SOURCE.bucket} -> ${DEST.bucket}`) + + console.log('\n📋 Listing SOURCE...') + const sourceKeys = await listAllObjects(source) + + console.log('\n📋 Listing DEST...') + const destKeys = await listAllObjects(dest) + + const destSet = new Set(destKeys) + const toCopy = sourceKeys.filter((key) => !destSet.has(key)) + + console.log(`\n📊 SOURCE objects: ${sourceKeys.length}`) + console.log(` DEST objects: ${destKeys.length}`) + console.log(` To copy: ${toCopy.length} (present in SOURCE, missing in DEST)`) + + if (toCopy.length === 0) { + console.log('\n✅ DEST already has everything from SOURCE.') + process.exit(0) + } + + console.log('\n🚚 Copying missing objects...') + let copied = 0 + for (let i = 0; i < toCopy.length; i += CONCURRENCY) { + const batch = toCopy.slice(i, i + CONCURRENCY) + await Promise.all( + batch.map(async (key) => { + await dest.write(key, source.file(key)) + copied++ + }) + ) + process.stdout.write(`\r Copied ${copied}/${toCopy.length}...`) + } + + console.log(`\n✅ Done — copied ${copied} objects to ${DEST.bucket}.`) +} catch (error) { + console.error(`\n❌ Error: ${error.message}`) + process.exit(1) +}