From 13610569cc6dc0018cbb960f651d96774dbab3f6 Mon Sep 17 00:00:00 2001
From: Anas Rashid <anasrasheed22@gmail.com>
Date: Thu, 8 Oct 2026 21:00:15 +0200
Subject: [PATCH] deploy/sync.sh: daily Wikisource -> divan-data -> PostgreSQL
 sync for the server

- updates a divan-data checkout, fetches new/edited works (incremental), rebuilds, upserts
  into Postgres; optional commit/push of data (DIVAN_DATA_PUSH=1); flock against overlap
- import prints progress only on a terminal
Validated locally: full run 1m46s, 382 poets / 11,087 poems imported.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
---
 README.md         | 11 +++++++++++
 api/src/import.ts |  2 +-
 deploy/sync.sh    | 39 +++++++++++++++++++++++++++++++++++++++
 3 files changed, 51 insertions(+), 1 deletion(-)
 create mode 100755 deploy/sync.sh

diff --git a/README.md b/README.md
index fe9e29b4..61138322 100644
--- a/README.md
+++ b/README.md
@@ -33,6 +33,17 @@ Settings: `DATABASE_URL` (API, default `postgres://divan:divan_local@localhost:5
 
 The import upserts, so re-running it after a divan-data sync applies the changes.
 
+## Daily content sync (server)
+
+`deploy/sync.sh` keeps a server current: it updates a divan-data checkout, fetches new and edited works from Wikisource (incremental, about a minute), rebuilds the export and upserts it into PostgreSQL. The site shows new content immediately. Runs are locked so they never overlap.
+
+```sh
+# crontab -e   (daily at 03:15; DATABASE_URL as for the API)
+15 3 * * * DATABASE_URL=postgres://divan:...@localhost:5432/divan /opt/divan/deploy/sync.sh >> /var/log/divan-sync.log 2>&1
+```
+
+Settings: `DIVAN_DATA_DIR` (default `/opt/divan-data`, cloned on first run), `DIVAN_APP_DIR` (default: this repo), `DIVAN_DATA_PUSH=1` to also commit and push data changes (needs git push access). Needs git, python3 and Node 24+.
+
 ## API
 
 | Endpoint | Returns |
diff --git a/api/src/import.ts b/api/src/import.ts
index 8c6e527d..1786aded 100644
--- a/api/src/import.ts
+++ b/api/src/import.ts
@@ -80,7 +80,7 @@ for (const entry of manifest.Poets) {
   } finally {
     client.release();
   }
-  process.stdout.write(`\r${poems} poems`);
+  if (process.stdout.isTTY) process.stdout.write(`\r${poems} poems`); // progress only when interactive (cron logs stay clean)
 }
 await pool.query(await readFile(new URL('../../db/featured.sql', import.meta.url), 'utf8'));
 console.log(`\nimported ${manifest.Poets.length} poets, ${poems} poems`);
diff --git a/deploy/sync.sh b/deploy/sync.sh
new file mode 100755
index 00000000..8ede1c40
--- /dev/null
+++ b/deploy/sync.sh
@@ -0,0 +1,39 @@
+#!/usr/bin/env bash
+# Daily content sync for a Divan server: Wikisource -> divan-data -> PostgreSQL.
+#
+#   1. update the divan-data checkout and fetch new/edited works from Wikisource (incremental)
+#   2. rebuild its search index and site export
+#   3. load the export into PostgreSQL (upserts; the site reads it live)
+#   4. optionally commit + push the refreshed data (DIVAN_DATA_PUSH=1, needs git push access)
+#
+# Schedule with cron, e.g. daily at 03:15:
+#   15 3 * * * /opt/divan/deploy/sync.sh >> /var/log/divan-sync.log 2>&1
+#
+# Settings (env): DIVAN_DATA_DIR (default /opt/divan-data), DIVAN_APP_DIR (default: this repo),
+#   DATABASE_URL (Postgres, as for the API), DIVAN_DATA_PUSH=1 to push data changes.
+# Needs: git, python3 (stdlib only), node 24+.
+set -euo pipefail
+
+APP_DIR=${DIVAN_APP_DIR:-$(cd "$(dirname "$0")/.." && pwd)}
+DATA_DIR=${DIVAN_DATA_DIR:-/opt/divan-data}
+LOCK=${TMPDIR:-/tmp}/divan-sync.lock
+
+# one run at a time (a slow Wikisource day must not overlap the next run)
+exec 9>"$LOCK"
+if command -v flock >/dev/null && ! flock -n 9; then echo "$(date -u +%FT%TZ) sync already running"; exit 0; fi
+
+echo "== $(date -u +%FT%TZ) divan sync"
+[ -d "$DATA_DIR/.git" ] || git clone -q https://github.com/anas-rashid/divan-data.git "$DATA_DIR"
+cd "$DATA_DIR"
+git pull -q --ff-only
+
+if [ "${DIVAN_DATA_PUSH:-0}" = 1 ]; then
+  ./update.sh                                   # fetch + rebuild, commit and push if the data changed
+else
+  python3 wikisource.py && python3 build_index.py && python3 export_divan.py
+fi
+
+cd "$APP_DIR/api"
+[ -d node_modules ] || npm ci --omit=dev --silent
+node src/import.ts "$DATA_DIR"
+echo "== $(date -u +%FT%TZ) done"