From 93825fe5d95b7ae1a9778389f7eb42b32d7db34e Mon Sep 17 00:00:00 2001 From: Anas Rashid Date: Fri, 9 Oct 2026 00:37:39 +0200 Subject: [PATCH] Publishing commits to divan-data git (#34); public names only; 'last edited' on poems MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - api/src/git.ts: each published version is a commit in the divan-data checkout (queued, one at a time): the moderator as author by public name with a placeholder email, the summary and version in the message, Reviewed-by / Approved-by / Divan-Revision / Divan-Version trailers; DIVAN_GIT_PUSH=1 pushes. The commit id is kept with the revision. - Privacy fix: published files and commits carry public names (profile name, else 'موڈریٹر '), never email addresses (divan-data is public). - Poem pages: 'دیوان کا ورژن …' with who edited, reviewed and published, the date, and a link to the commit diff (DIVAN_DATA_COMMIT_URL). - deploy/sync.sh: pull --rebase so publishing commits are kept. - Admins are super moderators (documented). Co-Authored-By: Claude Opus 5.5 --- README.md | 2 +- api/src/auth.ts | 3 +++ api/src/git.ts | 31 +++++++++++++++++++++++++++++++ api/src/moderation.test.ts | 21 ++++++++++++++++++--- api/src/moderation.ts | 27 +++++++++++++++++++++------ api/src/owned.ts | 2 +- api/src/server.ts | 10 ++++++++-- db/schema.sql | 3 +++ deploy/sync.sh | 2 +- web/src/pages/[...path].astro | 7 +++++++ 10 files changed, 94 insertions(+), 14 deletions(-) create mode 100644 api/src/git.ts diff --git a/README.md b/README.md index 7c76a965..0b13ecfe 100644 --- a/README.md +++ b/README.md @@ -35,7 +35,7 @@ The import upserts, so re-running it after a divan-data sync applies the changes **Accounts and admin.** Readers sign up with an email address and password (no email is sent). The first admin is made on the server: sign up on the site, then `npm run make-admin -- you@example.com` in `api/`. Admins manage users at `/admin` (search, password reset on a reader's request, disable, roles, delete) and see every admin action at `/admin/audit`. Moderators (L2 junior, L1 senior) get scoped permissions from admins: a scope (all poets, a poet, a book with everything in it, or one work), content types (poets, books, works, dictionary) and actions (create, edit, delete, arrange); `can()` in `api/src/permissions.ts` is the one check for moderation. -**Moderation.** Moderators open **ترمیم کریں** on a work they may edit, change its Divan text (see `docs/content-model.md`) with an edit summary and submit it. An L1 moderator covering that work approves, returns (with a reason) or rejects it; an admin publishes. L1 drafts go straight to the admin and an admin's own edits publish directly. Every step is recorded (`/mod`, the activity log, each work's history at `/mod/work/`). Publishing numbers the version, writes it to divan-data's `divan/` folder (`DIVAN_DATA_DIR`, default `../divan-data`) and shows it on the site at once; a draft started before a newer version was published cannot be published. +**Moderation.** Moderators open **ترمیم کریں** on a work they may edit, change its Divan text (see `docs/content-model.md`) with an edit summary and submit it. An L1 moderator covering that work approves, returns (with a reason) or rejects it; an admin publishes. L1 drafts go straight to the admin and an admin's own edits publish directly. Every step is recorded (`/mod`, the activity log, each work's history at `/mod/work/`). Admins are super moderators: they edit, review and publish any work without grants. Publishing numbers the version, writes it to divan-data's `divan/` folder (`DIVAN_DATA_DIR`, default `../divan-data`), **commits it there** (the moderator as author by public name, with `Reviewed-by:` and `Approved-by:` trailers; `DIVAN_GIT_PUSH=1` also pushes), and shows it on the site at once with a link to the commit (`DIVAN_DATA_COMMIT_URL`, `{sha}` replaced). Public content never carries email addresses. A draft started before a newer version was published cannot be published. The daily sync rebases on pull so these commits are kept. ## Daily content sync (server) diff --git a/api/src/auth.ts b/api/src/auth.ts index db881dd3..e3763df2 100644 --- a/api/src/auth.ts +++ b/api/src/auth.ts @@ -60,6 +60,9 @@ const signinByIp = limiter(20, 15 * 60_000), signinByEmail = limiter(8, 15 * 60_ function sha(t: string) { return createHash('sha256').update(t).digest('hex'); } +// a person's public name (shown on published content and in divan-data's git history): never their email +export const publicName = (u: { id: unknown; full_name?: string | null }) => u.full_name?.trim() || `موڈریٹر ${u.id}`; + export const publicUser = (u: any) => ({ id: Number(u.id), email: u.email, role: u.role as string, created_at: u.created_at, full_name: u.full_name ?? '', bio: u.bio ?? '', }); diff --git a/api/src/git.ts b/api/src/git.ts new file mode 100644 index 00000000..15e5259f --- /dev/null +++ b/api/src/git.ts @@ -0,0 +1,31 @@ +// Publishing to git (#34): each published version is a commit in the divan-data checkout, so the public git +// history is the record of every change. Commits run one at a time (ponytail: in-process queue; one API process). +// Moderators appear by their public name with a placeholder email, never their real address. +// DIVAN_GIT_PUSH=1 push after each commit (else the daily sync or a release pushes) +// DIVAN_GIT_EMAIL_DOMAIN domain of the placeholder emails (default users.noreply.divan) +import { execFile } from 'node:child_process'; +import { promisify } from 'node:util'; + +const run = promisify(execFile); +const git = (dir: string, ...args: string[]) => run('git', ['-C', dir, ...args], { maxBuffer: 16 * 1024 * 1024 }); +let queue: Promise = Promise.resolve(); + +export type Person = { id: number | string; name: string }; +// a moderator as a git identity: public name, placeholder email +export const identity = (p: Person) => `${p.name.replace(/[<>\n]/g, '')} `; + +// commit these files (paths relative to dir) as the author; returns the commit id, or null if dir is not a git +// checkout (publishing still works, there is just no commit) +export function commit(dir: string, files: string[], author: Person, message: string): Promise { + const job = queue.then(async () => { + try { await git(dir, 'rev-parse', '--is-inside-work-tree'); } catch { return null; } + await git(dir, 'add', '--', ...files); + await git(dir, '-c', 'user.name=Divan', '-c', `user.email=publish@${process.env.DIVAN_GIT_EMAIL_DOMAIN ?? 'users.noreply.divan'}`, + 'commit', '--quiet', `--author=${identity(author)}`, '-m', message, '--', ...files); + const sha = (await git(dir, 'rev-parse', 'HEAD')).stdout.trim(); + if (process.env.DIVAN_GIT_PUSH === '1') await git(dir, 'push', '--quiet').catch((e) => console.error('divan-data push failed:', e.message)); + return sha; + }); + queue = job.catch(() => {}); + return job; +} diff --git a/api/src/moderation.test.ts b/api/src/moderation.test.ts index 1bf2787b..ce716144 100644 --- a/api/src/moderation.test.ts +++ b/api/src/moderation.test.ts @@ -9,6 +9,7 @@ import { adminRoutes } from './admin.ts'; import { permissionRoutes } from './permissions.ts'; import { moderationRoutes } from './moderation.ts'; import { diffLines } from './diff.ts'; +import { execFileSync } from 'node:child_process'; import { pool } from './db.ts'; after(() => pool.end()); @@ -20,6 +21,8 @@ test('line diff: kept, removed and added lines', () => { test('pipeline: L2 drafts, L1 approves, admin publishes; returns, rejects, permissions, conflicts, history', async () => { const data = await mkdtemp(join(tmpdir(), 'divan-data-')); process.env.DIVAN_DATA_DIR = data; + const git = (...a: string[]) => execFileSync('git', ['-C', data, ...a], { encoding: 'utf8' }); + git('init', '-q'); git('-c', 'user.name=t', '-c', 'user.email=t@t', 'commit', '-q', '--allow-empty', '-m', 'start'); const app = Fastify(); authRoutes(app); adminRoutes(app); permissionRoutes(app); moderationRoutes(app); const run = Date.now(); @@ -31,6 +34,7 @@ test('pipeline: L2 drafts, L1 approves, admin publishes; returns, rejects, permi return { ...s, id: s.user.id }; }; const admin = await person('admin', 'admin'), l1 = await person('l1', 'mod-l1'), l2 = await person('l2', 'mod-l2'); + await call('POST', '/api/auth/profile', { full_name: 'نیا موڈریٹر' }, l2.token); // a public name; the L1 has none const other = await person('l1other', 'mod-l1'), reader = await person('reader', 'reader'); // a Ghalib ghazal; the work is restored at the end @@ -74,9 +78,20 @@ test('pipeline: L2 drafts, L1 approves, admin publishes; returns, rejects, permi // admin publishes: version 1 in divan-data and on the site assert.deepEqual((await call('POST', `/api/mod/revisions/${id}/publish`, {}, admin.token)).json(), { status: 'published', version: 1 }); const file = JSON.parse(await readFile(join(data, 'divan', poem.url.slice(1) + '.json'), 'utf8')); - assert.equal(file.Edited.by, `l2-${run}@divan.test`); - assert.equal(file.Edited.reviewedBy, `l1-${run}@divan.test`); - assert.equal(file.Edited.publishedBy, `admin-${run}@divan.test`); + // public names only in divan-data (it is public); a commit with the moderator as author and review trailers + assert.equal(file.Edited.by, 'نیا موڈریٹر'); + assert.equal(file.Edited.reviewedBy, `موڈریٹر ${l1.id}`); + assert.equal(file.Edited.publishedBy, `موڈریٹر ${admin.id}`); + assert.ok(!JSON.stringify(file).includes('@'), 'no email addresses'); + const log = git('log', '-1', '--format=%an <%ae>%n%B'); + assert.match(log, new RegExp(`^نیا موڈریٹر `)); + assert.match(log, /\(ورژن 1\)/); + assert.match(log, new RegExp(`Reviewed-by: موڈریٹر ${l1.id} approved // admin: publishes (an admin's own draft publishes directly), returns or rejects -// Publishing numbers the version, writes it to divan-data as Divan-owned content (owned.ts) and updates the site. +// Admins are super moderators: they can edit, review and publish any work without grants. +// Publishing numbers the version, writes it to divan-data as Divan-owned content (owned.ts), commits it there (git.ts) +// and updates the site. // If another version was published after the draft started, publishing is refused until the draft is redone. // GET /api/mod/can?poem= what the reader may do on a work // GET /api/mod/queue my drafts, drafts to review, drafts to publish @@ -23,6 +25,8 @@ import { fromPoem, writeOwned } from './owned.ts'; import { parse, toVerses } from './divantext.ts'; import { normalise } from './urdu.ts'; import { diffLines, changed } from './diff.ts'; +import { commit, identity } from './git.ts'; +import { publicName } from './auth.ts'; const dataDir = () => process.env.DIVAN_DATA_DIR ?? new URL('../../../divan-data', import.meta.url).pathname; const isModerator = (u: any) => ['mod-l2', 'mod-l1', 'admin'].includes(u?.role); @@ -79,13 +83,22 @@ async function publish(r: any, u: any, comment?: string) { throw Object.assign(new Error('اس دوران اس کلام کا نیا ورژن شائع ہو چکا ہے۔ مسودہ واپس بھیج کر تازہ متن پر دوبارہ بنوائیں۔'), { code: 409 }); const doc = parse(r.content), verses = toVerses(doc); const title = doc.meta['عنوان'] || cur.poem.title, version = cur.version + 1, at = new Date().toISOString(); - // divan-data first (the published record), then the site's database - await writeOwned(dataDir(), cur.poem.url, r.content, { by: r.author_email, at, version, reviewedBy: r.reviewer_email, publishedBy: u.email }); + // who did it, by public name (divan-data is public: never email addresses) + const people = async (id: unknown) => id ? (await pool.query('SELECT id, full_name FROM users WHERE id = $1', [id])).rows[0] ?? null : null; + const [author, reviewer] = await Promise.all([people(r.author_id), people(r.reviewer_id)]); + const who = (p: any) => p && { id: Number(p.id), name: publicName(p) }; + const credits = { by: who(author)?.name ?? 'موڈریٹر', reviewedBy: who(reviewer)?.name ?? null, publishedBy: publicName(u) }; + // divan-data first (the published record, committed to git), then the site's database + const files = await writeOwned(dataDir(), cur.poem.url, r.content, { ...credits, at, version, revision: Number(r.id) }); + const trailers = [`Divan-Revision: ${r.id}`, `Divan-Version: ${version}`, + ...(reviewer ? [`Reviewed-by: ${identityOf(reviewer)}`] : []), `Approved-by: ${identityOf(u)}`]; + const sha = await commit(dataDir(), files.map((f) => f.slice(dataDir().replace(/\/$/, '').length + 1)), + who(author) ?? { id: 0, name: 'موڈریٹر' }, `${title}: ${r.summary || 'ترمیم'} (ورژن ${version})\n\n${trailers.join('\n')}`); const client = await pool.connect(); try { await client.query('BEGIN'); - await client.query(`UPDATE revisions SET status = 'published', version = $2, publisher_email = $3, published_at = $4, updated_at = now() WHERE id = $1`, - [r.id, version, u.email, at]); + await client.query(`UPDATE revisions SET status = 'published', version = $2, publisher_email = $3, published_at = $4, commit = $5, credits = $6, updated_at = now() + WHERE id = $1`, [r.id, version, u.email, at, sha, credits]); await client.query('UPDATE poems SET title = $2, search_text = $3 WHERE id = $1', [r.entity_id, title, normalise([title, ...verses.map((v) => v.Text)].join(' '))]); await client.query('DELETE FROM verses WHERE poem_id = $1', [r.entity_id]); @@ -104,6 +117,8 @@ async function publish(r: any, u: any, comment?: string) { return version; } +const identityOf = (p: any) => identity({ id: Number(p.id), name: publicName(p) }); + export function moderationRoutes(app: FastifyInstance) { app.get<{ Querystring: { poem?: string } }>('/api/mod/can', async (req) => { const u = await sessionUser(req), poemId = Number(req.query.poem) || 0; @@ -202,7 +217,7 @@ export function moderationRoutes(app: FastifyInstance) { return { status: u.role === 'mod-l1' ? 'approved' : 'submitted' }; } if (action === 'approve') { - await pool.query(`UPDATE revisions SET status = 'approved', reviewer_email = $2, updated_at = now() WHERE id = $1`, [r.id, u.email]); + await pool.query(`UPDATE revisions SET status = 'approved', reviewer_id = $3, reviewer_email = $2, updated_at = now() WHERE id = $1`, [r.id, u.email, u.id]); await event(r.id, u, 'approved', comment); return { status: 'approved' }; } diff --git a/api/src/owned.ts b/api/src/owned.ts index 0ca295a8..d13816e7 100644 --- a/api/src/owned.ts +++ b/api/src/owned.ts @@ -31,7 +31,7 @@ export function fromPoem(poem: { Title: string; Verses: Verse[]; SourceUrl?: str // write a Divan-owned work (the .dtx and the generated .json); returns the paths written export async function writeOwned(dataDir: string, url: string, dtx: string, - edited: { by: string; at?: string; version?: number; reviewedBy?: string | null; publishedBy?: string }) { + edited: { by: string; at?: string; version?: number; revision?: number; reviewedBy?: string | null; publishedBy?: string }) { const doc = parse(dtx); const verses = toVerses(doc); if (!verses.length) throw new Error('the text has no verses or paragraphs'); diff --git a/api/src/server.ts b/api/src/server.ts index de0eb0cb..6a363279 100644 --- a/api/src/server.ts +++ b/api/src/server.ts @@ -53,7 +53,7 @@ app.get<{ Querystring: { url?: string } }>('/api/page', async (req, reply) => { const poem = (await pool.query('SELECT * FROM poems WHERE url = $1', [url])).rows[0]; if (poem) { - const [verses, poet, crumbs, siblings] = await Promise.all([ + const [verses, poet, crumbs, siblings, edited] = await Promise.all([ pool.query('SELECT vorder, position, couplet, text FROM verses WHERE poem_id = $1 ORDER BY vorder', [poem.id]), pool.query('SELECT id, url, nickname FROM poets WHERE id = $1', [poem.poet_id]), ancestors(poem.category_id), @@ -62,9 +62,15 @@ app.get<{ Querystring: { url?: string } }>('/api/page', async (req, reply) => { (SELECT url FROM poems WHERE category_id = $1 AND position > $2 ORDER BY position LIMIT 1) AS next`, [poem.category_id, poem.position], ), + // the latest Divan version (public names only) and its divan-data commit + pool.query(`SELECT version, published_at, credits, commit FROM revisions WHERE entity = 'work' AND entity_id = $1 AND status = 'published' + ORDER BY version DESC LIMIT 1`, [poem.id]), ]); + const e = edited.rows[0]; + const divan = e && { version: e.version, at: e.published_at, ...e.credits, + commit_url: e.commit ? (process.env.DIVAN_DATA_COMMIT_URL ?? 'https://git.anasrashid.net/anas/divan-data/commit/{sha}').replace('{sha}', e.commit) : null }; const { search_text, ...rest } = poem; - return { type: 'poem', poem: rest, poet: poet.rows[0], breadcrumbs: crumbs, verses: verses.rows, ...siblings.rows[0] }; + return { type: 'poem', poem: rest, poet: poet.rows[0], breadcrumbs: crumbs, verses: verses.rows, ...siblings.rows[0], divan }; } const cat = (await pool.query('SELECT * FROM categories WHERE url = $1', [url])).rows[0]; diff --git a/db/schema.sql b/db/schema.sql index d67dc630..42ab7eb8 100644 --- a/db/schema.sql +++ b/db/schema.sql @@ -170,6 +170,9 @@ CREATE TABLE IF NOT EXISTS revisions ( published_at timestamptz ); ALTER TABLE revisions ADD COLUMN IF NOT EXISTS base_content text NOT NULL DEFAULT ''; +ALTER TABLE revisions ADD COLUMN IF NOT EXISTS reviewer_id bigint REFERENCES users(id) ON DELETE SET NULL; +ALTER TABLE revisions ADD COLUMN IF NOT EXISTS commit text; -- divan-data commit of a published version +ALTER TABLE revisions ADD COLUMN IF NOT EXISTS credits jsonb; -- public names at publishing: by, reviewedBy, publishedBy CREATE INDEX IF NOT EXISTS revisions_entity ON revisions(entity, entity_id); CREATE INDEX IF NOT EXISTS revisions_status ON revisions(status); CREATE UNIQUE INDEX IF NOT EXISTS revisions_version ON revisions(entity, entity_id, version) WHERE version IS NOT NULL; diff --git a/deploy/sync.sh b/deploy/sync.sh index bfc01e8b..9ccd9194 100755 --- a/deploy/sync.sh +++ b/deploy/sync.sh @@ -26,7 +26,7 @@ if command -v flock >/dev/null && ! flock -n 9; then echo "$(date -u +%FT%TZ) sy echo "== $(date -u +%FT%TZ) divan sync" [ -d "$DATA_DIR/.git" ] || git clone -q https://github.com/anas-rashid/divan-data.git "$DATA_DIR" cd "$DATA_DIR" -git pull -q --ff-only +git pull -q --rebase # keep commits made by publishing in the Divan app (git.ts) if [ "${DIVAN_DATA_PUSH:-0}" = 1 ]; then ./update.sh # fetch + rebuild, commit and push if the data changed diff --git a/web/src/pages/[...path].astro b/web/src/pages/[...path].astro index fc7ffd92..f00c7c36 100644 --- a/web/src/pages/[...path].astro +++ b/web/src/pages/[...path].astro @@ -136,6 +136,13 @@ const title = page.type === 'poem' ? page.poem.title : page.type === 'poet' ? pa {page.next ? اگلا › : } {page.prev ? ‹ پچھلا : } + {page.divan && ( +

+ دیوان کا ورژن {ud(page.divan.version)} · ترمیم: {page.divan.by}{page.divan.reviewedBy && <> · جائزہ: {page.divan.reviewedBy}} · اشاعت: {page.divan.publishedBy} + · {new Date(page.divan.at).toISOString().slice(0, 10)} + {page.divan.commit_url && <> · تبدیلی دیکھیں} +

+ )} {page.poem.source_url && (

ماخذ: ویکی ماخذ · {page.poet && {page.poet.nickname}}

)}