Merge pull request 'Publishing commits to divan-data git; public names only; 'last edited' on poems' (#56) from feature/git-publishing into main
Reviewed-on: #56
This commit is contained in:
commit
271d826fc2
@ -35,7 +35,7 @@ The import upserts, so re-running it after a divan-data sync applies the changes
|
||||
|
||||
**Accounts and admin.** Readers sign up with an email address and password (no email is sent). The first admin is made on the server: sign up on the site, then `npm run make-admin -- you@example.com` in `api/`. Admins manage users at `/admin` (search, password reset on a reader's request, disable, roles, delete) and see every admin action at `/admin/audit`. Moderators (L2 junior, L1 senior) get scoped permissions from admins: a scope (all poets, a poet, a book with everything in it, or one work), content types (poets, books, works, dictionary) and actions (create, edit, delete, arrange); `can()` in `api/src/permissions.ts` is the one check for moderation.
|
||||
|
||||
**Moderation.** Moderators open **ترمیم کریں** on a work they may edit, change its Divan text (see `docs/content-model.md`) with an edit summary and submit it. An L1 moderator covering that work approves, returns (with a reason) or rejects it; an admin publishes. L1 drafts go straight to the admin and an admin's own edits publish directly. Every step is recorded (`/mod`, the activity log, each work's history at `/mod/work/<id>`). Publishing numbers the version, writes it to divan-data's `divan/` folder (`DIVAN_DATA_DIR`, default `../divan-data`) and shows it on the site at once; a draft started before a newer version was published cannot be published.
|
||||
**Moderation.** Moderators open **ترمیم کریں** on a work they may edit, change its Divan text (see `docs/content-model.md`) with an edit summary and submit it. An L1 moderator covering that work approves, returns (with a reason) or rejects it; an admin publishes. L1 drafts go straight to the admin and an admin's own edits publish directly. Every step is recorded (`/mod`, the activity log, each work's history at `/mod/work/<id>`). Admins are super moderators: they edit, review and publish any work without grants. Publishing numbers the version, writes it to divan-data's `divan/` folder (`DIVAN_DATA_DIR`, default `../divan-data`), **commits it there** (the moderator as author by public name, with `Reviewed-by:` and `Approved-by:` trailers; `DIVAN_GIT_PUSH=1` also pushes), and shows it on the site at once with a link to the commit (`DIVAN_DATA_COMMIT_URL`, `{sha}` replaced). Public content never carries email addresses. A draft started before a newer version was published cannot be published. The daily sync rebases on pull so these commits are kept.
|
||||
|
||||
## Daily content sync (server)
|
||||
|
||||
|
||||
@ -60,6 +60,9 @@ const signinByIp = limiter(20, 15 * 60_000), signinByEmail = limiter(8, 15 * 60_
|
||||
function sha(t: string) {
|
||||
return createHash('sha256').update(t).digest('hex');
|
||||
}
|
||||
// a person's public name (shown on published content and in divan-data's git history): never their email
|
||||
export const publicName = (u: { id: unknown; full_name?: string | null }) => u.full_name?.trim() || `موڈریٹر ${u.id}`;
|
||||
|
||||
export const publicUser = (u: any) => ({
|
||||
id: Number(u.id), email: u.email, role: u.role as string, created_at: u.created_at, full_name: u.full_name ?? '', bio: u.bio ?? '',
|
||||
});
|
||||
|
||||
31
api/src/git.ts
Normal file
31
api/src/git.ts
Normal file
@ -0,0 +1,31 @@
|
||||
// Publishing to git (#34): each published version is a commit in the divan-data checkout, so the public git
|
||||
// history is the record of every change. Commits run one at a time (ponytail: in-process queue; one API process).
|
||||
// Moderators appear by their public name with a placeholder email, never their real address.
|
||||
// DIVAN_GIT_PUSH=1 push after each commit (else the daily sync or a release pushes)
|
||||
// DIVAN_GIT_EMAIL_DOMAIN domain of the placeholder emails (default users.noreply.divan)
|
||||
import { execFile } from 'node:child_process';
|
||||
import { promisify } from 'node:util';
|
||||
|
||||
const run = promisify(execFile);
|
||||
const git = (dir: string, ...args: string[]) => run('git', ['-C', dir, ...args], { maxBuffer: 16 * 1024 * 1024 });
|
||||
let queue: Promise<unknown> = Promise.resolve();
|
||||
|
||||
export type Person = { id: number | string; name: string };
|
||||
// a moderator as a git identity: public name, placeholder email
|
||||
export const identity = (p: Person) => `${p.name.replace(/[<>\n]/g, '')} <moderator-${p.id}@${process.env.DIVAN_GIT_EMAIL_DOMAIN ?? 'users.noreply.divan'}>`;
|
||||
|
||||
// commit these files (paths relative to dir) as the author; returns the commit id, or null if dir is not a git
|
||||
// checkout (publishing still works, there is just no commit)
|
||||
export function commit(dir: string, files: string[], author: Person, message: string): Promise<string | null> {
|
||||
const job = queue.then(async () => {
|
||||
try { await git(dir, 'rev-parse', '--is-inside-work-tree'); } catch { return null; }
|
||||
await git(dir, 'add', '--', ...files);
|
||||
await git(dir, '-c', 'user.name=Divan', '-c', `user.email=publish@${process.env.DIVAN_GIT_EMAIL_DOMAIN ?? 'users.noreply.divan'}`,
|
||||
'commit', '--quiet', `--author=${identity(author)}`, '-m', message, '--', ...files);
|
||||
const sha = (await git(dir, 'rev-parse', 'HEAD')).stdout.trim();
|
||||
if (process.env.DIVAN_GIT_PUSH === '1') await git(dir, 'push', '--quiet').catch((e) => console.error('divan-data push failed:', e.message));
|
||||
return sha;
|
||||
});
|
||||
queue = job.catch(() => {});
|
||||
return job;
|
||||
}
|
||||
@ -9,6 +9,7 @@ import { adminRoutes } from './admin.ts';
|
||||
import { permissionRoutes } from './permissions.ts';
|
||||
import { moderationRoutes } from './moderation.ts';
|
||||
import { diffLines } from './diff.ts';
|
||||
import { execFileSync } from 'node:child_process';
|
||||
import { pool } from './db.ts';
|
||||
|
||||
after(() => pool.end());
|
||||
@ -20,6 +21,8 @@ test('line diff: kept, removed and added lines', () => {
|
||||
test('pipeline: L2 drafts, L1 approves, admin publishes; returns, rejects, permissions, conflicts, history', async () => {
|
||||
const data = await mkdtemp(join(tmpdir(), 'divan-data-'));
|
||||
process.env.DIVAN_DATA_DIR = data;
|
||||
const git = (...a: string[]) => execFileSync('git', ['-C', data, ...a], { encoding: 'utf8' });
|
||||
git('init', '-q'); git('-c', 'user.name=t', '-c', 'user.email=t@t', 'commit', '-q', '--allow-empty', '-m', 'start');
|
||||
const app = Fastify();
|
||||
authRoutes(app); adminRoutes(app); permissionRoutes(app); moderationRoutes(app);
|
||||
const run = Date.now();
|
||||
@ -31,6 +34,7 @@ test('pipeline: L2 drafts, L1 approves, admin publishes; returns, rejects, permi
|
||||
return { ...s, id: s.user.id };
|
||||
};
|
||||
const admin = await person('admin', 'admin'), l1 = await person('l1', 'mod-l1'), l2 = await person('l2', 'mod-l2');
|
||||
await call('POST', '/api/auth/profile', { full_name: 'نیا موڈریٹر' }, l2.token); // a public name; the L1 has none
|
||||
const other = await person('l1other', 'mod-l1'), reader = await person('reader', 'reader');
|
||||
|
||||
// a Ghalib ghazal; the work is restored at the end
|
||||
@ -74,9 +78,20 @@ test('pipeline: L2 drafts, L1 approves, admin publishes; returns, rejects, permi
|
||||
// admin publishes: version 1 in divan-data and on the site
|
||||
assert.deepEqual((await call('POST', `/api/mod/revisions/${id}/publish`, {}, admin.token)).json(), { status: 'published', version: 1 });
|
||||
const file = JSON.parse(await readFile(join(data, 'divan', poem.url.slice(1) + '.json'), 'utf8'));
|
||||
assert.equal(file.Edited.by, `l2-${run}@divan.test`);
|
||||
assert.equal(file.Edited.reviewedBy, `l1-${run}@divan.test`);
|
||||
assert.equal(file.Edited.publishedBy, `admin-${run}@divan.test`);
|
||||
// public names only in divan-data (it is public); a commit with the moderator as author and review trailers
|
||||
assert.equal(file.Edited.by, 'نیا موڈریٹر');
|
||||
assert.equal(file.Edited.reviewedBy, `موڈریٹر ${l1.id}`);
|
||||
assert.equal(file.Edited.publishedBy, `موڈریٹر ${admin.id}`);
|
||||
assert.ok(!JSON.stringify(file).includes('@'), 'no email addresses');
|
||||
const log = git('log', '-1', '--format=%an <%ae>%n%B');
|
||||
assert.match(log, new RegExp(`^نیا موڈریٹر <moderator-${l2.id}@users\\.noreply\\.divan>`));
|
||||
assert.match(log, /\(ورژن 1\)/);
|
||||
assert.match(log, new RegExp(`Reviewed-by: موڈریٹر ${l1.id} <moderator-${l1.id}@`));
|
||||
assert.match(log, new RegExp(`Approved-by: موڈریٹر ${admin.id} <moderator-${admin.id}@`));
|
||||
assert.ok(!log.includes('divan.test'), 'no real emails in git');
|
||||
assert.deepEqual(git('show', '--name-only', '--format=', 'HEAD').trim().split('\n').sort(), [`divan${poem.url}.dtx`, `divan${poem.url}.json`]);
|
||||
const committed = (await pool.query(`SELECT commit FROM revisions WHERE id = $1`, [id])).rows[0].commit;
|
||||
assert.equal(committed, git('rev-parse', 'HEAD').trim(), 'the commit is recorded with the version');
|
||||
assert.ok((await readFile(join(data, 'divan', poem.url.slice(1) + '.dtx'), 'utf8')).includes(misra + ' (ترمیم)'));
|
||||
assert.equal((await pool.query('SELECT text FROM verses WHERE poem_id = $1 AND couplet = 1 AND position = $2', [poem.id, 'Left'])).rows[0].text, misra + ' (ترمیم)');
|
||||
|
||||
|
||||
@ -5,7 +5,9 @@
|
||||
// L1 moderator (grant covering the work): approves, returns (with a comment) or rejects; an L1's own draft
|
||||
// goes straight to the admin step -> approved
|
||||
// admin: publishes (an admin's own draft publishes directly), returns or rejects
|
||||
// Publishing numbers the version, writes it to divan-data as Divan-owned content (owned.ts) and updates the site.
|
||||
// Admins are super moderators: they can edit, review and publish any work without grants.
|
||||
// Publishing numbers the version, writes it to divan-data as Divan-owned content (owned.ts), commits it there (git.ts)
|
||||
// and updates the site.
|
||||
// If another version was published after the draft started, publishing is refused until the draft is redone.
|
||||
// GET /api/mod/can?poem= what the reader may do on a work
|
||||
// GET /api/mod/queue my drafts, drafts to review, drafts to publish
|
||||
@ -23,6 +25,8 @@ import { fromPoem, writeOwned } from './owned.ts';
|
||||
import { parse, toVerses } from './divantext.ts';
|
||||
import { normalise } from './urdu.ts';
|
||||
import { diffLines, changed } from './diff.ts';
|
||||
import { commit, identity } from './git.ts';
|
||||
import { publicName } from './auth.ts';
|
||||
|
||||
const dataDir = () => process.env.DIVAN_DATA_DIR ?? new URL('../../../divan-data', import.meta.url).pathname;
|
||||
const isModerator = (u: any) => ['mod-l2', 'mod-l1', 'admin'].includes(u?.role);
|
||||
@ -79,13 +83,22 @@ async function publish(r: any, u: any, comment?: string) {
|
||||
throw Object.assign(new Error('اس دوران اس کلام کا نیا ورژن شائع ہو چکا ہے۔ مسودہ واپس بھیج کر تازہ متن پر دوبارہ بنوائیں۔'), { code: 409 });
|
||||
const doc = parse(r.content), verses = toVerses(doc);
|
||||
const title = doc.meta['عنوان'] || cur.poem.title, version = cur.version + 1, at = new Date().toISOString();
|
||||
// divan-data first (the published record), then the site's database
|
||||
await writeOwned(dataDir(), cur.poem.url, r.content, { by: r.author_email, at, version, reviewedBy: r.reviewer_email, publishedBy: u.email });
|
||||
// who did it, by public name (divan-data is public: never email addresses)
|
||||
const people = async (id: unknown) => id ? (await pool.query('SELECT id, full_name FROM users WHERE id = $1', [id])).rows[0] ?? null : null;
|
||||
const [author, reviewer] = await Promise.all([people(r.author_id), people(r.reviewer_id)]);
|
||||
const who = (p: any) => p && { id: Number(p.id), name: publicName(p) };
|
||||
const credits = { by: who(author)?.name ?? 'موڈریٹر', reviewedBy: who(reviewer)?.name ?? null, publishedBy: publicName(u) };
|
||||
// divan-data first (the published record, committed to git), then the site's database
|
||||
const files = await writeOwned(dataDir(), cur.poem.url, r.content, { ...credits, at, version, revision: Number(r.id) });
|
||||
const trailers = [`Divan-Revision: ${r.id}`, `Divan-Version: ${version}`,
|
||||
...(reviewer ? [`Reviewed-by: ${identityOf(reviewer)}`] : []), `Approved-by: ${identityOf(u)}`];
|
||||
const sha = await commit(dataDir(), files.map((f) => f.slice(dataDir().replace(/\/$/, '').length + 1)),
|
||||
who(author) ?? { id: 0, name: 'موڈریٹر' }, `${title}: ${r.summary || 'ترمیم'} (ورژن ${version})\n\n${trailers.join('\n')}`);
|
||||
const client = await pool.connect();
|
||||
try {
|
||||
await client.query('BEGIN');
|
||||
await client.query(`UPDATE revisions SET status = 'published', version = $2, publisher_email = $3, published_at = $4, updated_at = now() WHERE id = $1`,
|
||||
[r.id, version, u.email, at]);
|
||||
await client.query(`UPDATE revisions SET status = 'published', version = $2, publisher_email = $3, published_at = $4, commit = $5, credits = $6, updated_at = now()
|
||||
WHERE id = $1`, [r.id, version, u.email, at, sha, credits]);
|
||||
await client.query('UPDATE poems SET title = $2, search_text = $3 WHERE id = $1',
|
||||
[r.entity_id, title, normalise([title, ...verses.map((v) => v.Text)].join(' '))]);
|
||||
await client.query('DELETE FROM verses WHERE poem_id = $1', [r.entity_id]);
|
||||
@ -104,6 +117,8 @@ async function publish(r: any, u: any, comment?: string) {
|
||||
return version;
|
||||
}
|
||||
|
||||
const identityOf = (p: any) => identity({ id: Number(p.id), name: publicName(p) });
|
||||
|
||||
export function moderationRoutes(app: FastifyInstance) {
|
||||
app.get<{ Querystring: { poem?: string } }>('/api/mod/can', async (req) => {
|
||||
const u = await sessionUser(req), poemId = Number(req.query.poem) || 0;
|
||||
@ -202,7 +217,7 @@ export function moderationRoutes(app: FastifyInstance) {
|
||||
return { status: u.role === 'mod-l1' ? 'approved' : 'submitted' };
|
||||
}
|
||||
if (action === 'approve') {
|
||||
await pool.query(`UPDATE revisions SET status = 'approved', reviewer_email = $2, updated_at = now() WHERE id = $1`, [r.id, u.email]);
|
||||
await pool.query(`UPDATE revisions SET status = 'approved', reviewer_id = $3, reviewer_email = $2, updated_at = now() WHERE id = $1`, [r.id, u.email, u.id]);
|
||||
await event(r.id, u, 'approved', comment);
|
||||
return { status: 'approved' };
|
||||
}
|
||||
|
||||
@ -31,7 +31,7 @@ export function fromPoem(poem: { Title: string; Verses: Verse[]; SourceUrl?: str
|
||||
|
||||
// write a Divan-owned work (the .dtx and the generated .json); returns the paths written
|
||||
export async function writeOwned(dataDir: string, url: string, dtx: string,
|
||||
edited: { by: string; at?: string; version?: number; reviewedBy?: string | null; publishedBy?: string }) {
|
||||
edited: { by: string; at?: string; version?: number; revision?: number; reviewedBy?: string | null; publishedBy?: string }) {
|
||||
const doc = parse(dtx);
|
||||
const verses = toVerses(doc);
|
||||
if (!verses.length) throw new Error('the text has no verses or paragraphs');
|
||||
|
||||
@ -53,7 +53,7 @@ app.get<{ Querystring: { url?: string } }>('/api/page', async (req, reply) => {
|
||||
|
||||
const poem = (await pool.query('SELECT * FROM poems WHERE url = $1', [url])).rows[0];
|
||||
if (poem) {
|
||||
const [verses, poet, crumbs, siblings] = await Promise.all([
|
||||
const [verses, poet, crumbs, siblings, edited] = await Promise.all([
|
||||
pool.query('SELECT vorder, position, couplet, text FROM verses WHERE poem_id = $1 ORDER BY vorder', [poem.id]),
|
||||
pool.query('SELECT id, url, nickname FROM poets WHERE id = $1', [poem.poet_id]),
|
||||
ancestors(poem.category_id),
|
||||
@ -62,9 +62,15 @@ app.get<{ Querystring: { url?: string } }>('/api/page', async (req, reply) => {
|
||||
(SELECT url FROM poems WHERE category_id = $1 AND position > $2 ORDER BY position LIMIT 1) AS next`,
|
||||
[poem.category_id, poem.position],
|
||||
),
|
||||
// the latest Divan version (public names only) and its divan-data commit
|
||||
pool.query(`SELECT version, published_at, credits, commit FROM revisions WHERE entity = 'work' AND entity_id = $1 AND status = 'published'
|
||||
ORDER BY version DESC LIMIT 1`, [poem.id]),
|
||||
]);
|
||||
const e = edited.rows[0];
|
||||
const divan = e && { version: e.version, at: e.published_at, ...e.credits,
|
||||
commit_url: e.commit ? (process.env.DIVAN_DATA_COMMIT_URL ?? 'https://git.anasrashid.net/anas/divan-data/commit/{sha}').replace('{sha}', e.commit) : null };
|
||||
const { search_text, ...rest } = poem;
|
||||
return { type: 'poem', poem: rest, poet: poet.rows[0], breadcrumbs: crumbs, verses: verses.rows, ...siblings.rows[0] };
|
||||
return { type: 'poem', poem: rest, poet: poet.rows[0], breadcrumbs: crumbs, verses: verses.rows, ...siblings.rows[0], divan };
|
||||
}
|
||||
|
||||
const cat = (await pool.query('SELECT * FROM categories WHERE url = $1', [url])).rows[0];
|
||||
|
||||
@ -170,6 +170,9 @@ CREATE TABLE IF NOT EXISTS revisions (
|
||||
published_at timestamptz
|
||||
);
|
||||
ALTER TABLE revisions ADD COLUMN IF NOT EXISTS base_content text NOT NULL DEFAULT '';
|
||||
ALTER TABLE revisions ADD COLUMN IF NOT EXISTS reviewer_id bigint REFERENCES users(id) ON DELETE SET NULL;
|
||||
ALTER TABLE revisions ADD COLUMN IF NOT EXISTS commit text; -- divan-data commit of a published version
|
||||
ALTER TABLE revisions ADD COLUMN IF NOT EXISTS credits jsonb; -- public names at publishing: by, reviewedBy, publishedBy
|
||||
CREATE INDEX IF NOT EXISTS revisions_entity ON revisions(entity, entity_id);
|
||||
CREATE INDEX IF NOT EXISTS revisions_status ON revisions(status);
|
||||
CREATE UNIQUE INDEX IF NOT EXISTS revisions_version ON revisions(entity, entity_id, version) WHERE version IS NOT NULL;
|
||||
|
||||
@ -26,7 +26,7 @@ if command -v flock >/dev/null && ! flock -n 9; then echo "$(date -u +%FT%TZ) sy
|
||||
echo "== $(date -u +%FT%TZ) divan sync"
|
||||
[ -d "$DATA_DIR/.git" ] || git clone -q https://github.com/anas-rashid/divan-data.git "$DATA_DIR"
|
||||
cd "$DATA_DIR"
|
||||
git pull -q --ff-only
|
||||
git pull -q --rebase # keep commits made by publishing in the Divan app (git.ts)
|
||||
|
||||
if [ "${DIVAN_DATA_PUSH:-0}" = 1 ]; then
|
||||
./update.sh # fetch + rebuild, commit and push if the data changed
|
||||
|
||||
@ -136,6 +136,13 @@ const title = page.type === 'poem' ? page.poem.title : page.type === 'poet' ? pa
|
||||
{page.next ? <a href={page.next}>اگلا ›</a> : <span />}
|
||||
{page.prev ? <a href={page.prev}>‹ پچھلا</a> : <span />}
|
||||
</nav>
|
||||
{page.divan && (
|
||||
<p class="source divan-edit">
|
||||
دیوان کا ورژن {ud(page.divan.version)} · ترمیم: {page.divan.by}{page.divan.reviewedBy && <> · جائزہ: {page.divan.reviewedBy}</>} · اشاعت: {page.divan.publishedBy}
|
||||
· <bdi dir="ltr">{new Date(page.divan.at).toISOString().slice(0, 10)}</bdi>
|
||||
{page.divan.commit_url && <> · <a href={page.divan.commit_url}>تبدیلی دیکھیں</a></>}
|
||||
</p>
|
||||
)}
|
||||
{page.poem.source_url && (
|
||||
<p class="source">ماخذ: <a href={page.poem.source_url}>ویکی ماخذ</a> · {page.poet && <a href={page.poet.url}>{page.poet.nickname}</a>}</p>
|
||||
)}
|
||||
|
||||
Loading…
Reference in New Issue
Block a user