From 86b26ba737093ecead8e50324be104022ec5210e Mon Sep 17 00:00:00 2001 From: Anas Rashid Date: Sat, 10 Oct 2026 00:45:08 +0200 Subject: [PATCH] Statistics for admins (#86): page views, daily visitors and referring sites without cookies or stored addresses (one-day salted hash); moderation activity, accounts and library counts; /admin/stats; privacy page updated Co-Authored-By: Claude Opus 5.5 --- api/src/auth.ts | 2 +- api/src/server.ts | 2 + api/src/stats.test.ts | 42 +++++++++++++++ api/src/stats.ts | 63 +++++++++++++++++++++++ db/schema.sql | 6 +++ web/src/components/AdminNav.astro | 3 +- web/src/middleware.ts | 16 ++++++ web/src/pages/admin/stats.astro | 85 +++++++++++++++++++++++++++++++ web/src/pages/privacy.astro | 3 +- web/src/styles/global.css | 10 ++++ 10 files changed, 229 insertions(+), 3 deletions(-) create mode 100644 api/src/stats.test.ts create mode 100644 api/src/stats.ts create mode 100644 web/src/pages/admin/stats.astro diff --git a/api/src/auth.ts b/api/src/auth.ts index 8587688d..9b94057a 100644 --- a/api/src/auth.ts +++ b/api/src/auth.ts @@ -89,7 +89,7 @@ export async function sessionUser(req: FastifyRequest) { } const LOOPBACK = ['127.0.0.1', '::1', '::ffff:127.0.0.1']; -const fromSite = (req: FastifyRequest) => { +export const fromSite = (req: FastifyRequest) => { const key = process.env.DIVAN_SITE_KEY; if (!key) return LOOPBACK.includes(req.ip); const given = Buffer.from(String(req.headers['x-site-key'] ?? '')), want = Buffer.from(key); diff --git a/api/src/server.ts b/api/src/server.ts index 445742c9..a749d0b7 100644 --- a/api/src/server.ts +++ b/api/src/server.ts @@ -16,6 +16,7 @@ import { nameMatches } from './search.ts'; import { lookup, PUNCT } from './dictionary.ts'; import { authRoutes } from './auth.ts'; import { adminRoutes } from './admin.ts'; +import { statsRoutes } from './stats.ts'; import { permissionRoutes } from './permissions.ts'; import { libraryRoutes } from './library.ts'; import { moderationRoutes } from './moderation.ts'; @@ -220,6 +221,7 @@ siteRoutes(app); tagRoutes(app); ebookRoutes(app); feedRoutes(app); +statsRoutes(app); const port = Number(process.env.PORT ?? 4100); await app.listen({ port, host: process.env.HOST ?? '127.0.0.1' }); diff --git a/api/src/stats.test.ts b/api/src/stats.test.ts new file mode 100644 index 00000000..685a5648 --- /dev/null +++ b/api/src/stats.test.ts @@ -0,0 +1,42 @@ +import { test, after } from 'node:test'; +import assert from 'node:assert/strict'; +import Fastify from 'fastify'; +import { authRoutes } from './auth.ts'; +import { statsRoutes } from './stats.ts'; +import { pool } from './db.ts'; + +after(() => pool.end()); + +test('statistics: hits only from the site, counts without addresses, admin-only dashboard', async () => { + const app = Fastify(); + authRoutes(app); statsRoutes(app); + const run = Date.now(), path = `/stats-test-${run}`; + const hit = (body: object, headers: object = {}) => app.inject({ method: 'POST', url: '/api/stats/hit', payload: body, headers: headers as any }); + try { + process.env.DIVAN_SITE_KEY = 'site-key'; + assert.equal((await hit({ path, ip: '1.2.3.4', ua: 'x' })).statusCode, 403, 'not from the site'); + const site = { 'x-site-key': 'site-key' }; + for (const ip of ['1.2.3.4', '1.2.3.4', '5.6.7.8']) assert.equal((await hit({ path, ip, ua: 'Mozilla', ref: `ref-${run}.pk` }, site)).statusCode, 200); + const views = (await pool.query('SELECT views FROM stat_views WHERE path = $1', [path])).rows[0].views; + assert.equal(views, 3); + const hashes = (await pool.query('SELECT h FROM stat_visitors WHERE day = (now() AT TIME ZONE \'Asia/Karachi\')::date')).rows.map((r) => r.h); + assert.ok(!hashes.some((h) => h.includes('1.2.3.4')), 'no addresses stored'); + assert.equal((await pool.query('SELECT n FROM stat_referrers WHERE host = $1', [`ref-${run}.pk`])).rows[0].n, 3); + + const s = (await app.inject({ method: 'POST', url: '/api/auth/signup', payload: { email: `stats-${run}@divan.test`, password: 'pass-word-1' }, headers: { 'x-client-ip': `stats-${run}` } })).json(); + const get = () => app.inject({ method: 'GET', url: '/api/admin/stats?days=7', headers: { authorization: `Bearer ${s.token}` } }); + assert.equal((await get()).statusCode, 403, 'readers cannot see statistics'); + await pool.query(`UPDATE users SET role = 'admin' WHERE id = $1`, [s.user.id]); + const st = (await get()).json(); + assert.equal(st.daily.length, 7); + assert.ok(st.daily.at(-1).views >= 3 && st.daily.at(-1).visitors >= 2); + assert.ok(st.pages.some((p: any) => p.path === path && p.views === 3)); + assert.ok(st.accounts.roles.admin >= 1 && st.accounts.new >= 1); + } finally { + delete process.env.DIVAN_SITE_KEY; + await pool.query('DELETE FROM stat_views WHERE path = $1', [path]); + await pool.query('DELETE FROM stat_referrers WHERE host = $1', [`ref-${run}.pk`]); + await pool.query('DELETE FROM users WHERE email = $1', [`stats-${run}@divan.test`]); + await app.close(); + } +}); diff --git a/api/src/stats.ts b/api/src/stats.ts new file mode 100644 index 00000000..ff078e56 --- /dev/null +++ b/api/src/stats.ts @@ -0,0 +1,63 @@ +// Privacy-friendly statistics for admins (#86): page views, visitors and where they came from, plus moderation and +// account activity. No cookies and nothing personal is stored: the site reports a page view with the reader's address +// and browser, and only a one-day salted hash of them is kept (to count visitors per day); the salt is never stored. +// POST /api/stats/hit {path, ref, ip, ua} from the site only (x-site-key) +// GET /api/admin/stats?days=30 the admin dashboard's numbers +import { createHash, randomBytes } from 'node:crypto'; +import type { FastifyInstance } from 'fastify'; +import { pool } from './db.ts'; +import { fromSite } from './auth.ts'; +import { requireAdmin } from './admin.ts'; + +const TODAY = `(now() AT TIME ZONE 'Asia/Karachi')::date`; +// ponytail: the salt is per process; a restart mid-day counts a returning visitor once more that day +let salt = { day: '', key: randomBytes(32) }; +const visitor = (ip: string, ua: string) => { + const day = new Date().toLocaleDateString('en-CA', { timeZone: 'Asia/Karachi' }); + if (salt.day !== day) salt = { day, key: randomBytes(32) }; + return createHash('sha256').update(salt.key).update(ip).update('\n').update(ua).digest('base64url').slice(0, 22); +}; + +export function statsRoutes(app: FastifyInstance) { + app.post<{ Body: { path?: string; ref?: string; ip?: string; ua?: string } }>('/api/stats/hit', async (req, reply) => { + if (!fromSite(req)) return reply.code(403).send({ error: 'forbidden' }); + const { path = '', ref = '', ip = '', ua = '' } = req.body ?? {}; + if (!path.startsWith('/') || path.length > 300) return reply.code(400).send({ error: 'path' }); + await pool.query(`INSERT INTO stat_views (day, path, views) VALUES (${TODAY}, $1, 1) + ON CONFLICT (day, path) DO UPDATE SET views = stat_views.views + 1`, [path]); + await pool.query(`INSERT INTO stat_visitors (day, h) VALUES (${TODAY}, $1) ON CONFLICT DO NOTHING`, [visitor(ip, ua)]); + if (ref) await pool.query(`INSERT INTO stat_referrers (day, host, n) VALUES (${TODAY}, $1, 1) + ON CONFLICT (day, host) DO UPDATE SET n = stat_referrers.n + 1`, [ref.slice(0, 200)]); + return { ok: true }; + }); + + app.get<{ Querystring: { days?: string } }>('/api/admin/stats', async (req, reply) => { + if (!(await requireAdmin(req, reply))) return; + const days = Math.min(365, Math.max(1, Number(req.query.days) || 30)); + const since = `${TODAY} - ${days - 1}`; + const q = async (sql: string) => (await pool.query(sql)).rows; + const [daily, pages, referrers, actions, people, waiting, roles, accounts, library, content] = await Promise.all([ + q(`SELECT d::date::text AS day, coalesce(v.views, 0)::int AS views, coalesce(u.visitors, 0)::int AS visitors + FROM generate_series(${since}, ${TODAY}, interval '1 day') d + LEFT JOIN (SELECT day, sum(views) AS views FROM stat_views GROUP BY day) v ON v.day = d::date + LEFT JOIN (SELECT day, count(*) AS visitors FROM stat_visitors GROUP BY day) u ON u.day = d::date ORDER BY d`), + q(`SELECT s.path, sum(s.views)::int AS views, coalesce(p.title, c.title, t.nickname) AS title + FROM stat_views s LEFT JOIN poems p ON p.url = s.path LEFT JOIN categories c ON c.url = s.path LEFT JOIN poets t ON t.url = s.path + WHERE s.day >= ${since} GROUP BY s.path, p.title, c.title, t.nickname ORDER BY views DESC LIMIT 20`), + q(`SELECT host, sum(n)::int AS n FROM stat_referrers WHERE day >= ${since} GROUP BY host ORDER BY n DESC LIMIT 10`), + q(`SELECT action, count(*)::int AS n FROM revision_events WHERE at >= ${since} GROUP BY action ORDER BY n DESC`), + q(`SELECT actor_email AS email, count(*)::int AS n, count(*) FILTER (WHERE action = 'published')::int AS published + FROM revision_events WHERE at >= ${since} AND actor_email <> 'server' GROUP BY actor_email ORDER BY n DESC LIMIT 10`), + q(`SELECT status, count(*)::int AS n FROM revisions WHERE status IN ('submitted', 'approved') GROUP BY status`), + q(`SELECT role, count(*)::int AS n FROM users GROUP BY role`), + q(`SELECT count(*) FILTER (WHERE created_at >= ${since})::int AS new, + (SELECT count(DISTINCT user_id) FROM sessions WHERE created_at >= ${since})::int AS signed_in FROM users`), + q(`SELECT kind, count(*)::int AS n FROM library GROUP BY kind`), + q(`SELECT (SELECT count(*) FROM ebooks WHERE published)::int AS ebooks, (SELECT count(*) FROM tags)::int AS tags, + (SELECT count(*) FROM revisions WHERE status = 'published')::int AS versions`), + ]); + return { days, daily, pages, referrers, moderation: { actions, people, waiting: Object.fromEntries(waiting.map((r) => [r.status, r.n])) }, + accounts: { ...accounts[0], roles: Object.fromEntries(roles.map((r) => [r.role, r.n])) }, + library: Object.fromEntries(library.map((r) => [r.kind, r.n])), content: content[0] }; + }); +} diff --git a/db/schema.sql b/db/schema.sql index d26844ba..f90a5d3c 100644 --- a/db/schema.sql +++ b/db/schema.sql @@ -253,3 +253,9 @@ ALTER TABLE ebooks ADD COLUMN IF NOT EXISTS coauthor_ids integer[] NOT NULL DEFA CREATE INDEX IF NOT EXISTS ebooks_coauthors ON ebooks USING gin (coauthor_ids); ALTER TABLE entity_tags DROP CONSTRAINT IF EXISTS entity_tags_entity_check; ALTER TABLE entity_tags ADD CONSTRAINT entity_tags_entity_check CHECK (entity IN ('category', 'work', 'ebook')); + +-- visitor statistics (api/src/stats.ts, #86): counts only; no cookies, no addresses. A visitor is a hash of their +-- address and browser with a salt that lives one day in memory, so it cannot be traced back or followed across days. +CREATE TABLE IF NOT EXISTS stat_views (day date NOT NULL, path text NOT NULL, views integer NOT NULL DEFAULT 0, PRIMARY KEY (day, path)); +CREATE TABLE IF NOT EXISTS stat_visitors (day date NOT NULL, h text NOT NULL, PRIMARY KEY (day, h)); +CREATE TABLE IF NOT EXISTS stat_referrers (day date NOT NULL, host text NOT NULL, n integer NOT NULL DEFAULT 0, PRIMARY KEY (day, host)); diff --git a/web/src/components/AdminNav.astro b/web/src/components/AdminNav.astro index 450cd8a0..4d429a21 100644 --- a/web/src/components/AdminNav.astro +++ b/web/src/components/AdminNav.astro @@ -1,7 +1,8 @@ --- -const { current } = Astro.props as { current: 'users' | 'audit' }; +const { current } = Astro.props as { current: 'users' | 'audit' | 'stats' }; --- diff --git a/web/src/middleware.ts b/web/src/middleware.ts index a24b0bad..1c16449b 100644 --- a/web/src/middleware.ts +++ b/web/src/middleware.ts @@ -4,6 +4,21 @@ import { defineMiddleware } from 'astro:middleware'; import { auth, COOKIE } from './lib/auth'; +// visitor statistics (#86): each reading page shown is reported to the API without waiting; the API keeps only counts +// and a one-day hash. Bots, link previews, private pages (accounts, moderation) and errors are not counted. +const API = process.env.API_URL ?? 'http://127.0.0.1:4100'; +const PRIVATE = /^\/(api|mod|admin|account|library|notes|words|writers|signin|signup|_astro|ebooks\/file|health)(\/|$)/; +const BOT = /bot|crawl|spider|slurp|preview|fetch|curl|wget|python|java\/|go-http|headless|lighthouse|monitor/i; +function count(ctx: any, res: Response) { + const ua = ctx.request.headers.get('user-agent') ?? '', path = ctx.url.pathname; + if (ctx.request.method !== 'GET' || res.status !== 200 || !res.headers.get('content-type')?.includes('text/html') || !ua || BOT.test(ua) || PRIVATE.test(path)) return; + let ip = '', ref = ''; + try { ip = ctx.clientAddress; } catch {} + try { const h = new URL(ctx.request.headers.get('referer') ?? '').hostname; if (h !== ctx.url.hostname) ref = h.replace(/^www\./, ''); } catch {} + fetch(`${API}/api/stats/hit`, { method: 'POST', headers: { 'content-type': 'application/json', ...(process.env.DIVAN_SITE_KEY && { 'x-site-key': process.env.DIVAN_SITE_KEY }) }, + body: JSON.stringify({ path, ref, ip, ua }) }).catch(() => {}); +} + export const onRequest = defineMiddleware(async (ctx, next) => { const token = ctx.cookies.get(COOKIE)?.value; ctx.locals.user = null; @@ -21,6 +36,7 @@ export const onRequest = defineMiddleware(async (ctx, next) => { msg: String(err?.message ?? err), stack: String(err?.stack ?? '').split('\n').slice(0, 8).join('\n') })); throw err; } + count(ctx, res); if (ctx.cookies.get('divan-digits')?.value !== 'latn' || !res.headers.get('content-type')?.includes('text/html')) return res; // text only: scripts and styles are left alone (the page's own scripts look for Eastern digits) const html = (await res.text()).split(/(|)/) diff --git a/web/src/pages/admin/stats.astro b/web/src/pages/admin/stats.astro new file mode 100644 index 00000000..0ed4080f --- /dev/null +++ b/web/src/pages/admin/stats.astro @@ -0,0 +1,85 @@ +--- +// Admin: statistics (#86): readers (page views and visitors per day, top pages, where they came from), moderation +// (steps taken, by whom, waiting) and accounts. Counts only: nothing here identifies a reader (see /privacy). +import Base from '../../layouts/Base.astro'; +import AdminNav from '../../components/AdminNav.astro'; +import { asUser, COOKIE } from '../../lib/auth'; +import { ud } from '../../lib/urdu'; +import { ROLE } from '../../lib/roles'; +import { EVENT } from '../../lib/mod'; + +const me = Astro.locals.user; +if (!me) return Astro.redirect('/signin?next=/admin/stats'); +if (me.role !== 'admin') return new Response('صرف ایڈمن کے لیے', { status: 403 }); +const days = [7, 30, 90, 365].includes(Number(Astro.url.searchParams.get('days'))) ? Number(Astro.url.searchParams.get('days')) : 30; +const s = (await asUser(Astro.cookies.get(COOKIE)!.value, `/api/admin/stats?days=${days}`)).data; +const sum = (k: 'views' | 'visitors') => s.daily.reduce((n: number, d: any) => n + d[k], 0); +const max = Math.max(1, ...s.daily.map((d: any) => d.views)); +const LIB: Record = { poet: 'شاعر', category: 'کتابیں', poem: 'کلام', couplet: 'اشعار', phrase: 'ٹکڑے', word: 'الفاظ', page: 'صفحات' }; +const label = (d: string) => ud(d.slice(5).replace('-', '/')); +--- + +

ایڈمن

+ + + + +
+

روزانہ

+ +

صفحات · قارئین · {label(s.daily[0].day)} تا {label(s.daily.at(-1).day)}

+
+ +
+
+

سب سے زیادہ پڑھے گئے

+ {s.pages.length ? + {s.pages.map((p: any) => )} +
{p.title ?? p.path}{ud(p.views)}
:

ابھی کچھ نہیں۔

} +
+
+

کہاں سے آئے

+ {s.referrers.length ? + {s.referrers.map((r: any) => )} +
{r.host}{ud(r.n)}
:

ابھی کچھ نہیں (سیدھے آنے والے یہاں نہیں گنے جاتے)۔

} +
+
+

موڈریشن

+

جائزے کے منتظر: {ud(s.moderation.waiting.submitted ?? 0)} · اشاعت کے منتظر: {ud(s.moderation.waiting.approved ?? 0)} + · کل شائع شدہ ورژن: {ud(s.content.versions)}

+ + {s.moderation.actions.map((a: any) => )} +
{EVENT[a.action] ?? a.action}{ud(a.n)}
+ {s.moderation.people.length > 0 && <> +

سب سے زیادہ کام

+ + {s.moderation.people.map((p: any) => )} +
موڈریٹرقدماشاعتیں
{p.email}{ud(p.n)}{ud(p.published)}
+ } +
+
+

اکاؤنٹ اور محفوظ شدہ

+

اس مدت میں لاگ ان کرنے والے: {ud(s.accounts.signed_in)}

+ + {Object.entries(s.accounts.roles).map(([r, n]: any) => )} + {Object.entries(s.library).map(([k, n]: any) => )} + + +
{ROLE[r] ?? r}{ud(n)}
محفوظ: {LIB[k] ?? k}{ud(n)}
ای بکس{ud(s.content.ebooks)}
ٹیگ{ud(s.content.tags)}
+
+
+

ان اعداد میں کسی قاری کی شناخت نہیں: نہ کوکی، نہ پتا محفوظ ہوتا ہے (رازداری)۔

+ diff --git a/web/src/pages/privacy.astro b/web/src/pages/privacy.astro index ef347f71..ae19a857 100644 --- a/web/src/pages/privacy.astro +++ b/web/src/pages/privacy.astro @@ -6,7 +6,7 @@ import { brand } from '../lib/brand';

رازداری

-

آخری تبدیلی: ۹ اکتوبر ۲۰۲۶ء

+

آخری تبدیلی: ۱۰ اکتوبر ۲۰۲۶ء

{brand.name} پڑھنے کے لیے کسی اکاؤنٹ کی ضرورت نہیں۔ ہم صرف وہی محفوظ رکھتے ہیں جو سائٹ کے چلنے کے لیے ضروری ہے، اور اسے کسی کو بیچتے یا دیتے نہیں۔ نہ اشتہار ہیں، نہ اشتہاری یا جاسوس کوکیز، نہ کسی اور کمپنی کے ٹریکر۔ @@ -16,6 +16,7 @@ import { brand } from '../lib/brand';

  • آپ کی پسند (خط، رنگ، روشنی، پڑھنے کا انداز) آپ کے اپنے براؤزر میں رہتی ہے، ہمارے سرور پر نہیں۔ ہندسوں کی پسند ایک چھوٹی کوکی میں رہتی ہے تاکہ صفحہ انھی ہندسوں کے ساتھ آئے۔
  • لاگ ان اور اکاؤنٹ بنانے کی کوششوں کی حد کے لیے سرور آپ کا انٹرنیٹ پتہ (IP) تھوڑی دیر کے لیے یاد رکھتا ہے، اور غلطیوں کے ریکارڈ میں صفحے کا پتہ اور وقت درج ہوتا ہے۔ یہ صرف سائٹ کی حفاظت اور خرابیاں ٹھیک کرنے کے لیے ہیں اور کچھ دن بعد مٹ جاتے ہیں۔
    • +
  • گنتی: کون سا صفحہ کتنی بار پڑھا گیا، روزانہ کتنے قارئین آئے، اور کس ویب سائٹ سے (صرف اس کا نام) — صرف گنتی، بغیر کوکی کے۔ قارئین گننے کے لیے انٹرنیٹ پتے اور براؤزر کی ایک نہ پلٹنے والی شکل بنتی ہے جس کی کنجی ہر روز بدلتی ہے اور کہیں محفوظ نہیں ہوتی، اس لیے کسی کو پہچانا یا ایک دن سے دوسرے دن تک پیچھا نہیں کیا جا سکتا۔

اکاؤنٹ کے ساتھ

diff --git a/web/src/styles/global.css b/web/src/styles/global.css index dda64e75..d5f6b808 100644 --- a/web/src/styles/global.css +++ b/web/src/styles/global.css @@ -930,3 +930,13 @@ p, li { text-wrap: pretty; } .help h2::after { display: none; } .help-icon { display: grid; place-items: center; width: 34px; height: 34px; border-radius: 50%; background: var(--accent-soft); color: var(--accent); flex: none; } .help p { margin: 0 0 8px; line-height: 2; } + +/* admin statistics (#86): daily bars, views behind and visitors in front */ +.bars { display: flex; align-items: flex-end; gap: 2px; height: 160px; padding: 8px; border: 1px solid var(--border); border-radius: 10px; direction: ltr; } +.bar { flex: 1; height: 100%; position: relative; display: flex; align-items: flex-end; } +.bar span, .bar i { position: absolute; inset-inline: 0; bottom: 0; border-radius: 3px 3px 0 0; } +.bar span, .key-views { background: color-mix(in srgb, var(--accent) 35%, transparent); } +.bar i, .key-visitors { background: var(--accent); inset-inline: 25%; } +.key-views, .key-visitors { display: inline-block; width: 12px; height: 12px; border-radius: 3px; vertical-align: middle; } +.stats-grid { display: grid; grid-template-columns: repeat(auto-fit, minmax(min(100%, 340px), 1fr)); gap: 0 24px; } +.stats-range { display: flex; flex-wrap: wrap; gap: 8px; justify-content: center; margin: 10px 0; } -- 2.47.3