Divan v2: Node.js API + PostgreSQL (proof of concept)
- db/schema.sql: poets, categories, poems, verses; normalised search_text with pg_trgm index - api/: Fastify + pg, Node native TypeScript (no build step) - import.ts: loads divan-data (folder or CDN) with upserts; full import in ~36 s - server.ts: /api/poets, /api/page?url=, /api/search (Urdu-normalised ILIKE), /health - urdu.ts (+ test): normaliser ported from the .NET version Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Anas Rashid
parent
4a1d2e0642
commit
c450ec010a
1
api/.gitignore
vendored
Normal file
1
api/.gitignore
vendored
Normal file
@ -0,0 +1 @@
|
||||
node_modules/
|
||||
777
api/package-lock.json
generated
Normal file
777
api/package-lock.json
generated
Normal file
@ -0,0 +1,777 @@
|
||||
{
|
||||
"name": "divan-api",
|
||||
"lockfileVersion": 3,
|
||||
"requires": true,
|
||||
"packages": {
|
||||
"": {
|
||||
"name": "divan-api",
|
||||
"dependencies": {
|
||||
"fastify": "^5.12.5",
|
||||
"pg": "^8.23.1"
|
||||
},
|
||||
"engines": {
|
||||
"node": ">=24"
|
||||
}
|
||||
},
|
||||
"node_modules/@fastify/ajv-compiler": {
|
||||
"version": "4.0.6",
|
||||
"resolved": "https://registry.npmjs.org/@fastify/ajv-compiler/-/ajv-compiler-4.0.6.tgz",
|
||||
"integrity": "sha512-NtuzM0SfaMJbGlnjr9LWQUN5LzgSrbB8tf/wRZNas+4E1O/Nmzl53e7ruT61HDZyRCJGC6FxIogmNZO1c5ETBA==",
|
||||
"funding": [
|
||||
{
|
||||
"type": "github",
|
||||
"url": "https://github.com/sponsors/fastify"
|
||||
},
|
||||
{
|
||||
"type": "opencollective",
|
||||
"url": "https://opencollective.com/fastify"
|
||||
}
|
||||
],
|
||||
"license": "MIT",
|
||||
"dependencies": {
|
||||
"ajv": "^8.12.0",
|
||||
"ajv-formats": "^3.0.1",
|
||||
"fast-uri": "^4.0.0"
|
||||
}
|
||||
},
|
||||
"node_modules/@fastify/error": {
|
||||
"version": "4.2.0",
|
||||
"resolved": "https://registry.npmjs.org/@fastify/error/-/error-4.2.0.tgz",
|
||||
"integrity": "sha512-RSo3sVDXfHskiBZKBPRgnQTtIqpi/7zhJOEmAxCiBcM7d0uwdGdxLlsCaLzGs8v8NnxIRlfG0N51p5yFaOentQ==",
|
||||
"funding": [
|
||||
{
|
||||
"type": "github",
|
||||
"url": "https://github.com/sponsors/fastify"
|
||||
},
|
||||
{
|
||||
"type": "opencollective",
|
||||
"url": "https://opencollective.com/fastify"
|
||||
}
|
||||
],
|
||||
"license": "MIT"
|
||||
},
|
||||
"node_modules/@fastify/fast-json-stringify-compiler": {
|
||||
"version": "5.1.0",
|
||||
"resolved": "https://registry.npmjs.org/@fastify/fast-json-stringify-compiler/-/fast-json-stringify-compiler-5.1.0.tgz",
|
||||
"integrity": "sha512-PxcYtKLbQ8Z+yApiqjK8FwxIwvEj38k2OiLc17u8dkJSlmfi2wHHPaSnaoqBPQqtvF8YVsDgDpP2snDCfFrpfw==",
|
||||
"funding": [
|
||||
{
|
||||
"type": "github",
|
||||
"url": "https://github.com/sponsors/fastify"
|
||||
},
|
||||
{
|
||||
"type": "opencollective",
|
||||
"url": "https://opencollective.com/fastify"
|
||||
}
|
||||
],
|
||||
"license": "MIT",
|
||||
"dependencies": {
|
||||
"fast-json-stringify": "^7.0.0"
|
||||
}
|
||||
},
|
||||
"node_modules/@fastify/forwarded": {
|
||||
"version": "3.0.2",
|
||||
"resolved": "https://registry.npmjs.org/@fastify/forwarded/-/forwarded-3.0.2.tgz",
|
||||
"integrity": "sha512-NE8HgKLgYejV9lDpqkEFaDKMLYelJBVfHekhB0UKvX0ghagXRJqg68feg8er1NPXxG4N9i6vPxzt8E+3wHfcmA==",
|
||||
"funding": [
|
||||
{
|
||||
"type": "github",
|
||||
"url": "https://github.com/sponsors/fastify"
|
||||
},
|
||||
{
|
||||
"type": "opencollective",
|
||||
"url": "https://opencollective.com/fastify"
|
||||
}
|
||||
],
|
||||
"license": "MIT"
|
||||
},
|
||||
"node_modules/@fastify/merge-json-schemas": {
|
||||
"version": "0.2.1",
|
||||
"resolved": "https://registry.npmjs.org/@fastify/merge-json-schemas/-/merge-json-schemas-0.2.1.tgz",
|
||||
"integrity": "sha512-OA3KGBCy6KtIvLf8DINC5880o5iBlDX4SxzLQS8HorJAbqluzLRn80UXU0bxZn7UOFhFgpRJDasfwn9nG4FG4A==",
|
||||
"funding": [
|
||||
{
|
||||
"type": "github",
|
||||
"url": "https://github.com/sponsors/fastify"
|
||||
},
|
||||
{
|
||||
"type": "opencollective",
|
||||
"url": "https://opencollective.com/fastify"
|
||||
}
|
||||
],
|
||||
"license": "MIT",
|
||||
"dependencies": {
|
||||
"dequal": "^2.0.3"
|
||||
}
|
||||
},
|
||||
"node_modules/@fastify/proxy-addr": {
|
||||
"version": "5.1.1",
|
||||
"resolved": "https://registry.npmjs.org/@fastify/proxy-addr/-/proxy-addr-5.1.1.tgz",
|
||||
"integrity": "sha512-zv07Y9GEuDsJPegZoDFd4SDWaZOW8N2pa0GSrYmKpId/tjt1Hgo3BjZBVjdVpfVrHaA+Qv5jawtS2O50J5xM9g==",
|
||||
"funding": [
|
||||
{
|
||||
"type": "github",
|
||||
"url": "https://github.com/sponsors/fastify"
|
||||
},
|
||||
{
|
||||
"type": "opencollective",
|
||||
"url": "https://opencollective.com/fastify"
|
||||
}
|
||||
],
|
||||
"license": "MIT",
|
||||
"dependencies": {
|
||||
"@fastify/forwarded": "^3.0.0",
|
||||
"ipaddr.js": "^2.1.0"
|
||||
}
|
||||
},
|
||||
"node_modules/@pinojs/redact": {
|
||||
"version": "0.4.0",
|
||||
"resolved": "https://registry.npmjs.org/@pinojs/redact/-/redact-0.4.0.tgz",
|
||||
"integrity": "sha512-k2ENnmBugE/rzQfEcdWHcCY+/FM3VLzH9cYEsbdsoqrvzAKRhUZeRNhAZvB8OitQJ1TBed3yqWtdjzS6wJKBwg==",
|
||||
"license": "MIT"
|
||||
},
|
||||
"node_modules/abstract-logging": {
|
||||
"version": "2.0.1",
|
||||
"resolved": "https://registry.npmjs.org/abstract-logging/-/abstract-logging-2.0.1.tgz",
|
||||
"integrity": "sha512-2BjRTZxTPvheOvGbBslFSYOUkr+SjPtOnrLP33f+VIWLzezQpZcqVg7ja3L4dBXmzzgwT+a029jRx5PCi3JuiA==",
|
||||
"license": "MIT"
|
||||
},
|
||||
"node_modules/ajv": {
|
||||
"version": "8.20.0",
|
||||
"resolved": "https://registry.npmjs.org/ajv/-/ajv-8.20.0.tgz",
|
||||
"integrity": "sha512-Thbli+OlOj+iMPYFBVBfJ3OmCAnaSyNn4M1vz9T6Gka5Jt9ba/HIR56joy65tY6kx/FCF5VXNB819Y7/GUrBGA==",
|
||||
"license": "MIT",
|
||||
"dependencies": {
|
||||
"fast-deep-equal": "^3.1.3",
|
||||
"fast-uri": "^3.0.1",
|
||||
"json-schema-traverse": "^1.0.0",
|
||||
"require-from-string": "^2.0.2"
|
||||
},
|
||||
"funding": {
|
||||
"type": "github",
|
||||
"url": "https://github.com/sponsors/epoberezkin"
|
||||
}
|
||||
},
|
||||
"node_modules/ajv-formats": {
|
||||
"version": "3.0.1",
|
||||
"resolved": "https://registry.npmjs.org/ajv-formats/-/ajv-formats-3.0.1.tgz",
|
||||
"integrity": "sha512-8iUql50EUR+uUcdRQ3HDqa6EVyo3docL8g5WJ3FNcWmu62IbkGUue/pEyLBW8VGKKucTPgqeks4fIU1DA4yowQ==",
|
||||
"license": "MIT",
|
||||
"dependencies": {
|
||||
"ajv": "^8.0.0"
|
||||
},
|
||||
"peerDependencies": {
|
||||
"ajv": "^8.0.0"
|
||||
},
|
||||
"peerDependenciesMeta": {
|
||||
"ajv": {
|
||||
"optional": true
|
||||
}
|
||||
}
|
||||
},
|
||||
"node_modules/ajv/node_modules/fast-uri": {
|
||||
"version": "3.1.8",
|
||||
"resolved": "https://registry.npmjs.org/fast-uri/-/fast-uri-3.1.8.tgz",
|
||||
"integrity": "sha512-GZMtZUTNRpOVIECoXwLNZS5xUGE+mVNbTB8h/7Rwh2TFWcBQiPzTgyZi05BF9UMZKkLJv8XBRJTlU7zg8+ZfMg==",
|
||||
"funding": [
|
||||
{
|
||||
"type": "github",
|
||||
"url": "https://github.com/sponsors/fastify"
|
||||
},
|
||||
{
|
||||
"type": "opencollective",
|
||||
"url": "https://opencollective.com/fastify"
|
||||
}
|
||||
],
|
||||
"license": "BSD-3-Clause"
|
||||
},
|
||||
"node_modules/atomic-sleep": {
|
||||
"version": "1.0.0",
|
||||
"resolved": "https://registry.npmjs.org/atomic-sleep/-/atomic-sleep-1.0.0.tgz",
|
||||
"integrity": "sha512-kNOjDqAh7px0XWNI+4QbzoiR/nTkHAWNud2uvnJquD1/x5a7EQZMJT0AczqK0Qn67oY/TTQ1LbUKajZpp3I9tQ==",
|
||||
"license": "MIT",
|
||||
"engines": {
|
||||
"node": ">=8.0.0"
|
||||
}
|
||||
},
|
||||
"node_modules/avvio": {
|
||||
"version": "9.3.0",
|
||||
"resolved": "https://registry.npmjs.org/avvio/-/avvio-9.3.0.tgz",
|
||||
"integrity": "sha512-g2tQ7LE7oOSqDfwEm3M+ZCMTJc7KiZCdJ4UwyZJb5ckTKyYu50OYmvv0mCFXPuYXoM4zkSt8zM9XQ9KCvxA74A==",
|
||||
"funding": [
|
||||
{
|
||||
"type": "github",
|
||||
"url": "https://github.com/sponsors/fastify"
|
||||
},
|
||||
{
|
||||
"type": "opencollective",
|
||||
"url": "https://opencollective.com/fastify"
|
||||
}
|
||||
],
|
||||
"license": "MIT",
|
||||
"dependencies": {
|
||||
"@fastify/error": "^4.0.0",
|
||||
"fastq": "^1.17.1"
|
||||
}
|
||||
},
|
||||
"node_modules/cookie": {
|
||||
"version": "1.1.1",
|
||||
"resolved": "https://registry.npmjs.org/cookie/-/cookie-1.1.1.tgz",
|
||||
"integrity": "sha512-ei8Aos7ja0weRpFzJnEA9UHJ/7XQmqglbRwnf2ATjcB9Wq874VKH9kfjjirM6UhU2/E5fFYadylyhFldcqSidQ==",
|
||||
"license": "MIT",
|
||||
"engines": {
|
||||
"node": ">=18"
|
||||
},
|
||||
"funding": {
|
||||
"type": "opencollective",
|
||||
"url": "https://opencollective.com/express"
|
||||
}
|
||||
},
|
||||
"node_modules/dequal": {
|
||||
"version": "2.0.3",
|
||||
"resolved": "https://registry.npmjs.org/dequal/-/dequal-2.0.3.tgz",
|
||||
"integrity": "sha512-0je+qPKHEMohvfRTCEo3CrPG6cAzAYgmzKyxRiYSSDkS6eGJdyVJm7WaYA5ECaAD9wLB2T4EEeymA5aFVcYXCA==",
|
||||
"license": "MIT",
|
||||
"engines": {
|
||||
"node": ">=6"
|
||||
}
|
||||
},
|
||||
"node_modules/fast-decode-uri-component": {
|
||||
"version": "1.0.1",
|
||||
"resolved": "https://registry.npmjs.org/fast-decode-uri-component/-/fast-decode-uri-component-1.0.1.tgz",
|
||||
"integrity": "sha512-WKgKWg5eUxvRZGwW8FvfbaH7AXSh2cL+3j5fMGzUMCxWBJ3dV3a7Wz8y2f/uQ0e3B6WmodD3oS54jTQ9HVTIIg==",
|
||||
"license": "MIT"
|
||||
},
|
||||
"node_modules/fast-deep-equal": {
|
||||
"version": "3.1.3",
|
||||
"resolved": "https://registry.npmjs.org/fast-deep-equal/-/fast-deep-equal-3.1.3.tgz",
|
||||
"integrity": "sha512-f3qQ9oQy9j2AhBe/H9VC91wLmKBCCU/gDOnKNAYG5hswO7BLKj09Hc5HYNz9cGI++xlpDCIgDaitVs03ATR84Q==",
|
||||
"license": "MIT"
|
||||
},
|
||||
"node_modules/fast-json-stringify": {
|
||||
"version": "7.0.1",
|
||||
"resolved": "https://registry.npmjs.org/fast-json-stringify/-/fast-json-stringify-7.0.1.tgz",
|
||||
"integrity": "sha512-eRSayARSbbwlBjpP4vnTTIRD5QPcIrmihPxDeN1DtKnHPg66UuJLx+8hlK1kaFdjvzyQ/dzALoi4vwAQ+T+iZA==",
|
||||
"funding": [
|
||||
{
|
||||
"type": "github",
|
||||
"url": "https://github.com/sponsors/fastify"
|
||||
},
|
||||
{
|
||||
"type": "opencollective",
|
||||
"url": "https://opencollective.com/fastify"
|
||||
}
|
||||
],
|
||||
"license": "MIT",
|
||||
"dependencies": {
|
||||
"@fastify/merge-json-schemas": "^0.2.0",
|
||||
"ajv": "^8.12.0",
|
||||
"ajv-formats": "^3.0.1",
|
||||
"fast-uri": "^4.0.0",
|
||||
"json-schema-ref-resolver": "^3.0.0",
|
||||
"rfdc": "^1.2.0"
|
||||
}
|
||||
},
|
||||
"node_modules/fast-querystring": {
|
||||
"version": "1.1.2",
|
||||
"resolved": "https://registry.npmjs.org/fast-querystring/-/fast-querystring-1.1.2.tgz",
|
||||
"integrity": "sha512-g6KuKWmFXc0fID8WWH0jit4g0AGBoJhCkJMb1RmbsSEUNvQ+ZC8D6CUZ+GtF8nMzSPXnhiePyyqqipzNNEnHjg==",
|
||||
"license": "MIT",
|
||||
"dependencies": {
|
||||
"fast-decode-uri-component": "^1.0.1"
|
||||
}
|
||||
},
|
||||
"node_modules/fast-uri": {
|
||||
"version": "4.2.1",
|
||||
"resolved": "https://registry.npmjs.org/fast-uri/-/fast-uri-4.2.1.tgz",
|
||||
"integrity": "sha512-TmHQgewjHtMq1E5QKA0tOE0yeYGQs25KZC/ziJpubRtWI15W92e6vPFydWOeZBVROSHBYw/QhD9d5OefdD6LDg==",
|
||||
"funding": [
|
||||
{
|
||||
"type": "github",
|
||||
"url": "https://github.com/sponsors/fastify"
|
||||
},
|
||||
{
|
||||
"type": "opencollective",
|
||||
"url": "https://opencollective.com/fastify"
|
||||
}
|
||||
],
|
||||
"license": "BSD-3-Clause"
|
||||
},
|
||||
"node_modules/fastify": {
|
||||
"version": "5.12.5",
|
||||
"resolved": "https://registry.npmjs.org/fastify/-/fastify-5.12.5.tgz",
|
||||
"integrity": "sha512-OB2k1dlxs5/NAABqeKV2FUHkSD2BbENsCak8yULVcymn3fHIPDVa9TI3SDnJSWYSllZmSYuZXy2gTnsT+Sut1A==",
|
||||
"funding": [
|
||||
{
|
||||
"type": "github",
|
||||
"url": "https://github.com/sponsors/fastify"
|
||||
},
|
||||
{
|
||||
"type": "opencollective",
|
||||
"url": "https://opencollective.com/fastify"
|
||||
}
|
||||
],
|
||||
"license": "MIT",
|
||||
"dependencies": {
|
||||
"@fastify/ajv-compiler": "^4.0.5",
|
||||
"@fastify/error": "^4.0.0",
|
||||
"@fastify/fast-json-stringify-compiler": "^5.0.0",
|
||||
"@fastify/proxy-addr": "^5.0.0",
|
||||
"abstract-logging": "^2.0.1",
|
||||
"avvio": "^9.0.0",
|
||||
"fast-json-stringify": "^7.0.0",
|
||||
"find-my-way": "^9.6.0",
|
||||
"light-my-request": "^6.0.0",
|
||||
"pino": "^9.14.0 || ^10.1.0",
|
||||
"process-warning": "^5.1.0",
|
||||
"rfdc": "^1.3.1",
|
||||
"secure-json-parse": "^4.0.0",
|
||||
"semver": "^7.6.0",
|
||||
"toad-cache": "^3.7.0"
|
||||
}
|
||||
},
|
||||
"node_modules/fastq": {
|
||||
"version": "1.20.3",
|
||||
"resolved": "https://registry.npmjs.org/fastq/-/fastq-1.20.3.tgz",
|
||||
"integrity": "sha512-XKv5nnLs6nLF71NgiKJLIZFLkPyIEuOselLG7ujZnGrRfQK8HpvY+WqKhAJUAdLomwVHErVS4LfxFlPq0/FTAw==",
|
||||
"license": "ISC",
|
||||
"dependencies": {
|
||||
"reusify": "^1.0.4"
|
||||
}
|
||||
},
|
||||
"node_modules/find-my-way": {
|
||||
"version": "9.9.0",
|
||||
"resolved": "https://registry.npmjs.org/find-my-way/-/find-my-way-9.9.0.tgz",
|
||||
"integrity": "sha512-sJsgZ1sQH2UDuowPuMKg8az7Qc8F0jnj+SKkFWU/+T0xcFlgV5skgXOGUqmQzOdmW6ALA7AhJINWx3qFBkbLHA==",
|
||||
"license": "MIT",
|
||||
"dependencies": {
|
||||
"fast-deep-equal": "^3.1.3",
|
||||
"fast-querystring": "^1.0.0",
|
||||
"safe-regex2": "^5.0.0"
|
||||
},
|
||||
"engines": {
|
||||
"node": ">=20"
|
||||
}
|
||||
},
|
||||
"node_modules/ipaddr.js": {
|
||||
"version": "2.5.0",
|
||||
"resolved": "https://registry.npmjs.org/ipaddr.js/-/ipaddr.js-2.5.0.tgz",
|
||||
"integrity": "sha512-aq+t5NAc+cS6rZQQVWC2x98CPqGtKKTMDd4Gaodv0wShnItdKg/51djkGJ1hqH+Oy0ivDftCbSLCQob8zso01w==",
|
||||
"license": "MIT",
|
||||
"engines": {
|
||||
"node": ">= 10"
|
||||
}
|
||||
},
|
||||
"node_modules/json-schema-ref-resolver": {
|
||||
"version": "3.0.0",
|
||||
"resolved": "https://registry.npmjs.org/json-schema-ref-resolver/-/json-schema-ref-resolver-3.0.0.tgz",
|
||||
"integrity": "sha512-hOrZIVL5jyYFjzk7+y7n5JDzGlU8rfWDuYyHwGa2WA8/pcmMHezp2xsVwxrebD/Q9t8Nc5DboieySDpCp4WG4A==",
|
||||
"funding": [
|
||||
{
|
||||
"type": "github",
|
||||
"url": "https://github.com/sponsors/fastify"
|
||||
},
|
||||
{
|
||||
"type": "opencollective",
|
||||
"url": "https://opencollective.com/fastify"
|
||||
}
|
||||
],
|
||||
"license": "MIT",
|
||||
"dependencies": {
|
||||
"dequal": "^2.0.3"
|
||||
}
|
||||
},
|
||||
"node_modules/json-schema-traverse": {
|
||||
"version": "1.0.0",
|
||||
"resolved": "https://registry.npmjs.org/json-schema-traverse/-/json-schema-traverse-1.0.0.tgz",
|
||||
"integrity": "sha512-NM8/P9n3XjXhIZn1lLhkFaACTOURQXjWhV4BA/RnOv8xvgqtqpAX9IO4mRQxSx1Rlo4tqzeqb0sOlruaOy3dug==",
|
||||
"license": "MIT"
|
||||
},
|
||||
"node_modules/light-my-request": {
|
||||
"version": "6.6.0",
|
||||
"resolved": "https://registry.npmjs.org/light-my-request/-/light-my-request-6.6.0.tgz",
|
||||
"integrity": "sha512-CHYbu8RtboSIoVsHZ6Ye4cj4Aw/yg2oAFimlF7mNvfDV192LR7nDiKtSIfCuLT7KokPSTn/9kfVLm5OGN0A28A==",
|
||||
"funding": [
|
||||
{
|
||||
"type": "github",
|
||||
"url": "https://github.com/sponsors/fastify"
|
||||
},
|
||||
{
|
||||
"type": "opencollective",
|
||||
"url": "https://opencollective.com/fastify"
|
||||
}
|
||||
],
|
||||
"license": "BSD-3-Clause",
|
||||
"dependencies": {
|
||||
"cookie": "^1.0.1",
|
||||
"process-warning": "^4.0.0",
|
||||
"set-cookie-parser": "^2.6.0"
|
||||
}
|
||||
},
|
||||
"node_modules/light-my-request/node_modules/process-warning": {
|
||||
"version": "4.0.1",
|
||||
"resolved": "https://registry.npmjs.org/process-warning/-/process-warning-4.0.1.tgz",
|
||||
"integrity": "sha512-3c2LzQ3rY9d0hc1emcsHhfT9Jwz0cChib/QN89oME2R451w5fy3f0afAhERFZAwrbDU43wk12d0ORBpDVME50Q==",
|
||||
"funding": [
|
||||
{
|
||||
"type": "github",
|
||||
"url": "https://github.com/sponsors/fastify"
|
||||
},
|
||||
{
|
||||
"type": "opencollective",
|
||||
"url": "https://opencollective.com/fastify"
|
||||
}
|
||||
],
|
||||
"license": "MIT"
|
||||
},
|
||||
"node_modules/on-exit-leak-free": {
|
||||
"version": "2.1.2",
|
||||
"resolved": "https://registry.npmjs.org/on-exit-leak-free/-/on-exit-leak-free-2.1.2.tgz",
|
||||
"integrity": "sha512-0eJJY6hXLGf1udHwfNftBqH+g73EU4B504nZeKpz1sYRKafAghwxEJunB2O7rDZkL4PGfsMVnTXZ2EjibbqcsA==",
|
||||
"license": "MIT",
|
||||
"engines": {
|
||||
"node": ">=14.0.0"
|
||||
}
|
||||
},
|
||||
"node_modules/pg": {
|
||||
"version": "8.23.1",
|
||||
"resolved": "https://registry.npmjs.org/pg/-/pg-8.23.1.tgz",
|
||||
"integrity": "sha512-aL96AHANtWjPLDOLqnhx+ngp9+UK7ETEU8VJrDCGvsSSi/mGLcWYsS6Herg7lmaBJe4uwrfqsa7gTEFaSizDoQ==",
|
||||
"license": "MIT",
|
||||
"dependencies": {
|
||||
"pg-connection-string": "^2.14.1",
|
||||
"pg-pool": "^3.14.0",
|
||||
"pg-protocol": "^1.16.1",
|
||||
"pg-types": "2.2.0",
|
||||
"pgpass": "1.0.5"
|
||||
},
|
||||
"engines": {
|
||||
"node": ">= 16.0.0"
|
||||
},
|
||||
"optionalDependencies": {
|
||||
"pg-cloudflare": "^1.4.1"
|
||||
},
|
||||
"peerDependencies": {
|
||||
"pg-native": ">=3.0.1"
|
||||
},
|
||||
"peerDependenciesMeta": {
|
||||
"pg-native": {
|
||||
"optional": true
|
||||
}
|
||||
}
|
||||
},
|
||||
"node_modules/pg-cloudflare": {
|
||||
"version": "1.4.1",
|
||||
"resolved": "https://registry.npmjs.org/pg-cloudflare/-/pg-cloudflare-1.4.1.tgz",
|
||||
"integrity": "sha512-6PQbsFWZcp9EmJEwy5cGQ2La+AMWpP46lgbb8X+U/XsHIUweYDNCpeuKck5RxL2MdVFi7krbbEi5nX4Zh7JhrQ==",
|
||||
"license": "MIT",
|
||||
"optional": true
|
||||
},
|
||||
"node_modules/pg-connection-string": {
|
||||
"version": "2.14.1",
|
||||
"resolved": "https://registry.npmjs.org/pg-connection-string/-/pg-connection-string-2.14.1.tgz",
|
||||
"integrity": "sha512-qR3kGNPBLpCNtz0evbKA0Y/MRFXwSSdT+pTJvYp/bXTcReZbvX1kzF0IyTc1QnxqF7AZbOeBhNL8R5mYQZV/MA==",
|
||||
"license": "MIT"
|
||||
},
|
||||
"node_modules/pg-int8": {
|
||||
"version": "1.0.1",
|
||||
"resolved": "https://registry.npmjs.org/pg-int8/-/pg-int8-1.0.1.tgz",
|
||||
"integrity": "sha512-WCtabS6t3c8SkpDBUlb1kjOs7l66xsGdKpIPZsg4wR+B3+u9UAum2odSsF9tnvxg80h4ZxLWMy4pRjOsFIqQpw==",
|
||||
"license": "ISC",
|
||||
"engines": {
|
||||
"node": ">=4.0.0"
|
||||
}
|
||||
},
|
||||
"node_modules/pg-pool": {
|
||||
"version": "3.14.0",
|
||||
"resolved": "https://registry.npmjs.org/pg-pool/-/pg-pool-3.14.0.tgz",
|
||||
"integrity": "sha512-gKtPkFdQPU3DksooVLi9LsjZxrsBUZIpa+7aVx+LV5pNh0KzP4Zleud2po+ConrxbuXGBJ6Hfer6hdgpIBpBaw==",
|
||||
"license": "MIT",
|
||||
"peerDependencies": {
|
||||
"pg": ">=8.0"
|
||||
}
|
||||
},
|
||||
"node_modules/pg-protocol": {
|
||||
"version": "1.16.1",
|
||||
"resolved": "https://registry.npmjs.org/pg-protocol/-/pg-protocol-1.16.1.tgz",
|
||||
"integrity": "sha512-p9VOFMiHB/ZbJATetbg+99PxssTVSQRnyuPSQ67mN1+1KBOjZaZ83ZQzltnxPhJwSsC3nwVjJ10DVJlerbFzLg==",
|
||||
"license": "MIT"
|
||||
},
|
||||
"node_modules/pg-types": {
|
||||
"version": "2.2.0",
|
||||
"resolved": "https://registry.npmjs.org/pg-types/-/pg-types-2.2.0.tgz",
|
||||
"integrity": "sha512-qTAAlrEsl8s4OiEQY69wDvcMIdQN6wdz5ojQiOy6YRMuynxenON0O5oCpJI6lshc6scgAY8qvJ2On/p+CXY0GA==",
|
||||
"license": "MIT",
|
||||
"dependencies": {
|
||||
"pg-int8": "1.0.1",
|
||||
"postgres-array": "~2.0.0",
|
||||
"postgres-bytea": "~1.0.0",
|
||||
"postgres-date": "~1.0.4",
|
||||
"postgres-interval": "^1.1.0"
|
||||
},
|
||||
"engines": {
|
||||
"node": ">=4"
|
||||
}
|
||||
},
|
||||
"node_modules/pgpass": {
|
||||
"version": "1.0.5",
|
||||
"resolved": "https://registry.npmjs.org/pgpass/-/pgpass-1.0.5.tgz",
|
||||
"integrity": "sha512-FdW9r/jQZhSeohs1Z3sI1yxFQNFvMcnmfuj4WBMUTxOrAyLMaTcE1aAMBiTlbMNaXvBCQuVi0R7hd8udDSP7ug==",
|
||||
"license": "MIT",
|
||||
"dependencies": {
|
||||
"split2": "^4.1.0"
|
||||
}
|
||||
},
|
||||
"node_modules/pino": {
|
||||
"version": "10.4.0",
|
||||
"resolved": "https://registry.npmjs.org/pino/-/pino-10.4.0.tgz",
|
||||
"integrity": "sha512-bk1ZMTwG/Vymx+zPoa28MhNzucFvXJWp3csrSuxuHPSMeUettaCoQ5Jvg07mtC4JGNpcfTwz41oVCnNgBK9+vA==",
|
||||
"license": "MIT",
|
||||
"dependencies": {
|
||||
"@pinojs/redact": "^0.4.0",
|
||||
"atomic-sleep": "^1.0.0",
|
||||
"on-exit-leak-free": "^2.1.0",
|
||||
"pino-abstract-transport": "^3.0.0",
|
||||
"pino-std-serializers": "^7.0.0",
|
||||
"process-warning": "^5.0.0",
|
||||
"quick-format-unescaped": "^4.0.3",
|
||||
"real-require": "^1.0.0",
|
||||
"safe-stable-stringify": "^2.3.1",
|
||||
"sonic-boom": "^4.0.1",
|
||||
"thread-stream": "^4.0.0"
|
||||
},
|
||||
"bin": {
|
||||
"pino": "bin.js"
|
||||
}
|
||||
},
|
||||
"node_modules/pino-abstract-transport": {
|
||||
"version": "3.0.0",
|
||||
"resolved": "https://registry.npmjs.org/pino-abstract-transport/-/pino-abstract-transport-3.0.0.tgz",
|
||||
"integrity": "sha512-wlfUczU+n7Hy/Ha5j9a/gZNy7We5+cXp8YL+X+PG8S0KXxw7n/JXA3c46Y0zQznIJ83URJiwy7Lh56WLokNuxg==",
|
||||
"license": "MIT",
|
||||
"dependencies": {
|
||||
"split2": "^4.0.0"
|
||||
}
|
||||
},
|
||||
"node_modules/pino-std-serializers": {
|
||||
"version": "7.1.0",
|
||||
"resolved": "https://registry.npmjs.org/pino-std-serializers/-/pino-std-serializers-7.1.0.tgz",
|
||||
"integrity": "sha512-BndPH67/JxGExRgiX1dX0w1FvZck5Wa4aal9198SrRhZjH3GxKQUKIBnYJTdj2HDN3UQAS06HlfcSbQj2OHmaw==",
|
||||
"license": "MIT"
|
||||
},
|
||||
"node_modules/postgres-array": {
|
||||
"version": "2.0.0",
|
||||
"resolved": "https://registry.npmjs.org/postgres-array/-/postgres-array-2.0.0.tgz",
|
||||
"integrity": "sha512-VpZrUqU5A69eQyW2c5CA1jtLecCsN2U/bD6VilrFDWq5+5UIEVO7nazS3TEcHf1zuPYO/sqGvUvW62g86RXZuA==",
|
||||
"license": "MIT",
|
||||
"engines": {
|
||||
"node": ">=4"
|
||||
}
|
||||
},
|
||||
"node_modules/postgres-bytea": {
|
||||
"version": "1.0.1",
|
||||
"resolved": "https://registry.npmjs.org/postgres-bytea/-/postgres-bytea-1.0.1.tgz",
|
||||
"integrity": "sha512-5+5HqXnsZPE65IJZSMkZtURARZelel2oXUEO8rH83VS/hxH5vv1uHquPg5wZs8yMAfdv971IU+kcPUczi7NVBQ==",
|
||||
"license": "MIT",
|
||||
"engines": {
|
||||
"node": ">=0.10.0"
|
||||
}
|
||||
},
|
||||
"node_modules/postgres-date": {
|
||||
"version": "1.0.7",
|
||||
"resolved": "https://registry.npmjs.org/postgres-date/-/postgres-date-1.0.7.tgz",
|
||||
"integrity": "sha512-suDmjLVQg78nMK2UZ454hAG+OAW+HQPZ6n++TNDUX+L0+uUlLywnoxJKDou51Zm+zTCjrCl0Nq6J9C5hP9vK/Q==",
|
||||
"license": "MIT",
|
||||
"engines": {
|
||||
"node": ">=0.10.0"
|
||||
}
|
||||
},
|
||||
"node_modules/postgres-interval": {
|
||||
"version": "1.2.0",
|
||||
"resolved": "https://registry.npmjs.org/postgres-interval/-/postgres-interval-1.2.0.tgz",
|
||||
"integrity": "sha512-9ZhXKM/rw350N1ovuWHbGxnGh/SNJ4cnxHiM0rxE4VN41wsg8P8zWn9hv/buK00RP4WvlOyr/RBDiptyxVbkZQ==",
|
||||
"license": "MIT",
|
||||
"dependencies": {
|
||||
"xtend": "^4.0.0"
|
||||
},
|
||||
"engines": {
|
||||
"node": ">=0.10.0"
|
||||
}
|
||||
},
|
||||
"node_modules/process-warning": {
|
||||
"version": "5.1.0",
|
||||
"resolved": "https://registry.npmjs.org/process-warning/-/process-warning-5.1.0.tgz",
|
||||
"integrity": "sha512-jQSaVHsPgtyw60e1rQ/A+/ArPEj/S8pS/vFnyGa/gYFXrKk/6RuDkoqVDQ5NI5MmS01698ltlAk0NoDBNLujRw==",
|
||||
"funding": [
|
||||
{
|
||||
"type": "github",
|
||||
"url": "https://github.com/sponsors/fastify"
|
||||
},
|
||||
{
|
||||
"type": "opencollective",
|
||||
"url": "https://opencollective.com/fastify"
|
||||
}
|
||||
],
|
||||
"license": "MIT"
|
||||
},
|
||||
"node_modules/quick-format-unescaped": {
|
||||
"version": "4.0.4",
|
||||
"resolved": "https://registry.npmjs.org/quick-format-unescaped/-/quick-format-unescaped-4.0.4.tgz",
|
||||
"integrity": "sha512-tYC1Q1hgyRuHgloV/YXs2w15unPVh8qfu/qCTfhTYamaw7fyhumKa2yGpdSo87vY32rIclj+4fWYQXUMs9EHvg==",
|
||||
"license": "MIT"
|
||||
},
|
||||
"node_modules/real-require": {
|
||||
"version": "1.0.0",
|
||||
"resolved": "https://registry.npmjs.org/real-require/-/real-require-1.0.0.tgz",
|
||||
"integrity": "sha512-P4nbQYQfePJxRSmY+v/KINxVucm4NF3p3s7pJveMTtom52FR4YGltUQLB8idDXwDDWW+eYrWDFbuzUnjoWHF7g==",
|
||||
"license": "MIT"
|
||||
},
|
||||
"node_modules/require-from-string": {
|
||||
"version": "2.0.2",
|
||||
"resolved": "https://registry.npmjs.org/require-from-string/-/require-from-string-2.0.2.tgz",
|
||||
"integrity": "sha512-Xf0nWe6RseziFMu+Ap9biiUbmplq6S9/p+7w7YXP/JBHhrUDDUhwa+vANyubuqfZWTveU//DYVGsDG7RKL/vEw==",
|
||||
"license": "MIT",
|
||||
"engines": {
|
||||
"node": ">=0.10.0"
|
||||
}
|
||||
},
|
||||
"node_modules/ret": {
|
||||
"version": "0.5.0",
|
||||
"resolved": "https://registry.npmjs.org/ret/-/ret-0.5.0.tgz",
|
||||
"integrity": "sha512-I1XxrZSQ+oErkRR4jYbAyEEu2I0avBvvMM5JN+6EBprOGRCs63ENqZ3vjavq8fBw2+62G5LF5XelKwuJpcvcxw==",
|
||||
"license": "MIT",
|
||||
"engines": {
|
||||
"node": ">=10"
|
||||
}
|
||||
},
|
||||
"node_modules/reusify": {
|
||||
"version": "1.1.0",
|
||||
"resolved": "https://registry.npmjs.org/reusify/-/reusify-1.1.0.tgz",
|
||||
"integrity": "sha512-g6QUff04oZpHs0eG5p83rFLhHeV00ug/Yf9nZM6fLeUrPguBTkTQOdpAWWspMh55TZfVQDPaN3NQJfbVRAxdIw==",
|
||||
"license": "MIT",
|
||||
"engines": {
|
||||
"iojs": ">=1.0.0",
|
||||
"node": ">=0.10.0"
|
||||
}
|
||||
},
|
||||
"node_modules/rfdc": {
|
||||
"version": "1.4.1",
|
||||
"resolved": "https://registry.npmjs.org/rfdc/-/rfdc-1.4.1.tgz",
|
||||
"integrity": "sha512-q1b3N5QkRUWUl7iyylaaj3kOpIT0N2i9MqIEQXP73GVsN9cw3fdx8X63cEmWhJGi2PPCF23Ijp7ktmd39rawIA==",
|
||||
"license": "MIT"
|
||||
},
|
||||
"node_modules/safe-regex2": {
|
||||
"version": "5.1.1",
|
||||
"resolved": "https://registry.npmjs.org/safe-regex2/-/safe-regex2-5.1.1.tgz",
|
||||
"integrity": "sha512-mOSBvHGDZMuIEZMdOz/aCEYDCv0E7nfcNsIhUF+/P+xC7Hyf3FkvymqgPbg9D1EdSGu+uKbJgy09K/RKKc7kJA==",
|
||||
"funding": [
|
||||
{
|
||||
"type": "github",
|
||||
"url": "https://github.com/sponsors/fastify"
|
||||
},
|
||||
{
|
||||
"type": "opencollective",
|
||||
"url": "https://opencollective.com/fastify"
|
||||
}
|
||||
],
|
||||
"license": "MIT",
|
||||
"dependencies": {
|
||||
"ret": "~0.5.0"
|
||||
},
|
||||
"bin": {
|
||||
"safe-regex2": "bin/safe-regex2.js"
|
||||
}
|
||||
},
|
||||
"node_modules/safe-stable-stringify": {
|
||||
"version": "2.5.0",
|
||||
"resolved": "https://registry.npmjs.org/safe-stable-stringify/-/safe-stable-stringify-2.5.0.tgz",
|
||||
"integrity": "sha512-b3rppTKm9T+PsVCBEOUR46GWI7fdOs00VKZ1+9c1EWDaDMvjQc6tUwuFyIprgGgTcWoVHSKrU8H31ZHA2e0RHA==",
|
||||
"license": "MIT",
|
||||
"engines": {
|
||||
"node": ">=10"
|
||||
}
|
||||
},
|
||||
"node_modules/secure-json-parse": {
|
||||
"version": "4.1.0",
|
||||
"resolved": "https://registry.npmjs.org/secure-json-parse/-/secure-json-parse-4.1.0.tgz",
|
||||
"integrity": "sha512-l4KnYfEyqYJxDwlNVyRfO2E4NTHfMKAWdUuA8J0yve2Dz/E/PdBepY03RvyJpssIpRFwJoCD55wA+mEDs6ByWA==",
|
||||
"funding": [
|
||||
{
|
||||
"type": "github",
|
||||
"url": "https://github.com/sponsors/fastify"
|
||||
},
|
||||
{
|
||||
"type": "opencollective",
|
||||
"url": "https://opencollective.com/fastify"
|
||||
}
|
||||
],
|
||||
"license": "BSD-3-Clause"
|
||||
},
|
||||
"node_modules/semver": {
|
||||
"version": "7.8.5",
|
||||
"resolved": "https://registry.npmjs.org/semver/-/semver-7.8.5.tgz",
|
||||
"integrity": "sha512-Y7/KDsb8LjooZpwaqGyulO6DQlksgCncchHGk+sZIY4SBvUocMBEFH5Ur1fI4dV+Jvl0w6cjvucaIi40puRioA==",
|
||||
"license": "ISC",
|
||||
"bin": {
|
||||
"semver": "bin/semver.js"
|
||||
},
|
||||
"engines": {
|
||||
"node": ">=10"
|
||||
}
|
||||
},
|
||||
"node_modules/set-cookie-parser": {
|
||||
"version": "2.7.2",
|
||||
"resolved": "https://registry.npmjs.org/set-cookie-parser/-/set-cookie-parser-2.7.2.tgz",
|
||||
"integrity": "sha512-oeM1lpU/UvhTxw+g3cIfxXHyJRc/uidd3yK1P242gzHds0udQBYzs3y8j4gCCW+ZJ7ad0yctld8RYO+bdurlvw==",
|
||||
"license": "MIT"
|
||||
},
|
||||
"node_modules/sonic-boom": {
|
||||
"version": "4.2.1",
|
||||
"resolved": "https://registry.npmjs.org/sonic-boom/-/sonic-boom-4.2.1.tgz",
|
||||
"integrity": "sha512-w6AxtubXa2wTXAUsZMMWERrsIRAdrK0Sc+FUytWvYAhBJLyuI4llrMIC1DtlNSdI99EI86KZum2MMq3EAZlF9Q==",
|
||||
"license": "MIT",
|
||||
"dependencies": {
|
||||
"atomic-sleep": "^1.0.0"
|
||||
}
|
||||
},
|
||||
"node_modules/split2": {
|
||||
"version": "4.2.0",
|
||||
"resolved": "https://registry.npmjs.org/split2/-/split2-4.2.0.tgz",
|
||||
"integrity": "sha512-UcjcJOWknrNkF6PLX83qcHM6KHgVKNkV62Y8a5uYDVv9ydGQVwAHMKqHdJje1VTWpljG0WYpCDhrCdAOYH4TWg==",
|
||||
"license": "ISC",
|
||||
"engines": {
|
||||
"node": ">= 10.x"
|
||||
}
|
||||
},
|
||||
"node_modules/thread-stream": {
|
||||
"version": "4.2.0",
|
||||
"resolved": "https://registry.npmjs.org/thread-stream/-/thread-stream-4.2.0.tgz",
|
||||
"integrity": "sha512-e2zZ96wSChazBsbENf/Pcm/4swHt2cEKQ92rhUjkL9GCKiTDJIaTBenjE/m9DXi0QBmTMDkFDdOomUy20A1tDQ==",
|
||||
"license": "MIT",
|
||||
"dependencies": {
|
||||
"real-require": "^1.0.0"
|
||||
},
|
||||
"engines": {
|
||||
"node": ">=20"
|
||||
}
|
||||
},
|
||||
"node_modules/toad-cache": {
|
||||
"version": "3.7.4",
|
||||
"resolved": "https://registry.npmjs.org/toad-cache/-/toad-cache-3.7.4.tgz",
|
||||
"integrity": "sha512-m1TdR/rvT7kgGJZhspNtXdsdYk0fddFpJJFlG5s+UkPFo6lkLoZ3YLOaovPYjq1R75NP5JfeTlSHaOsE09peCg==",
|
||||
"license": "MIT",
|
||||
"engines": {
|
||||
"node": ">=20"
|
||||
}
|
||||
},
|
||||
"node_modules/xtend": {
|
||||
"version": "4.0.2",
|
||||
"resolved": "https://registry.npmjs.org/xtend/-/xtend-4.0.2.tgz",
|
||||
"integrity": "sha512-LKYU1iAXJXUgAXn9URjiu+MWhyUXHsvfp7mcuYm9dSUKK0/CjtrUwFAxD82/mCWbtLsGjFIad0wIsod4zrTAEQ==",
|
||||
"license": "MIT",
|
||||
"engines": {
|
||||
"node": ">=0.4"
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
18
api/package.json
Normal file
18
api/package.json
Normal file
@ -0,0 +1,18 @@
|
||||
{
|
||||
"name": "divan-api",
|
||||
"private": true,
|
||||
"type": "module",
|
||||
"description": "Divan API: Node.js + PostgreSQL",
|
||||
"engines": {
|
||||
"node": ">=24"
|
||||
},
|
||||
"scripts": {
|
||||
"start": "node src/server.ts",
|
||||
"import": "node src/import.ts",
|
||||
"test": "node --test src/*.test.ts"
|
||||
},
|
||||
"dependencies": {
|
||||
"fastify": "^5.12.5",
|
||||
"pg": "^8.23.1"
|
||||
}
|
||||
}
|
||||
5
api/src/db.ts
Normal file
5
api/src/db.ts
Normal file
@ -0,0 +1,5 @@
|
||||
import pg from 'pg';
|
||||
|
||||
export const pool = new pg.Pool({
|
||||
connectionString: process.env.DATABASE_URL ?? 'postgres://divan:divan_local@localhost:5433/divan',
|
||||
});
|
||||
84
api/src/import.ts
Normal file
84
api/src/import.ts
Normal file
@ -0,0 +1,84 @@
|
||||
// Load divan-data (ganjoor-data layout: manifest.json, poets/**/_cat.json, poems) into PostgreSQL.
|
||||
// Source: a local divan-data folder or the public CDN. Upserts, so re-running applies sync changes.
|
||||
// node src/import.ts [../../divan-data | https://cdn.jsdelivr.net/gh/anas-rashid/divan-data@main/]
|
||||
import { readFile } from 'node:fs/promises';
|
||||
import { join } from 'node:path';
|
||||
import { pool } from './db.ts';
|
||||
import { normalise } from './urdu.ts';
|
||||
|
||||
const SOURCE = process.argv[2] ?? new URL('../../../divan-data', import.meta.url).pathname;
|
||||
const isHttp = /^https?:/.test(SOURCE);
|
||||
|
||||
async function read(path: string): Promise<any> {
|
||||
if (isHttp) {
|
||||
const res = await fetch(SOURCE.replace(/\/?$/, '/') + path);
|
||||
if (!res.ok) throw new Error(`${res.status} ${path}`);
|
||||
return res.json();
|
||||
}
|
||||
return JSON.parse(await readFile(join(SOURCE, path), 'utf8'));
|
||||
}
|
||||
|
||||
const schema = await readFile(new URL('../../db/schema.sql', import.meta.url), 'utf8');
|
||||
await pool.query(schema);
|
||||
|
||||
const manifest = await read('manifest.json');
|
||||
console.log(`${manifest.PoetsCount} poets, ${manifest.PoemsCount} poems from ${SOURCE}`);
|
||||
|
||||
let poems = 0;
|
||||
for (const entry of manifest.Poets) {
|
||||
const poet = await read(`poets${entry.FullUrl}/poet.json`);
|
||||
const client = await pool.connect();
|
||||
try {
|
||||
await client.query('BEGIN');
|
||||
await client.query(
|
||||
`INSERT INTO poets (id, url, name, nickname, description, birth_year_ah, death_year_ah)
|
||||
VALUES ($1,$2,$3,$4,$5,$6,$7)
|
||||
ON CONFLICT (id) DO UPDATE SET url=$2, name=$3, nickname=$4, description=$5, birth_year_ah=$6, death_year_ah=$7`,
|
||||
[poet.Id, poet.FullUrl, poet.Name, poet.Nickname, poet.Description,
|
||||
poet.ValidBirthDate ? poet.BirthYearInLHijri : null, poet.ValidDeathDate ? poet.DeathYearInLHijri : null],
|
||||
);
|
||||
// categories depth-first so parents exist before children
|
||||
const walk = async (url: string, position: number): Promise<void> => {
|
||||
const cat = await read(`poets${url}/_cat.json`);
|
||||
await client.query(
|
||||
`INSERT INTO categories (id, poet_id, parent_id, url, title, position) VALUES ($1,$2,$3,$4,$5,$6)
|
||||
ON CONFLICT (id) DO UPDATE SET poet_id=$2, parent_id=$3, url=$4, title=$5, position=$6`,
|
||||
[cat.Id, cat.PoetId, cat.ParentId, cat.FullUrl, cat.Title, position],
|
||||
);
|
||||
for (const [i, ref] of cat.Poems.entries()) {
|
||||
const p = await read(`poets${ref.FullUrl}.json`);
|
||||
const searchText = normalise([p.Title, ...p.Verses.map((v: any) => v.Text)].join(' '));
|
||||
await client.query(
|
||||
`INSERT INTO poems (id, category_id, poet_id, url, title, full_title, source_url, position, search_text)
|
||||
VALUES ($1,$2,$3,$4,$5,$6,$7,$8,$9)
|
||||
ON CONFLICT (id) DO UPDATE SET category_id=$2, poet_id=$3, url=$4, title=$5, full_title=$6,
|
||||
source_url=$7, position=$8, search_text=$9`,
|
||||
[p.Id, cat.Id, poet.Id, p.FullUrl, p.Title, p.FullTitle, p.SourceUrl ?? null, i, searchText],
|
||||
);
|
||||
await client.query('DELETE FROM verses WHERE poem_id = $1', [p.Id]);
|
||||
if (p.Verses.length) {
|
||||
// one multi-row insert per poem
|
||||
const values: unknown[] = [];
|
||||
const rows = p.Verses.map((v: any, j: number) => {
|
||||
values.push(p.Id, v.VOrder, v.Position, v.CoupletIndex ?? 0, v.Text);
|
||||
const o = j * 5;
|
||||
return `($${o + 1},$${o + 2},$${o + 3},$${o + 4},$${o + 5})`;
|
||||
});
|
||||
await client.query(`INSERT INTO verses (poem_id, vorder, position, couplet, text) VALUES ${rows.join(',')}`, values);
|
||||
}
|
||||
poems++;
|
||||
}
|
||||
for (const [i, child] of cat.ChildCats.entries()) await walk(child.FullUrl, i);
|
||||
};
|
||||
await walk(entry.FullUrl, 0);
|
||||
await client.query('COMMIT');
|
||||
} catch (e) {
|
||||
await client.query('ROLLBACK');
|
||||
throw e;
|
||||
} finally {
|
||||
client.release();
|
||||
}
|
||||
process.stdout.write(`\r${poems} poems`);
|
||||
}
|
||||
console.log(`\nimported ${manifest.Poets.length} poets, ${poems} poems`);
|
||||
await pool.end();
|
||||
101
api/src/server.ts
Normal file
101
api/src/server.ts
Normal file
@ -0,0 +1,101 @@
|
||||
// Divan API (Fastify + PostgreSQL)
|
||||
// GET /api/poets all poets
|
||||
// GET /api/page?url=/p238/... poet, category or poem at that URL
|
||||
// GET /api/search?q=&poet=&page=
|
||||
// GET /health
|
||||
import Fastify from 'fastify';
|
||||
import { pool } from './db.ts';
|
||||
import { likePatterns } from './urdu.ts';
|
||||
|
||||
const app = Fastify({ logger: { level: process.env.LOG_LEVEL ?? 'info' } });
|
||||
const PAGE_SIZE = 20;
|
||||
|
||||
app.get('/health', async () => {
|
||||
await pool.query('SELECT 1');
|
||||
return { ok: true };
|
||||
});
|
||||
|
||||
app.get('/api/poets', async () => {
|
||||
const { rows } = await pool.query(
|
||||
'SELECT id, url, name, nickname, birth_year_ah, death_year_ah FROM poets ORDER BY birth_year_ah NULLS LAST, nickname',
|
||||
);
|
||||
return rows;
|
||||
});
|
||||
|
||||
// breadcrumb from a category up to the poet root
|
||||
async function ancestors(categoryId: number) {
|
||||
const { rows } = await pool.query(
|
||||
`WITH RECURSIVE up AS (
|
||||
SELECT id, parent_id, url, title, 0 AS depth FROM categories WHERE id = $1
|
||||
UNION ALL
|
||||
SELECT c.id, c.parent_id, c.url, c.title, up.depth + 1 FROM categories c JOIN up ON c.id = up.parent_id)
|
||||
SELECT url, title FROM up ORDER BY depth DESC`,
|
||||
[categoryId],
|
||||
);
|
||||
return rows;
|
||||
}
|
||||
|
||||
app.get<{ Querystring: { url?: string } }>('/api/page', async (req, reply) => {
|
||||
const url = '/' + (req.query.url ?? '').replace(/^\/+|\/+$/g, '');
|
||||
|
||||
const poem = (await pool.query('SELECT * FROM poems WHERE url = $1', [url])).rows[0];
|
||||
if (poem) {
|
||||
const [verses, poet, crumbs, siblings] = await Promise.all([
|
||||
pool.query('SELECT vorder, position, couplet, text FROM verses WHERE poem_id = $1 ORDER BY vorder', [poem.id]),
|
||||
pool.query('SELECT id, url, nickname FROM poets WHERE id = $1', [poem.poet_id]),
|
||||
ancestors(poem.category_id),
|
||||
pool.query(
|
||||
`SELECT (SELECT url FROM poems WHERE category_id = $1 AND position < $2 ORDER BY position DESC LIMIT 1) AS prev,
|
||||
(SELECT url FROM poems WHERE category_id = $1 AND position > $2 ORDER BY position LIMIT 1) AS next`,
|
||||
[poem.category_id, poem.position],
|
||||
),
|
||||
]);
|
||||
const { search_text, ...rest } = poem;
|
||||
return { type: 'poem', poem: rest, poet: poet.rows[0], breadcrumbs: crumbs, verses: verses.rows, ...siblings.rows[0] };
|
||||
}
|
||||
|
||||
const cat = (await pool.query('SELECT * FROM categories WHERE url = $1', [url])).rows[0];
|
||||
if (cat) {
|
||||
const [poet, crumbs, children, poems] = await Promise.all([
|
||||
pool.query('SELECT * FROM poets WHERE id = $1', [cat.poet_id]),
|
||||
ancestors(cat.id),
|
||||
pool.query(
|
||||
`SELECT c.url, c.title, (SELECT count(*) FROM poems p WHERE p.category_id = c.id)::int AS poems
|
||||
FROM categories c WHERE c.parent_id = $1 ORDER BY c.position, c.id`,
|
||||
[cat.id],
|
||||
),
|
||||
pool.query('SELECT url, title FROM poems WHERE category_id = $1 ORDER BY position, id', [cat.id]),
|
||||
]);
|
||||
return {
|
||||
type: cat.parent_id === null ? 'poet' : 'category',
|
||||
category: cat, poet: poet.rows[0], breadcrumbs: crumbs, children: children.rows, poems: poems.rows,
|
||||
};
|
||||
}
|
||||
return reply.code(404).send({ error: 'not found' });
|
||||
});
|
||||
|
||||
app.get<{ Querystring: { q?: string; poet?: string; page?: string } }>('/api/search', async (req) => {
|
||||
const patterns = likePatterns(req.query.q ?? '');
|
||||
if (!patterns.length) return { total: 0, page: 1, results: [] };
|
||||
const page = Math.max(1, Number(req.query.page) || 1);
|
||||
const params: unknown[] = [...patterns];
|
||||
const where = patterns.map((_, i) => `p.search_text ILIKE $${i + 1}`);
|
||||
if (req.query.poet) {
|
||||
params.push(Number(req.query.poet));
|
||||
where.push(`p.poet_id = $${params.length}`);
|
||||
}
|
||||
const sql = `FROM poems p JOIN poets t ON t.id = p.poet_id WHERE ${where.join(' AND ')}`;
|
||||
const [count, rows] = await Promise.all([
|
||||
pool.query(`SELECT count(*)::int AS n ${sql}`, params),
|
||||
pool.query(
|
||||
`SELECT p.url, p.title, t.nickname AS poet, t.url AS poet_url,
|
||||
(SELECT v.text FROM verses v WHERE v.poem_id = p.id ORDER BY v.vorder LIMIT 1) AS first_line
|
||||
${sql} ORDER BY t.birth_year_ah NULLS LAST, p.id LIMIT ${PAGE_SIZE} OFFSET ${(page - 1) * PAGE_SIZE}`,
|
||||
params,
|
||||
),
|
||||
]);
|
||||
return { total: count.rows[0].n, page, pageSize: PAGE_SIZE, results: rows.rows };
|
||||
});
|
||||
|
||||
const port = Number(process.env.PORT ?? 4100);
|
||||
await app.listen({ port, host: process.env.HOST ?? '127.0.0.1' });
|
||||
20
api/src/urdu.test.ts
Normal file
20
api/src/urdu.test.ts
Normal file
@ -0,0 +1,20 @@
|
||||
import { test } from 'node:test';
|
||||
import assert from 'node:assert/strict';
|
||||
import { normalise, likePatterns } from './urdu.ts';
|
||||
|
||||
test('Urdu variants and diacritics normalise to the same form', () => {
|
||||
const same = (a: string, b: string) => assert.equal(normalise(a), normalise(b), `${a} vs ${b}`);
|
||||
same('دلِ ناداں', 'دل ناداں'); // izafat kasra
|
||||
same('کوئی امید', 'كوئي اميد'); // arabic kaf/yeh
|
||||
same('ہوئی', 'هوئی'); // arabic heh vs heh goal
|
||||
same('نالۂ', 'نالہ'); // heh goal with hamza
|
||||
same('دیکھا۔', 'دیکھا'); // urdu full stop
|
||||
assert.notEqual(normalise('کھ'), normalise('کہ'), 'do-chashmi heh must stay distinct');
|
||||
});
|
||||
|
||||
test('search patterns: words, phrases, escaping, empty', () => {
|
||||
assert.deepEqual(likePatterns('دل ناداں'), ['%دل%', '%ناداں%']);
|
||||
assert.deepEqual(likePatterns('"دل ناداں"'), ['%دل ناداں%']);
|
||||
assert.deepEqual(likePatterns('50%_x'), ['%50\\%\\_x%']);
|
||||
assert.deepEqual(likePatterns(' '), []);
|
||||
});
|
||||
44
api/src/urdu.ts
Normal file
44
api/src/urdu.ts
Normal file
@ -0,0 +1,44 @@
|
||||
// Urdu text normalisation for search. Applied to stored text at import and to search terms,
|
||||
// so both sides compare in the same form. Ported from the .NET LanguageUtils.MakeTextSearchable.
|
||||
|
||||
const STRIP = new RegExp(
|
||||
'[' +
|
||||
'\u064B-\u0652' + // Arabic diacritics: tanween, fatha, kasra, damma, shadda, sukun
|
||||
'\u0654\u0670\u0657\u0658\u0615' + // hamza above, superscript alef, Urdu diacritics
|
||||
'\u200D\u200E\u200F' + // zwj, ltr/rtl marks
|
||||
'.\u060C!\u061F:\u061B;*()\\[\\]"\'\u00AB\u00BB\u06D4' + // punctuation incl. Urdu full stop
|
||||
']',
|
||||
'g',
|
||||
);
|
||||
|
||||
const LETTERS: Record<string, string> = {
|
||||
'\u064A': '\u06CC', // arabic yeh -> farsi/urdu yeh
|
||||
'\u0649': '\u06CC', // alef maksura -> yeh
|
||||
'\u0626': '\u06CC', // yeh with hamza -> yeh (upstream rule)
|
||||
'\u0643': '\u06A9', // arabic kaf -> keheh
|
||||
'\u06C2': '\u06C1', // heh goal with hamza -> heh goal
|
||||
'\u0647': '\u06C1', // arabic heh -> heh goal (do-chashmi heh U+06BE stays distinct)
|
||||
'\u06D3': '\u06D2', // yeh barree with hamza -> yeh barree
|
||||
'\u0623': '\u0627', // alef with hamza above -> alef
|
||||
'\u0625': '\u0627', // alef with hamza below -> alef
|
||||
};
|
||||
const LETTER_RE = new RegExp('[' + Object.keys(LETTERS).join('') + ']', 'g');
|
||||
|
||||
export function normalise(text: string): string {
|
||||
return text
|
||||
.replace(/\u200C/g, ' ') // zwnj -> space
|
||||
.replace(STRIP, '')
|
||||
.replace(LETTER_RE, (c) => LETTERS[c])
|
||||
.replace(/\s+/g, ' ')
|
||||
.trim();
|
||||
}
|
||||
|
||||
// ILIKE patterns for a search term: "quoted phrase" -> one pattern, otherwise one per word (all must match).
|
||||
export function likePatterns(term: string): string[] {
|
||||
const t = (term ?? '').trim();
|
||||
if (!t) return [];
|
||||
const phrase = t.length > 1 && t.startsWith('"') && t.endsWith('"');
|
||||
const n = normalise(t);
|
||||
const parts = phrase ? [n] : n.split(' ');
|
||||
return parts.filter(Boolean).map((p) => '%' + p.replace(/[\\%_]/g, (c) => '\\' + c) + '%');
|
||||
}
|
||||
49
db/schema.sql
Normal file
49
db/schema.sql
Normal file
@ -0,0 +1,49 @@
|
||||
-- Divan PostgreSQL schema. Ids and URLs come from divan-data (stable across syncs).
|
||||
-- Search: poems.search_text holds Urdu-normalised text (see api/src/urdu.ts); the pg_trgm
|
||||
-- index makes substring search (ILIKE '%word%') fast without a language dictionary.
|
||||
|
||||
CREATE EXTENSION IF NOT EXISTS pg_trgm;
|
||||
|
||||
CREATE TABLE IF NOT EXISTS poets (
|
||||
id integer PRIMARY KEY,
|
||||
url text NOT NULL UNIQUE, -- '/p238'
|
||||
name text NOT NULL,
|
||||
nickname text NOT NULL,
|
||||
description text, -- short intro (Urdu Wikipedia lead)
|
||||
birth_year_ah integer, -- Hijri, approximate
|
||||
death_year_ah integer
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS categories (
|
||||
id integer PRIMARY KEY,
|
||||
poet_id integer NOT NULL REFERENCES poets(id) ON DELETE CASCADE,
|
||||
parent_id integer REFERENCES categories(id) ON DELETE CASCADE, -- NULL = the poet's root
|
||||
url text NOT NULL UNIQUE, -- '/p238/shaeri/c641'
|
||||
title text NOT NULL,
|
||||
position integer NOT NULL DEFAULT 0
|
||||
);
|
||||
CREATE INDEX IF NOT EXISTS categories_parent ON categories(parent_id);
|
||||
CREATE INDEX IF NOT EXISTS categories_poet ON categories(poet_id);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS poems (
|
||||
id integer PRIMARY KEY,
|
||||
category_id integer NOT NULL REFERENCES categories(id) ON DELETE CASCADE,
|
||||
poet_id integer NOT NULL REFERENCES poets(id) ON DELETE CASCADE,
|
||||
url text NOT NULL UNIQUE, -- '/p238/shaeri/c641/sh6556'
|
||||
title text NOT NULL,
|
||||
full_title text NOT NULL,
|
||||
source_url text, -- Wikisource page
|
||||
position integer NOT NULL DEFAULT 0,
|
||||
search_text text NOT NULL DEFAULT '' -- normalised title + verses
|
||||
);
|
||||
CREATE INDEX IF NOT EXISTS poems_category ON poems(category_id);
|
||||
CREATE INDEX IF NOT EXISTS poems_search ON poems USING gin (search_text gin_trgm_ops);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS verses (
|
||||
poem_id integer NOT NULL REFERENCES poems(id) ON DELETE CASCADE,
|
||||
vorder integer NOT NULL,
|
||||
position text NOT NULL, -- Right | Left | Single | Paragraph
|
||||
couplet integer NOT NULL,
|
||||
text text NOT NULL,
|
||||
PRIMARY KEY (poem_id, vorder)
|
||||
);
|
||||
Loading…
Reference in New Issue
Block a user