diff --git a/GanjooRazor/wwwroot/robots.txt b/GanjooRazor/wwwroot/robots.txt index 2643a5ee..8867f53f 100644 --- a/GanjooRazor/wwwroot/robots.txt +++ b/GanjooRazor/wwwroot/robots.txt @@ -1,3 +1,3 @@ User-agent: * Disallow: -Sitemap: http://ganjoor.net/sitemap.xml.gz \ No newline at end of file +Sitemap: http://ganjoor.net/sitemap.xml \ No newline at end of file diff --git a/RMuseum/Controllers/GanjoorController.cs b/RMuseum/Controllers/GanjoorController.cs index ea17eb13..39365544 100644 --- a/RMuseum/Controllers/GanjoorController.cs +++ b/RMuseum/Controllers/GanjoorController.cs @@ -1444,6 +1444,26 @@ namespace RMuseum.Controllers return BadRequest(res.ExceptionString); } + /// + /// start building sitemap + /// + /// + + [HttpPost("sitemap")] + [Authorize(Policy = RMuseumSecurableItem.GanjoorEntityShortName + ":" + SecurableItem.ModifyOperationShortName)] + [ProducesResponseType((int)HttpStatusCode.OK)] + [ProducesResponseType((int)HttpStatusCode.BadRequest, Type = typeof(string))] + [ProducesResponseType((int)HttpStatusCode.NotFound)] + public IActionResult StartBuildingSitemap() + { + var res = _ganjoorService.StartBuildingSitemap(); + + if (!string.IsNullOrEmpty(res.ExceptionString)) + return BadRequest(res.ExceptionString); + + return Ok(); + } + /// /// Ganjoor Service /// diff --git a/RMuseum/RMuseum.xml b/RMuseum/RMuseum.xml index 3194bc74..8c2b7345 100644 --- a/RMuseum/RMuseum.xml +++ b/RMuseum/RMuseum.xml @@ -1089,6 +1089,12 @@ + + + start building sitemap + + + Ganjoor Service @@ -7588,6 +7594,12 @@ + + + build sitemap + + + IArtifactService implementation @@ -8549,6 +8561,9 @@ IGanjoorService implementation + + IGanjoorService implementation + @@ -8556,6 +8571,12 @@ + + + build sitemap + + + export to sqlite diff --git a/RMuseum/Services/IGanjoorService.cs b/RMuseum/Services/IGanjoorService.cs index 6aad25c6..a2ea7407 100644 --- a/RMuseum/Services/IGanjoorService.cs +++ b/RMuseum/Services/IGanjoorService.cs @@ -439,5 +439,12 @@ namespace RMuseum.Services /// /// RServiceResult StartGeneratingSubCatsTOC(Guid userId, int catId); + + + /// + /// build sitemap + /// + /// + RServiceResult StartBuildingSitemap(); } } diff --git a/RMuseum/Services/Implementation/GanjoorService-Imports/GanjoorService-MySqlImports.cs b/RMuseum/Services/Implementation/GanjoorService-Partials/GanjoorService-MySqlImports.cs similarity index 100% rename from RMuseum/Services/Implementation/GanjoorService-Imports/GanjoorService-MySqlImports.cs rename to RMuseum/Services/Implementation/GanjoorService-Partials/GanjoorService-MySqlImports.cs diff --git a/RMuseum/Services/Implementation/GanjoorService-Imports/GanjoorService-SQLiteImport.cs b/RMuseum/Services/Implementation/GanjoorService-Partials/GanjoorService-SQLiteImport.cs similarity index 100% rename from RMuseum/Services/Implementation/GanjoorService-Imports/GanjoorService-SQLiteImport.cs rename to RMuseum/Services/Implementation/GanjoorService-Partials/GanjoorService-SQLiteImport.cs diff --git a/RMuseum/Services/Implementation/GanjoorService-Partials/GanjoorService-SitemapGenerator.cs b/RMuseum/Services/Implementation/GanjoorService-Partials/GanjoorService-SitemapGenerator.cs new file mode 100644 index 00000000..04f2c532 --- /dev/null +++ b/RMuseum/Services/Implementation/GanjoorService-Partials/GanjoorService-SitemapGenerator.cs @@ -0,0 +1,134 @@ +using Microsoft.EntityFrameworkCore; +using RMuseum.DbContext; +using RSecurityBackend.Models.Generic; +using RSecurityBackend.Services.Implementation; +using System; +using System.Collections.Generic; +using System.Data; +using System.IO; +using System.Linq; +using System.Xml.Linq; + +namespace RMuseum.Services.Implementation +{ + /// + /// IGanjoorService implementation + /// + public partial class GanjoorService : IGanjoorService + { + + private void WriteSitemap(string filePath, List urls) + { + if (File.Exists(filePath)) + File.Delete(filePath); + XNamespace ns = "http://www.sitemaps.org/schemas/sitemap/0.9"; + XNamespace xsiNs = "http://www.w3.org/2001/XMLSchema-instance"; + + XDocument xDoc = new XDocument( + new XDeclaration("1.0", "UTF-8", "no"), + new XElement(ns + "urlset", + new XAttribute(XNamespace.Xmlns + "xsi", xsiNs), + new XAttribute(xsiNs + "schemaLocation", + "http://www.sitemaps.org/schemas/sitemap/0.9 http://www.sitemaps.org/schemas/sitemap/0.9/sitemap.xsd"), + from url in urls + select new XElement(ns + "url", + new XElement(ns + "loc", $"https://ganjoor.net{url}")) + ) + ); + + xDoc.Save(filePath); + } + + /// + /// build sitemap + /// + /// + public RServiceResult StartBuildingSitemap() + { + try + { + + _backgroundTaskQueue.QueueBackgroundWorkItem + ( + async token => + { + using (RMuseumDbContext context = new RMuseumDbContext(new DbContextOptions())) //this is long running job, so _context might be already been freed/collected by GC + { + LongRunningJobProgressServiceEF jobProgressServiceEF = new LongRunningJobProgressServiceEF(context); + var job = (await jobProgressServiceEF.NewJob("BuildSitemap", "Query data")).Result; + try + { + string xmlSitemap = _configuration.GetSection("Ganjoor")["SitemapLocation"]; + if (File.Exists(xmlSitemap)) + File.Delete(xmlSitemap); + + string dir = Path.GetDirectoryName(xmlSitemap); + + await jobProgressServiceEF.UpdateJob(job.Id, 0, "", false); + + List sitemaps = new List(); + + string firstSitemap = Path.Combine(dir, $"1.xml"); + + sitemaps.Add(firstSitemap); + + var urls = await context.GanjoorPages.Where(p => p.PoetId == null).OrderBy(p => p.Id).Select(p => p.FullUrl).ToListAsync(); + + urls.Remove("/audioclip"); + urls.Remove("/simi"); + urls.Remove("/tags"); + urls.Remove("/amar"); + urls.Remove("/tools"); + urls.Remove("/embed"); + + WriteSitemap(firstSitemap, + urls + ); + + foreach (var poet in await context.GanjoorPoets.ToListAsync()) + { + await jobProgressServiceEF.UpdateJob(job.Id, poet.Id, "", false); + + string poetSitemap = Path.Combine(dir, $"{poet.Id}.xml"); + + WriteSitemap(poetSitemap, + await context.GanjoorPages.Where(p => p.PoetId == poet.Id).OrderBy(p => p.Id).Select(p => p.FullUrl).ToListAsync() + ); + + sitemaps.Add(poetSitemap); + } + + XNamespace ns = "http://www.sitemaps.org/schemas/sitemap/0.9"; + XNamespace xsiNs = "http://www.w3.org/2001/XMLSchema-instance"; + XDocument xDoc = new XDocument( + new XDeclaration("1.0", "UTF-8", "no"), + new XElement(ns + "sitemapindex", + new XAttribute(XNamespace.Xmlns + "xsi", xsiNs), + new XAttribute(xsiNs + "schemaLocation", + "http://www.sitemaps.org/schemas/sitemap/0.9 http://www.sitemaps.org/schemas/sitemap/0.9/sitemap.xsd"), + from sitemap in sitemaps + select new XElement(ns + "sitemap", + new XElement(ns + "loc", $"https://ganjoor.net/{Path.GetFileName(sitemap)}")) + ) + ); + + xDoc.Save(xmlSitemap); + + await jobProgressServiceEF.UpdateJob(job.Id, 100, "", true); + } + catch(Exception exp) + { + await jobProgressServiceEF.UpdateJob(job.Id, 100, "", false, exp.ToString()); + } + + } + }); + return new RServiceResult(true); + } + catch(Exception exp) + { + return new RServiceResult(false, exp.ToString()); + } + } + } +} \ No newline at end of file diff --git a/RMuseum/appsettings.json b/RMuseum/appsettings.json index e40ac904..6255b786 100644 --- a/RMuseum/appsettings.json +++ b/RMuseum/appsettings.json @@ -71,7 +71,8 @@ "ShowAccountInfo": "True" }, "Ganjoor": { - "SystemEmail": "admin@ganjoor.net" + "SystemEmail": "admin@ganjoor.net", + "SitemapLocation": "C:\\inetpub\\ganjoor\\wwwroot\\sitemap.xml" }, "AllowedHosts": "*" }