From 9739fc9f3415dd408235b4315433a60b1afcdf72 Mon Sep 17 00:00:00 2001 From: Hamid Reza Mohammadi Date: Fri, 13 Dec 2024 21:32:31 +0330 Subject: [PATCH] #455 processing poems by poets --- RMuseum/Controllers/ArtifactController.cs | 6 +- RMuseum/RMuseum.xml | 8 +- RMuseum/Services/IArtifactService.cs | 4 +- .../ArtificatService-AIImageCreate.cs | 449 +++++++++--------- 4 files changed, 225 insertions(+), 242 deletions(-) diff --git a/RMuseum/Controllers/ArtifactController.cs b/RMuseum/Controllers/ArtifactController.cs index 9c93d66c..0cf97e57 100644 --- a/RMuseum/Controllers/ArtifactController.cs +++ b/RMuseum/Controllers/ArtifactController.cs @@ -2112,8 +2112,6 @@ namespace RMuseum.Controllers /// /// create images for ganjoor /// - /// - /// /// /// @@ -2122,11 +2120,11 @@ namespace RMuseum.Controllers [ProducesResponseType((int)HttpStatusCode.OK)] [ProducesResponseType((int)HttpStatusCode.BadRequest, Type = typeof(string))] [ProducesResponseType((int)HttpStatusCode.Unauthorized)] - public async Task OpenAIStartCreatingImagesForPoemsAsync(int startFrom = 0, int count = 0, int poetId = 0) + public async Task OpenAIStartCreatingImagesForPoemsAsync(int poetId = 0) { try { - await _artifactService.OpenAIStartCreatingImagesForPoemsAsync(startFrom, count, poetId); + await _artifactService.OpenAIStartCreatingImagesForPoemsAsync(poetId); return Ok(); } catch (Exception exp) diff --git a/RMuseum/RMuseum.xml b/RMuseum/RMuseum.xml index 22a0da55..242ac65a 100644 --- a/RMuseum/RMuseum.xml +++ b/RMuseum/RMuseum.xml @@ -636,12 +636,10 @@ - + create images for ganjoor - - @@ -15268,12 +15266,10 @@ - + create images for ganjoor - - diff --git a/RMuseum/Services/IArtifactService.cs b/RMuseum/Services/IArtifactService.cs index f14f3588..c4680a8f 100644 --- a/RMuseum/Services/IArtifactService.cs +++ b/RMuseum/Services/IArtifactService.cs @@ -602,11 +602,9 @@ namespace RMuseum.Services /// /// create images for ganjoor /// - /// - /// /// /// - Task OpenAIStartCreatingImagesForPoemsAsync(int startFrom, int count, int poetId); + Task OpenAIStartCreatingImagesForPoemsAsync(int poetId); } } diff --git a/RMuseum/Services/Implementation/ArtifactService-Partials/ArtificatService-AIImageCreate.cs b/RMuseum/Services/Implementation/ArtifactService-Partials/ArtificatService-AIImageCreate.cs index 9bd087d2..dc71425e 100644 --- a/RMuseum/Services/Implementation/ArtifactService-Partials/ArtificatService-AIImageCreate.cs +++ b/RMuseum/Services/Implementation/ArtifactService-Partials/ArtificatService-AIImageCreate.cs @@ -24,7 +24,7 @@ namespace RMuseum.Services.Implementation /// public partial class ArtifactService : IArtifactService { - public async Task OpenAIStartCreatingImagesForPoemsAsync(int startFrom, int count, int poetId) + public async Task OpenAIStartCreatingImagesForPoemsAsync(int poetId) { string systemEmail = $"{Configuration.GetSection("Ganjoor")["SystemEmail"]}"; var systemUserId = (Guid)(await _userService.FindUserByEmail(systemEmail)).Result.Id; @@ -36,7 +36,7 @@ namespace RMuseum.Services.Implementation using (RMuseumDbContext context = new RMuseumDbContext(new DbContextOptions())) //this is long running job, so _context might be already been freed/collected by GC { LongRunningJobProgressServiceEF jobProgressServiceEF = new LongRunningJobProgressServiceEF(context); - var job = (await jobProgressServiceEF.NewJob($"OpenAIStartCreatingImagesForPoems - start: {startFrom} - count: {count} - poet: {poetId}", "Open AI initialization")).Result; + var job = (await jobProgressServiceEF.NewJob($"OpenAIStartCreatingImagesForPoems", "Open AI initialization")).Result; try { @@ -72,42 +72,28 @@ namespace RMuseum.Services.Implementation } await jobProgressServiceEF.UpdateJob(job.Id, 0, $"Query Poems"); - var poems = - poetId != 0 ? - await context.GanjoorPoems.Include(p => p.Cat).AsNoTracking().Where(p => p.Cat.PoetId == poetId).ToListAsync() : - count == 0 ? - startFrom == 0 ? - await context.GanjoorPoems.AsNoTracking().ToListAsync() - : - await context.GanjoorPoems.AsNoTracking() - .Skip(startFrom) - .ToListAsync() - : - await context.GanjoorPoems.AsNoTracking() - .Skip(startFrom) - .Take(count) - .ToListAsync(); - await jobProgressServiceEF.UpdateJob(job.Id, 0, $"Poems count = {poems.Count}"); - - - await jobProgressServiceEF.UpdateJob(job.Id, 0, $"Starting, {poems.Count}"); - - for (var i = 0; i < poems.Count; i++) + var poets = await context.GanjoorPoets.AsNoTracking().Where(p => poetId == 0 || p.Id == poetId).OrderBy(p => p.Id).ToListAsync(); + foreach (var poet in poets) { - var poem = poems[i]; - await jobProgressServiceEF.UpdateJob(job.Id, i, $"{i} از {poems.Count} - {poem.FullTitle}"); - if (true == await context.GanjoorLinks.Where(l => l.ArtifactId == book.Id && l.GanjoorPostId == poem.Id).AnyAsync()) - continue; - - if (!string.IsNullOrEmpty(poem.PlainText)) + var poems = await context.GanjoorPoems.Include(p => p.Cat).AsNoTracking().Where(p => p.Cat.PoetId == poet.Id).ToListAsync(); + await jobProgressServiceEF.UpdateJob(job.Id, 0, $"PoetId = {poetId}, Starting, {poems.Count}"); + for (var i = 0; i < poems.Count; i++) { - await jobProgressServiceEF.UpdateJob(job.Id, i, $"{i} از {poems.Count} - {poem.FullTitle} - 1"); - //1 - bool hasStories = false; - var hasStoriesResult = await openAiService.ChatCompletion.CreateCompletion(new ChatCompletionCreateRequest + var poem = poems[i]; + await jobProgressServiceEF.UpdateJob(job.Id, i, $"{i} از {poems.Count} - {poem.FullTitle}"); + + if (true == await context.GanjoorLinks.Where(l => l.ArtifactId == book.Id && l.GanjoorPostId == poem.Id).AnyAsync()) + continue; + + if (!string.IsNullOrEmpty(poem.PlainText)) { - Messages = new List + await jobProgressServiceEF.UpdateJob(job.Id, i, $"{i} از {poems.Count} - {poem.FullTitle} - 1"); + //1 + bool hasStories = false; + var hasStoriesResult = await openAiService.ChatCompletion.CreateCompletion(new ChatCompletionCreateRequest + { + Messages = new List { ChatMessage.FromSystem( "آیا در متن زیر داستانی وجود دارد؟ اگر داستانی وجود ندارد 0 و اگر یک یا چند داستان وجود دارد 1 برگردان.."+ @@ -120,28 +106,28 @@ namespace RMuseum.Services.Implementation + poem.PlainText ), }, - Model = Betalgo.Ranul.OpenAI.ObjectModels.Models.Gpt_4o, - }); - if (hasStoriesResult.Successful) - { - string resHasStories = hasStoriesResult.Choices.First().Message.Content; - if (resHasStories.Contains("اکتبر")) + Model = Betalgo.Ranul.OpenAI.ObjectModels.Models.Gpt_4o, + }); + if (hasStoriesResult.Successful) { - resHasStories = "0"; + string resHasStories = hasStoriesResult.Choices.First().Message.Content; + if (resHasStories.Contains("اکتبر")) + { + resHasStories = "0"; + } + hasStories = resHasStories != "0"; + } + if (!hasStories) + { + await jobProgressServiceEF.UpdateJob(job.Id, i, $"{i} از {poems.Count} - {poem.FullTitle} - بدون داستان"); + continue; } - hasStories = resHasStories != "0"; - } - if (!hasStories) - { - await jobProgressServiceEF.UpdateJob(job.Id, i, $"{i} از {poems.Count} - {poem.FullTitle} - بدون داستان"); - continue; - } - //2 - await jobProgressServiceEF.UpdateJob(job.Id, i, $"{i} از {poems.Count} - {poem.FullTitle} - 2"); - var storyResult = await openAiService.ChatCompletion.CreateCompletion(new ChatCompletionCreateRequest - { - Messages = new List + //2 + await jobProgressServiceEF.UpdateJob(job.Id, i, $"{i} از {poems.Count} - {poem.FullTitle} - 2"); + var storyResult = await openAiService.ChatCompletion.CreateCompletion(new ChatCompletionCreateRequest + { + Messages = new List { ChatMessage.FromSystem( "اگر در متن زیر داستانی وجود دارد آن را به نثر ساده بنویس. اگر چند داستان وجود دارد سعی کن همه را به طور خلاصه بیان کنی. متنت حالت نقل داستان داشته باشد و در آن نتیجه‌گیری نکن. اگر داستانی وجود ندارد خالی برگردان و هیچ توضیح اضافه‌ای مثل این که داستانی وجود ندارد نده."+ @@ -154,31 +140,31 @@ namespace RMuseum.Services.Implementation + poem.PlainText ), }, - Model = Betalgo.Ranul.OpenAI.ObjectModels.Models.Gpt_4o, - }); - string story = ""; - if (storyResult.Successful) - { - story = storyResult.Choices.First().Message.Content; - if (story.Contains("اکتبر")) + Model = Betalgo.Ranul.OpenAI.ObjectModels.Models.Gpt_4o, + }); + string story = ""; + if (storyResult.Successful) { - story = ""; + story = storyResult.Choices.First().Message.Content; + if (story.Contains("اکتبر")) + { + story = ""; + } + story = story.Trim(); } - story = story.Trim(); - } - if (string.IsNullOrEmpty(story)) - { - await jobProgressServiceEF.UpdateJob(job.Id, i, $"{i} از {poems.Count} - {poem.FullTitle} - بدون متن داستان"); - continue; - } + if (string.IsNullOrEmpty(story)) + { + await jobProgressServiceEF.UpdateJob(job.Id, i, $"{i} از {poems.Count} - {poem.FullTitle} - بدون متن داستان"); + continue; + } - await jobProgressServiceEF.UpdateJob(job.Id, i, $"{i} از {poems.Count} - {poem.FullTitle} - 3"); + await jobProgressServiceEF.UpdateJob(job.Id, i, $"{i} از {poems.Count} - {poem.FullTitle} - 3"); - //3 - var depictionResult = await openAiService.ChatCompletion.CreateCompletion(new ChatCompletionCreateRequest - { - Messages = new List + //3 + var depictionResult = await openAiService.ChatCompletion.CreateCompletion(new ChatCompletionCreateRequest + { + Messages = new List { ChatMessage.FromSystem( "Create a DALL-E propmt for depicting this story using an photorealistic image with Persian miniature style:" @@ -188,195 +174,200 @@ namespace RMuseum.Services.Implementation story ), }, - Model = Betalgo.Ranul.OpenAI.ObjectModels.Models.Gpt_4o, - }); + Model = Betalgo.Ranul.OpenAI.ObjectModels.Models.Gpt_4o, + }); - string prompt = ""; - if (depictionResult.Successful) - { - prompt = depictionResult.Choices.First().Message.Content; - prompt = prompt.Trim(); - } - - if (string.IsNullOrEmpty(prompt)) - { - await jobProgressServiceEF.UpdateJob(job.Id, i, $"{i} از {poems.Count} - {poem.FullTitle} - بدون فرمان"); - continue; - } - - await jobProgressServiceEF.UpdateJob(job.Id, i, $"{i} از {poems.Count} - {poem.FullTitle} - 4"); - - //4 - var imageCreationResult = await openAiService.Image.CreateImage(new ImageCreateRequest - ( - prompt - ) - { - Model = "dall-e-3" - } - ); - if (imageCreationResult.Successful) - { - string imageUrl = imageCreationResult.Results.Select(u => u.Url).First(); - - if (!string.IsNullOrEmpty(imageUrl)) + string prompt = ""; + if (depictionResult.Successful) { + prompt = depictionResult.Choices.First().Message.Content; + prompt = prompt.Trim(); + } + if (string.IsNullOrEmpty(prompt)) + { + await jobProgressServiceEF.UpdateJob(job.Id, i, $"{i} از {poems.Count} - {poem.FullTitle} - بدون فرمان"); + continue; + } - int order = 1 + await context.Items.Where(i => i.RArtifactMasterRecordId == book.Id).CountAsync(); + await jobProgressServiceEF.UpdateJob(job.Id, i, $"{i} از {poems.Count} - {poem.FullTitle} - 4"); - RArtifactItemRecord page = new RArtifactItemRecord() + //4 + var imageCreationResult = await openAiService.Image.CreateImage(new ImageCreateRequest + ( + prompt + ) + { + Model = "dall-e-3" + } + ); + if (imageCreationResult.Successful) + { + string imageUrl = imageCreationResult.Results.Select(u => u.Url).First(); + + if (!string.IsNullOrEmpty(imageUrl)) { - Name = $"تصویر {order} - {poem.FullTitle}", - NameInEnglish = $"Image {order} of {book.NameInEnglish}", - Description = story, - DescriptionInEnglish = prompt, - Order = order, - FriendlyUrl = $"p{$"{order}".PadLeft(7, '0')}", - LastModified = DateTime.Now, - RArtifactMasterRecordId = book.Id, - }; - var promptTag = await TagHandler.PrepareAttribute(context, "AI Prompt", prompt, 1); - var ganjoorTag = await TagHandler.PrepareAttribute(context, "Ganjoor Link", poem.FullTitle, 1); - ganjoorTag.ValueSupplement = $"https://ganjoor.net{poem.FullUrl}"; - var storyTag = await TagHandler.PrepareAttribute(context, "Story", story, 1); - page.Tags = [promptTag, ganjoorTag, storyTag]; - if ( - File.Exists - ( - Path.Combine(Path.Combine(Path.Combine(_pictureFileService.ImageStoragePath, friendlyUrl), "orig"), $"{order}".PadLeft(7, '0') + ".jpg") + int order = 1 + await context.Items.Where(i => i.RArtifactMasterRecordId == book.Id).CountAsync(); + + RArtifactItemRecord page = new RArtifactItemRecord() + { + Name = $"تصویر {order} - {poem.FullTitle}", + NameInEnglish = $"Image {order} of {book.NameInEnglish}", + Description = story, + DescriptionInEnglish = prompt, + Order = order, + FriendlyUrl = $"p{$"{order}".PadLeft(7, '0')}", + LastModified = DateTime.Now, + RArtifactMasterRecordId = book.Id, + }; + + var promptTag = await TagHandler.PrepareAttribute(context, "AI Prompt", prompt, 1); + var ganjoorTag = await TagHandler.PrepareAttribute(context, "Ganjoor Link", poem.FullTitle, 1); + ganjoorTag.ValueSupplement = $"https://ganjoor.net{poem.FullUrl}"; + var storyTag = await TagHandler.PrepareAttribute(context, "Story", story, 1); + page.Tags = [promptTag, ganjoorTag, storyTag]; + + if ( + File.Exists + ( + Path.Combine(Path.Combine(Path.Combine(_pictureFileService.ImageStoragePath, friendlyUrl), "orig"), $"{order}".PadLeft(7, '0') + ".jpg") + ) + ) + { + File.Delete + ( + Path.Combine(Path.Combine(Path.Combine(_pictureFileService.ImageStoragePath, friendlyUrl), "orig"), $"{order}".PadLeft(7, '0') + ".jpg") + ); + } + if ( + + File.Exists + ( + Path.Combine(Path.Combine(Path.Combine(_pictureFileService.ImageStoragePath, friendlyUrl), "norm"), $"{order}".PadLeft(7, '0') + ".jpg") + ) + + ) + { + File.Delete + ( + Path.Combine(Path.Combine(Path.Combine(_pictureFileService.ImageStoragePath, friendlyUrl), "norm"), $"{order}".PadLeft(7, '0') + ".jpg") + ); + } + if ( + + File.Exists + ( + Path.Combine(Path.Combine(Path.Combine(_pictureFileService.ImageStoragePath, friendlyUrl), "thumb"), $"{order}".PadLeft(7, '0') + ".jpg") + ) + ) + { + File.Delete + ( + Path.Combine(Path.Combine(Path.Combine(_pictureFileService.ImageStoragePath, friendlyUrl), "thumb"), $"{order}".PadLeft(7, '0') + ".jpg") + ); + } + + using (var httpClient = new HttpClient()) + { + var imageData = await httpClient.GetByteArrayAsync(imageUrl); + using var stream = new MemoryStream(imageData); + using var originalImage = Image.FromStream(stream); + using (Stream jpegStream = new MemoryStream()) + { + originalImage.Save(jpegStream, System.Drawing.Imaging.ImageFormat.Jpeg); + jpegStream.Seek(0, SeekOrigin.Begin); + + RServiceResult picture = await _pictureFileService.Add(page.Name, page.Description, 1, null, imageUrl, jpegStream, $"{order}".PadLeft(7, '0') + ".jpg", friendlyUrl); + if (picture.Result == null) + { + throw new Exception($"_pictureFileService.Add : {picture.ExceptionString}"); + } + page.Images = [picture.Result]; + page.CoverImageIndex = 0; + } + context.Add(page); + await context.SaveChangesAsync(); + } + + + if (book.ItemCount == 0) + { + book.CoverImage = RPictureFile.Duplicate(page.Images.First()); + book.Status = PublishStatus.Published; + book.ItemCount = order; + context.Update(book); + await context.SaveChangesAsync(); + + var resFTPUpload = await _UploadArtifactToExternalServer(book, context, false); + if (!string.IsNullOrEmpty(resFTPUpload.ExceptionString)) + { + await jobProgressServiceEF.UpdateJob(job.Id, 100, "", false, $"UploadArtifactToExternalServer: {resFTPUpload.ExceptionString}"); + context.Update(job); + await context.SaveChangesAsync(); + return; + } + } + else + { + book.ItemCount = order; + context.Update(book); + await context.SaveChangesAsync(); + } + + var resUpload = await _UploadArtifactPageToExternalServer(page, context, friendlyUrl, false); + if (resUpload.Result != true) + { + await jobProgressServiceEF.UpdateJob(job.Id, 100, "", false, resUpload.ExceptionString); + return; + } + - ) - { File.Delete ( Path.Combine(Path.Combine(Path.Combine(_pictureFileService.ImageStoragePath, friendlyUrl), "orig"), $"{order}".PadLeft(7, '0') + ".jpg") ); - } - if ( - File.Exists - ( - Path.Combine(Path.Combine(Path.Combine(_pictureFileService.ImageStoragePath, friendlyUrl), "norm"), $"{order}".PadLeft(7, '0') + ".jpg") - ) - - ) - { File.Delete ( Path.Combine(Path.Combine(Path.Combine(_pictureFileService.ImageStoragePath, friendlyUrl), "norm"), $"{order}".PadLeft(7, '0') + ".jpg") ); - } - if ( - - File.Exists - ( - Path.Combine(Path.Combine(Path.Combine(_pictureFileService.ImageStoragePath, friendlyUrl), "thumb"), $"{order}".PadLeft(7, '0') + ".jpg") - ) - ) - { File.Delete ( Path.Combine(Path.Combine(Path.Combine(_pictureFileService.ImageStoragePath, friendlyUrl), "thumb"), $"{order}".PadLeft(7, '0') + ".jpg") ); - } - using (var httpClient = new HttpClient()) - { - var imageData = await httpClient.GetByteArrayAsync(imageUrl); - using var stream = new MemoryStream(imageData); - using var originalImage = Image.FromStream(stream); - using (Stream jpegStream = new MemoryStream()) - { - originalImage.Save(jpegStream, System.Drawing.Imaging.ImageFormat.Jpeg); - jpegStream.Seek(0, SeekOrigin.Begin); - RServiceResult picture = await _pictureFileService.Add(page.Name, page.Description, 1, null, imageUrl, jpegStream, $"{order}".PadLeft(7, '0') + ".jpg", friendlyUrl); - if (picture.Result == null) - { - throw new Exception($"_pictureFileService.Add : {picture.ExceptionString}"); - } - page.Images = [picture.Result]; - page.CoverImageIndex = 0; - } - context.Add(page); + GanjoorLink suggestion = + new GanjoorLink() + { + GanjoorPostId = poem.Id, + GanjoorTitle = poem.FullTitle, + GanjoorUrl = $"https://ganjoor.net{poem.FullUrl}", + ArtifactId = book.Id, + ItemId = page.Id, + SuggestedById = systemUserId, + SuggestionDate = DateTime.Now, + ReviewResult = ReviewResult.Approved, + DisplayOnPage = true, + Synchronized = true, + }; + + context.GanjoorLinks.Add(suggestion); await context.SaveChangesAsync(); } - - - if (book.ItemCount == 0) - { - book.CoverImage = RPictureFile.Duplicate(page.Images.First()); - book.Status = PublishStatus.Published; - book.ItemCount = order; - context.Update(book); - await context.SaveChangesAsync(); - - var resFTPUpload = await _UploadArtifactToExternalServer(book, context, false); - if (!string.IsNullOrEmpty(resFTPUpload.ExceptionString)) - { - await jobProgressServiceEF.UpdateJob(job.Id, 100, "", false, $"UploadArtifactToExternalServer: {resFTPUpload.ExceptionString}"); - context.Update(job); - await context.SaveChangesAsync(); - return; - } - } - else - { - book.ItemCount = order; - context.Update(book); - await context.SaveChangesAsync(); - } - - var resUpload = await _UploadArtifactPageToExternalServer(page, context, friendlyUrl, false); - if (resUpload.Result != true) - { - await jobProgressServiceEF.UpdateJob(job.Id, 100, "", false, resUpload.ExceptionString); - return; - } - - - File.Delete - ( - Path.Combine(Path.Combine(Path.Combine(_pictureFileService.ImageStoragePath, friendlyUrl), "orig"), $"{order}".PadLeft(7, '0') + ".jpg") - ); - - File.Delete - ( - Path.Combine(Path.Combine(Path.Combine(_pictureFileService.ImageStoragePath, friendlyUrl), "norm"), $"{order}".PadLeft(7, '0') + ".jpg") - ); - File.Delete - ( - Path.Combine(Path.Combine(Path.Combine(_pictureFileService.ImageStoragePath, friendlyUrl), "thumb"), $"{order}".PadLeft(7, '0') + ".jpg") - ); - - - GanjoorLink suggestion = - new GanjoorLink() - { - GanjoorPostId = poem.Id, - GanjoorTitle = poem.FullTitle, - GanjoorUrl = $"https://ganjoor.net{poem.FullUrl}", - ArtifactId = book.Id, - ItemId = page.Id, - SuggestedById = systemUserId, - SuggestionDate = DateTime.Now, - ReviewResult = ReviewResult.Approved, - DisplayOnPage = true, - Synchronized = true, - }; - - context.GanjoorLinks.Add(suggestion); - await context.SaveChangesAsync(); } } } } + + + + await jobProgressServiceEF.UpdateJob(job.Id, 100, "", true); } catch (Exception exp)