anas/divan
1
0
Fork 0
Code Issues 12 Pull Requests Actions Packages Projects Releases Wiki Activity

#455 processing poems by poets

Browse Source
This commit is contained in:
Hamid Reza Mohammadi 2024-12-13 21:32:31 +03:30
parent 2dcefb5c34
commit 9739fc9f34
4 changed files with 225 additions and 242 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
RMuseum/Controllers/ArtifactController.cs
View File

@ -2112,8 +2112,6 @@ namespace RMuseum.Controllers
/// <summary>
/// create images for ganjoor
/// </summary>
/// <param name="startFrom"></param>
/// <param name="count"></param>
/// <param name="poetId"></param>
/// <returns></returns>
@ -2122,11 +2120,11 @@ namespace RMuseum.Controllers
[ProducesResponseType((int)HttpStatusCode.OK)]
[ProducesResponseType((int)HttpStatusCode.BadRequest, Type = typeof(string))]
[ProducesResponseType((int)HttpStatusCode.Unauthorized)]
public async Task<IActionResult> OpenAIStartCreatingImagesForPoemsAsync(int startFrom = 0, int count = 0, int poetId = 0)
public async Task<IActionResult> OpenAIStartCreatingImagesForPoemsAsync(int poetId = 0)
{
try
{
await _artifactService.OpenAIStartCreatingImagesForPoemsAsync(startFrom, count, poetId);
await _artifactService.OpenAIStartCreatingImagesForPoemsAsync(poetId);
return Ok();
}
catch (Exception exp)

8
RMuseum/RMuseum.xml
View File

@ -636,12 +636,10 @@
<param name="skipUpload"></param>
<returns></returns>
</member>
<member name="M:RMuseum.Controllers.ArtifactController.OpenAIStartCreatingImagesForPoemsAsync(System.Int32,System.Int32,System.Int32)">
<member name="M:RMuseum.Controllers.ArtifactController.OpenAIStartCreatingImagesForPoemsAsync(System.Int32)">
<summary>
create images for ganjoor
</summary>
<param name="startFrom"></param>
<param name="count"></param>
<param name="poetId"></param>
<returns></returns>
</member>
@ -15268,12 +15266,10 @@
<param name="skipUpload"></param>
<returns></returns>
</member>
<member name="M:RMuseum.Services.IArtifactService.OpenAIStartCreatingImagesForPoemsAsync(System.Int32,System.Int32,System.Int32)">
<member name="M:RMuseum.Services.IArtifactService.OpenAIStartCreatingImagesForPoemsAsync(System.Int32)">
<summary>
create images for ganjoor
</summary>
<param name="startFrom"></param>
<param name="count"></param>
<param name="poetId"></param>
<returns></returns>
</member>

4
RMuseum/Services/IArtifactService.cs
View File

@ -602,11 +602,9 @@ namespace RMuseum.Services
/// <summary>
/// create images for ganjoor
/// </summary>
/// <param name="startFrom"></param>
/// <param name="count"></param>
/// <param name="poetId"></param>
/// <returns></returns>
Task OpenAIStartCreatingImagesForPoemsAsync(int startFrom, int count, int poetId);
Task OpenAIStartCreatingImagesForPoemsAsync(int poetId);
}
}

449
RMuseum/Services/Implementation/ArtifactService-Partials/ArtificatService-AIImageCreate.cs
View File

@ -24,7 +24,7 @@ namespace RMuseum.Services.Implementation
/// </summary>
public partial class ArtifactService : IArtifactService
{
public async Task OpenAIStartCreatingImagesForPoemsAsync(int startFrom, int count, int poetId)
public async Task OpenAIStartCreatingImagesForPoemsAsync(int poetId)
{
string systemEmail = $"{Configuration.GetSection("Ganjoor")["SystemEmail"]}";
var systemUserId = (Guid)(await _userService.FindUserByEmail(systemEmail)).Result.Id;
@ -36,7 +36,7 @@ namespace RMuseum.Services.Implementation
using (RMuseumDbContext context = new RMuseumDbContext(new DbContextOptions<RMuseumDbContext>())) //this is long running job, so _context might be already been freed/collected by GC
{
LongRunningJobProgressServiceEF jobProgressServiceEF = new LongRunningJobProgressServiceEF(context);
var job = (await jobProgressServiceEF.NewJob($"OpenAIStartCreatingImagesForPoems - start: {startFrom} - count: {count} - poet: {poetId}", "Open AI initialization")).Result;
var job = (await jobProgressServiceEF.NewJob($"OpenAIStartCreatingImagesForPoems", "Open AI initialization")).Result;
try
{
@ -72,42 +72,28 @@ namespace RMuseum.Services.Implementation
}
await jobProgressServiceEF.UpdateJob(job.Id, 0, $"Query Poems");
var poems =
poetId != 0 ?
await context.GanjoorPoems.Include(p => p.Cat).AsNoTracking().Where(p => p.Cat.PoetId == poetId).ToListAsync() :
count == 0 ?
startFrom == 0 ?
await context.GanjoorPoems.AsNoTracking().ToListAsync()
:
await context.GanjoorPoems.AsNoTracking()
.Skip(startFrom)
.ToListAsync()
:
await context.GanjoorPoems.AsNoTracking()
.Skip(startFrom)
.Take(count)
.ToListAsync();
await jobProgressServiceEF.UpdateJob(job.Id, 0, $"Poems count = {poems.Count}");
await jobProgressServiceEF.UpdateJob(job.Id, 0, $"Starting, {poems.Count}");
for (var i = 0; i < poems.Count; i++)
var poets = await context.GanjoorPoets.AsNoTracking().Where(p => poetId == 0 || p.Id == poetId).OrderBy(p => p.Id).ToListAsync();
foreach (var poet in poets)
{
var poem = poems[i];
await jobProgressServiceEF.UpdateJob(job.Id, i, $"{i} از {poems.Count} - {poem.FullTitle}");
if (true == await context.GanjoorLinks.Where(l => l.ArtifactId == book.Id && l.GanjoorPostId == poem.Id).AnyAsync())
continue;
if (!string.IsNullOrEmpty(poem.PlainText))
var poems = await context.GanjoorPoems.Include(p => p.Cat).AsNoTracking().Where(p => p.Cat.PoetId == poet.Id).ToListAsync();
await jobProgressServiceEF.UpdateJob(job.Id, 0, $"PoetId = {poetId}, Starting, {poems.Count}");
for (var i = 0; i < poems.Count; i++)
{
await jobProgressServiceEF.UpdateJob(job.Id, i, $"{i} از {poems.Count} - {poem.FullTitle} - 1");
//1
bool hasStories = false;
var hasStoriesResult = await openAiService.ChatCompletion.CreateCompletion(new ChatCompletionCreateRequest
var poem = poems[i];
await jobProgressServiceEF.UpdateJob(job.Id, i, $"{i} از {poems.Count} - {poem.FullTitle}");
if (true == await context.GanjoorLinks.Where(l => l.ArtifactId == book.Id && l.GanjoorPostId == poem.Id).AnyAsync())
continue;
if (!string.IsNullOrEmpty(poem.PlainText))
{
Messages = new List<ChatMessage>
await jobProgressServiceEF.UpdateJob(job.Id, i, $"{i} از {poems.Count} - {poem.FullTitle} - 1");
//1
bool hasStories = false;
var hasStoriesResult = await openAiService.ChatCompletion.CreateCompletion(new ChatCompletionCreateRequest
{
Messages = new List<ChatMessage>
{
ChatMessage.FromSystem(
"آیا در متن زیر داستانی وجود دارد؟ اگر داستانی وجود ندارد 0 و اگر یک یا چند داستان وجود دارد 1 برگردان.."+
@ -120,28 +106,28 @@ namespace RMuseum.Services.Implementation
+ poem.PlainText
),
},
Model = Betalgo.Ranul.OpenAI.ObjectModels.Models.Gpt_4o,
});
if (hasStoriesResult.Successful)
{
string resHasStories = hasStoriesResult.Choices.First().Message.Content;
if (resHasStories.Contains("اکتبر"))
Model = Betalgo.Ranul.OpenAI.ObjectModels.Models.Gpt_4o,
});
if (hasStoriesResult.Successful)
{
resHasStories = "0";
string resHasStories = hasStoriesResult.Choices.First().Message.Content;
if (resHasStories.Contains("اکتبر"))
{
resHasStories = "0";
}
hasStories = resHasStories != "0";
}
if (!hasStories)
{
await jobProgressServiceEF.UpdateJob(job.Id, i, $"{i} از {poems.Count} - {poem.FullTitle} - بدون داستان");
continue;
}
hasStories = resHasStories != "0";
}
if (!hasStories)
{
await jobProgressServiceEF.UpdateJob(job.Id, i, $"{i} از {poems.Count} - {poem.FullTitle} - بدون داستان");
continue;
}
//2
await jobProgressServiceEF.UpdateJob(job.Id, i, $"{i} از {poems.Count} - {poem.FullTitle} - 2");
var storyResult = await openAiService.ChatCompletion.CreateCompletion(new ChatCompletionCreateRequest
{
Messages = new List<ChatMessage>
//2
await jobProgressServiceEF.UpdateJob(job.Id, i, $"{i} از {poems.Count} - {poem.FullTitle} - 2");
var storyResult = await openAiService.ChatCompletion.CreateCompletion(new ChatCompletionCreateRequest
{
Messages = new List<ChatMessage>
{
ChatMessage.FromSystem(
"اگر در متن زیر داستانی وجود دارد آن را به نثر ساده بنویس. اگر چند داستان وجود دارد سعی کن همه را به طور خلاصه بیان کنی. متنت حالت نقل داستان داشته باشد و در آن نتیجه‌گیری نکن. اگر داستانی وجود ندارد خالی برگردان و هیچ توضیح اضافه‌ای مثل این که داستانی وجود ندارد نده."+
@ -154,31 +140,31 @@ namespace RMuseum.Services.Implementation
+ poem.PlainText
),
},
Model = Betalgo.Ranul.OpenAI.ObjectModels.Models.Gpt_4o,
});
string story = "";
if (storyResult.Successful)
{
story = storyResult.Choices.First().Message.Content;
if (story.Contains("اکتبر"))
Model = Betalgo.Ranul.OpenAI.ObjectModels.Models.Gpt_4o,
});
string story = "";
if (storyResult.Successful)
{
story = "";
story = storyResult.Choices.First().Message.Content;
if (story.Contains("اکتبر"))
{
story = "";
}
story = story.Trim();
}
story = story.Trim();
}
if (string.IsNullOrEmpty(story))
{
await jobProgressServiceEF.UpdateJob(job.Id, i, $"{i} از {poems.Count} - {poem.FullTitle} - بدون متن داستان");
continue;
}
if (string.IsNullOrEmpty(story))
{
await jobProgressServiceEF.UpdateJob(job.Id, i, $"{i} از {poems.Count} - {poem.FullTitle} - بدون متن داستان");
continue;
}
await jobProgressServiceEF.UpdateJob(job.Id, i, $"{i} از {poems.Count} - {poem.FullTitle} - 3");
await jobProgressServiceEF.UpdateJob(job.Id, i, $"{i} از {poems.Count} - {poem.FullTitle} - 3");
//3
var depictionResult = await openAiService.ChatCompletion.CreateCompletion(new ChatCompletionCreateRequest
{
Messages = new List<ChatMessage>
//3
var depictionResult = await openAiService.ChatCompletion.CreateCompletion(new ChatCompletionCreateRequest
{
Messages = new List<ChatMessage>
{
ChatMessage.FromSystem(
"Create a DALL-E propmt for depicting this story using an photorealistic image with Persian miniature style:"
@ -188,195 +174,200 @@ namespace RMuseum.Services.Implementation
story
),
},
Model = Betalgo.Ranul.OpenAI.ObjectModels.Models.Gpt_4o,
});
Model = Betalgo.Ranul.OpenAI.ObjectModels.Models.Gpt_4o,
});
string prompt = "";
if (depictionResult.Successful)
{
prompt = depictionResult.Choices.First().Message.Content;
prompt = prompt.Trim();
}
if (string.IsNullOrEmpty(prompt))
{
await jobProgressServiceEF.UpdateJob(job.Id, i, $"{i} از {poems.Count} - {poem.FullTitle} - بدون فرمان");
continue;
}
await jobProgressServiceEF.UpdateJob(job.Id, i, $"{i} از {poems.Count} - {poem.FullTitle} - 4");
//4
var imageCreationResult = await openAiService.Image.CreateImage(new ImageCreateRequest
(
prompt
)
{
Model = "dall-e-3"
}
);
if (imageCreationResult.Successful)
{
string imageUrl = imageCreationResult.Results.Select(u => u.Url).First();
if (!string.IsNullOrEmpty(imageUrl))
string prompt = "";
if (depictionResult.Successful)
{
prompt = depictionResult.Choices.First().Message.Content;
prompt = prompt.Trim();
}
if (string.IsNullOrEmpty(prompt))
{
await jobProgressServiceEF.UpdateJob(job.Id, i, $"{i} از {poems.Count} - {poem.FullTitle} - بدون فرمان");
continue;
}
int order = 1 + await context.Items.Where(i => i.RArtifactMasterRecordId == book.Id).CountAsync();
await jobProgressServiceEF.UpdateJob(job.Id, i, $"{i} از {poems.Count} - {poem.FullTitle} - 4");
RArtifactItemRecord page = new RArtifactItemRecord()
//4
var imageCreationResult = await openAiService.Image.CreateImage(new ImageCreateRequest
(
prompt
)
{
Model = "dall-e-3"
}
);
if (imageCreationResult.Successful)
{
string imageUrl = imageCreationResult.Results.Select(u => u.Url).First();
if (!string.IsNullOrEmpty(imageUrl))
{
Name = $"تصویر {order} - {poem.FullTitle}",
NameInEnglish = $"Image {order} of {book.NameInEnglish}",
Description = story,
DescriptionInEnglish = prompt,
Order = order,
FriendlyUrl = $"p{$"{order}".PadLeft(7, '0')}",
LastModified = DateTime.Now,
RArtifactMasterRecordId = book.Id,
};
var promptTag = await TagHandler.PrepareAttribute(context, "AI Prompt", prompt, 1);
var ganjoorTag = await TagHandler.PrepareAttribute(context, "Ganjoor Link", poem.FullTitle, 1);
ganjoorTag.ValueSupplement = $"https://ganjoor.net{poem.FullUrl}";
var storyTag = await TagHandler.PrepareAttribute(context, "Story", story, 1);
page.Tags = [promptTag, ganjoorTag, storyTag];
if (
File.Exists
(
Path.Combine(Path.Combine(Path.Combine(_pictureFileService.ImageStoragePath, friendlyUrl), "orig"), $"{order}".PadLeft(7, '0') + ".jpg")
int order = 1 + await context.Items.Where(i => i.RArtifactMasterRecordId == book.Id).CountAsync();
RArtifactItemRecord page = new RArtifactItemRecord()
{
Name = $"تصویر {order} - {poem.FullTitle}",
NameInEnglish = $"Image {order} of {book.NameInEnglish}",
Description = story,
DescriptionInEnglish = prompt,
Order = order,
FriendlyUrl = $"p{$"{order}".PadLeft(7, '0')}",
LastModified = DateTime.Now,
RArtifactMasterRecordId = book.Id,
};
var promptTag = await TagHandler.PrepareAttribute(context, "AI Prompt", prompt, 1);
var ganjoorTag = await TagHandler.PrepareAttribute(context, "Ganjoor Link", poem.FullTitle, 1);
ganjoorTag.ValueSupplement = $"https://ganjoor.net{poem.FullUrl}";
var storyTag = await TagHandler.PrepareAttribute(context, "Story", story, 1);
page.Tags = [promptTag, ganjoorTag, storyTag];
if (
File.Exists
(
Path.Combine(Path.Combine(Path.Combine(_pictureFileService.ImageStoragePath, friendlyUrl), "orig"), $"{order}".PadLeft(7, '0') + ".jpg")
)
)
{
File.Delete
(
Path.Combine(Path.Combine(Path.Combine(_pictureFileService.ImageStoragePath, friendlyUrl), "orig"), $"{order}".PadLeft(7, '0') + ".jpg")
);
}
if (
File.Exists
(
Path.Combine(Path.Combine(Path.Combine(_pictureFileService.ImageStoragePath, friendlyUrl), "norm"), $"{order}".PadLeft(7, '0') + ".jpg")
)
)
{
File.Delete
(
Path.Combine(Path.Combine(Path.Combine(_pictureFileService.ImageStoragePath, friendlyUrl), "norm"), $"{order}".PadLeft(7, '0') + ".jpg")
);
}
if (
File.Exists
(
Path.Combine(Path.Combine(Path.Combine(_pictureFileService.ImageStoragePath, friendlyUrl), "thumb"), $"{order}".PadLeft(7, '0') + ".jpg")
)
)
{
File.Delete
(
Path.Combine(Path.Combine(Path.Combine(_pictureFileService.ImageStoragePath, friendlyUrl), "thumb"), $"{order}".PadLeft(7, '0') + ".jpg")
);
}
using (var httpClient = new HttpClient())
{
var imageData = await httpClient.GetByteArrayAsync(imageUrl);
using var stream = new MemoryStream(imageData);
using var originalImage = Image.FromStream(stream);
using (Stream jpegStream = new MemoryStream())
{
originalImage.Save(jpegStream, System.Drawing.Imaging.ImageFormat.Jpeg);
jpegStream.Seek(0, SeekOrigin.Begin);
RServiceResult<RPictureFile> picture = await _pictureFileService.Add(page.Name, page.Description, 1, null, imageUrl, jpegStream, $"{order}".PadLeft(7, '0') + ".jpg", friendlyUrl);
if (picture.Result == null)
{
throw new Exception($"_pictureFileService.Add : {picture.ExceptionString}");
}
page.Images = [picture.Result];
page.CoverImageIndex = 0;
}
context.Add(page);
await context.SaveChangesAsync();
}
if (book.ItemCount == 0)
{
book.CoverImage = RPictureFile.Duplicate(page.Images.First());
book.Status = PublishStatus.Published;
book.ItemCount = order;
context.Update(book);
await context.SaveChangesAsync();
var resFTPUpload = await _UploadArtifactToExternalServer(book, context, false);
if (!string.IsNullOrEmpty(resFTPUpload.ExceptionString))
{
await jobProgressServiceEF.UpdateJob(job.Id, 100, "", false, $"UploadArtifactToExternalServer: {resFTPUpload.ExceptionString}");
context.Update(job);
await context.SaveChangesAsync();
return;
}
}
else
{
book.ItemCount = order;
context.Update(book);
await context.SaveChangesAsync();
}
var resUpload = await _UploadArtifactPageToExternalServer(page, context, friendlyUrl, false);
if (resUpload.Result != true)
{
await jobProgressServiceEF.UpdateJob(job.Id, 100, "", false, resUpload.ExceptionString);
return;
}
)
{
File.Delete
(
Path.Combine(Path.Combine(Path.Combine(_pictureFileService.ImageStoragePath, friendlyUrl), "orig"), $"{order}".PadLeft(7, '0') + ".jpg")
);
}
if (
File.Exists
(
Path.Combine(Path.Combine(Path.Combine(_pictureFileService.ImageStoragePath, friendlyUrl), "norm"), $"{order}".PadLeft(7, '0') + ".jpg")
)
)
{
File.Delete
(
Path.Combine(Path.Combine(Path.Combine(_pictureFileService.ImageStoragePath, friendlyUrl), "norm"), $"{order}".PadLeft(7, '0') + ".jpg")
);
}
if (
File.Exists
(
Path.Combine(Path.Combine(Path.Combine(_pictureFileService.ImageStoragePath, friendlyUrl), "thumb"), $"{order}".PadLeft(7, '0') + ".jpg")
)
)
{
File.Delete
(
Path.Combine(Path.Combine(Path.Combine(_pictureFileService.ImageStoragePath, friendlyUrl), "thumb"), $"{order}".PadLeft(7, '0') + ".jpg")
);
}
using (var httpClient = new HttpClient())
{
var imageData = await httpClient.GetByteArrayAsync(imageUrl);
using var stream = new MemoryStream(imageData);
using var originalImage = Image.FromStream(stream);
using (Stream jpegStream = new MemoryStream())
{
originalImage.Save(jpegStream, System.Drawing.Imaging.ImageFormat.Jpeg);
jpegStream.Seek(0, SeekOrigin.Begin);
RServiceResult<RPictureFile> picture = await _pictureFileService.Add(page.Name, page.Description, 1, null, imageUrl, jpegStream, $"{order}".PadLeft(7, '0') + ".jpg", friendlyUrl);
if (picture.Result == null)
{
throw new Exception($"_pictureFileService.Add : {picture.ExceptionString}");
}
page.Images = [picture.Result];
page.CoverImageIndex = 0;
}
context.Add(page);
GanjoorLink suggestion =
new GanjoorLink()
{
GanjoorPostId = poem.Id,
GanjoorTitle = poem.FullTitle,
GanjoorUrl = $"https://ganjoor.net{poem.FullUrl}",
ArtifactId = book.Id,
ItemId = page.Id,
SuggestedById = systemUserId,
SuggestionDate = DateTime.Now,
ReviewResult = ReviewResult.Approved,
DisplayOnPage = true,
Synchronized = true,
};
context.GanjoorLinks.Add(suggestion);
await context.SaveChangesAsync();
}
if (book.ItemCount == 0)
{
book.CoverImage = RPictureFile.Duplicate(page.Images.First());
book.Status = PublishStatus.Published;
book.ItemCount = order;
context.Update(book);
await context.SaveChangesAsync();
var resFTPUpload = await _UploadArtifactToExternalServer(book, context, false);
if (!string.IsNullOrEmpty(resFTPUpload.ExceptionString))
{
await jobProgressServiceEF.UpdateJob(job.Id, 100, "", false, $"UploadArtifactToExternalServer: {resFTPUpload.ExceptionString}");
context.Update(job);
await context.SaveChangesAsync();
return;
}
}
else
{
book.ItemCount = order;
context.Update(book);
await context.SaveChangesAsync();
}
var resUpload = await _UploadArtifactPageToExternalServer(page, context, friendlyUrl, false);
if (resUpload.Result != true)
{
await jobProgressServiceEF.UpdateJob(job.Id, 100, "", false, resUpload.ExceptionString);
return;
}
File.Delete
(
Path.Combine(Path.Combine(Path.Combine(_pictureFileService.ImageStoragePath, friendlyUrl), "orig"), $"{order}".PadLeft(7, '0') + ".jpg")
);
File.Delete
(
Path.Combine(Path.Combine(Path.Combine(_pictureFileService.ImageStoragePath, friendlyUrl), "norm"), $"{order}".PadLeft(7, '0') + ".jpg")
);
File.Delete
(
Path.Combine(Path.Combine(Path.Combine(_pictureFileService.ImageStoragePath, friendlyUrl), "thumb"), $"{order}".PadLeft(7, '0') + ".jpg")
);
GanjoorLink suggestion =
new GanjoorLink()
{
GanjoorPostId = poem.Id,
GanjoorTitle = poem.FullTitle,
GanjoorUrl = $"https://ganjoor.net{poem.FullUrl}",
ArtifactId = book.Id,
ItemId = page.Id,
SuggestedById = systemUserId,
SuggestionDate = DateTime.Now,
ReviewResult = ReviewResult.Approved,
DisplayOnPage = true,
Synchronized = true,
};
context.GanjoorLinks.Add(suggestion);
await context.SaveChangesAsync();
}
}
}
}
await jobProgressServiceEF.UpdateJob(job.Id, 100, "", true);
}
catch (Exception exp)