anas/divan
1
0
Fork 0
Code Issues 12 Pull Requests Actions Packages Projects Releases Wiki Activity

#453 multiple threads for ai

Browse Source
This commit is contained in:
Hamid Reza Mohammadi 2024-12-07 21:58:01 +03:30
parent b95f1ee393
commit 4f689c4c92
4 changed files with 100 additions and 20 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

12
RMuseum/Controllers/GanjoorController.cs
View File

@ -4464,28 +4464,32 @@ namespace RMuseum.Controllers
/// <summary>
/// fill couplet summaries using open ai
/// </summary>
/// <param name="startFrom"></param>
/// <param name="count"></param>
/// <returns></returns>
[HttpPut("ai/generate/summaries")]
[Authorize(Policy = RMuseumSecurableItem.GanjoorEntityShortName + ":" + SecurableItem.ModifyOperationShortName)]
[ProducesResponseType((int)HttpStatusCode.OK)]
[ProducesResponseType((int)HttpStatusCode.BadRequest, Type = typeof(string))]
public IActionResult OpenAIStartFillingCoupletSummaries()
public IActionResult OpenAIStartFillingCoupletSummaries(int startFrom = 0, int count = 0)
{
_ganjoorService.OpenAIStartFillingCoupletSummaries();
_ganjoorService.OpenAIStartFillingCoupletSummaries(startFrom, count);
return Ok();
}
/// <summary>
/// fill poem summaries using open ai
/// </summary>
/// <param name="startFrom"></param>
/// <param name="count"></param>
/// <returns></returns>
[HttpPut("ai/generate/poem/summaries")]
[Authorize(Policy = RMuseumSecurableItem.GanjoorEntityShortName + ":" + SecurableItem.ModifyOperationShortName)]
[ProducesResponseType((int)HttpStatusCode.OK)]
[ProducesResponseType((int)HttpStatusCode.BadRequest, Type = typeof(string))]
public IActionResult OpenAIStartFillingPoemSummaries()
public IActionResult OpenAIStartFillingPoemSummaries(int startFrom = 0, int count = 0)
{
_ganjoorService.OpenAIStartFillingPoemSummaries();
_ganjoorService.OpenAIStartFillingPoemSummaries(startFrom, count);
return Ok();
}

24
RMuseum/RMuseum.xml
View File

@ -2289,16 +2289,20 @@
<param name="catId"></param>
<returns></returns>
</member>
<member name="M:RMuseum.Controllers.GanjoorController.OpenAIStartFillingCoupletSummaries">
<member name="M:RMuseum.Controllers.GanjoorController.OpenAIStartFillingCoupletSummaries(System.Int32,System.Int32)">
<summary>
fill couplet summaries using open ai
</summary>
<param name="startFrom"></param>
<param name="count"></param>
<returns></returns>
</member>
<member name="M:RMuseum.Controllers.GanjoorController.OpenAIStartFillingPoemSummaries">
<member name="M:RMuseum.Controllers.GanjoorController.OpenAIStartFillingPoemSummaries(System.Int32,System.Int32)">
<summary>
fill poem summaries using open ai
</summary>
<param name="startFrom"></param>
<param name="count"></param>
<returns></returns>
</member>
<member name="P:RMuseum.Controllers.GanjoorController.ReadOnlyMode">
@ -16985,15 +16989,19 @@
<param name="catId"></param>
<returns></returns>
</member>
<member name="M:RMuseum.Services.IGanjoorService.OpenAIStartFillingCoupletSummaries">
<member name="M:RMuseum.Services.IGanjoorService.OpenAIStartFillingCoupletSummaries(System.Int32,System.Int32)">
<summary>
fill couplet summaries using open ai
</summary>
<param name="startFrom"></param>
<param name="count"></param>
</member>
<member name="M:RMuseum.Services.IGanjoorService.OpenAIStartFillingPoemSummaries">
<member name="M:RMuseum.Services.IGanjoorService.OpenAIStartFillingPoemSummaries(System.Int32,System.Int32)">
<summary>
fill poem summaries using open ai
</summary>
<param name="startFrom"></param>
<param name="count"></param>
</member>
<member name="T:RMuseum.Services.IGanjoorTranslationService">
<summary>
@ -18919,15 +18927,19 @@
<param name="naskbanPassword"></param>
<returns>number of synched items</returns>
</member>
<member name="M:RMuseum.Services.Implementation.GanjoorService.OpenAIStartFillingCoupletSummaries">
<member name="M:RMuseum.Services.Implementation.GanjoorService.OpenAIStartFillingCoupletSummaries(System.Int32,System.Int32)">
<summary>
fill couplet summaries using open ai
</summary>
<param name="startFrom"></param>
<param name="count"></param>
</member>
<member name="M:RMuseum.Services.Implementation.GanjoorService.OpenAIStartFillingPoemSummaries">
<member name="M:RMuseum.Services.Implementation.GanjoorService.OpenAIStartFillingPoemSummaries(System.Int32,System.Int32)">
<summary>
fill poem summaries using open ai
</summary>
<param name="startFrom"></param>
<param name="count"></param>
</member>
<member name="M:RMuseum.Services.Implementation.GanjoorService.GetCategoryPaperSourcesAsync(System.Int32)">
<summary>

8
RMuseum/Services/IGanjoorService.cs
View File

@ -1533,13 +1533,17 @@ namespace RMuseum.Services
/// <summary>
/// fill couplet summaries using open ai
/// </summary>
void OpenAIStartFillingCoupletSummaries();
/// <param name="startFrom"></param>
/// <param name="count"></param>
void OpenAIStartFillingCoupletSummaries(int startFrom, int count);
/// <summary>
/// fill poem summaries using open ai
/// </summary>
void OpenAIStartFillingPoemSummaries();
/// <param name="startFrom"></param>
/// <param name="count"></param>
void OpenAIStartFillingPoemSummaries(int startFrom, int count);
}
}

76
RMuseum/Services/Implementation/GanjoorService-Partials/GanjoorService-OpenAI-CoupletSummaries.cs
View File

@ -20,7 +20,9 @@ namespace RMuseum.Services.Implementation
/// <summary>
/// fill couplet summaries using open ai
/// </summary>
public void OpenAIStartFillingCoupletSummaries()
/// <param name="startFrom"></param>
/// <param name="count"></param>
public void OpenAIStartFillingCoupletSummaries(int startFrom, int count)
{
_backgroundTaskQueue.QueueBackgroundWorkItem
(
@ -29,7 +31,7 @@ namespace RMuseum.Services.Implementation
using (RMuseumDbContext context = new RMuseumDbContext(new DbContextOptions<RMuseumDbContext>())) //this is long running job, so _context might be already been freed/collected by GC
{
LongRunningJobProgressServiceEF jobProgressServiceEF = new LongRunningJobProgressServiceEF(context);
var job = (await jobProgressServiceEF.NewJob($"OpenAIFillCoupletSummaries", "Open AI initialization")).Result;
var job = (await jobProgressServiceEF.NewJob($"OpenAIFillCoupletSummaries - start: {startFrom} - count: {count}", "Open AI initialization")).Result;
try
{
@ -41,7 +43,10 @@ namespace RMuseum.Services.Implementation
await jobProgressServiceEF.UpdateJob(job.Id, 0, "Query data");
var verses = await context.GanjoorVerses.AsNoTracking()
var verses =
count == 0 ?
startFrom == 0 ?
await context.GanjoorVerses.AsNoTracking()
.Where(v =>
(
v.VersePosition == VersePosition.Right
@ -55,7 +60,42 @@ namespace RMuseum.Services.Implementation
&&
string.IsNullOrEmpty(v.CoupletSummary)
)
.ToListAsync();
.ToListAsync()
:
await context.GanjoorVerses.AsNoTracking()
.Where(v =>
(
v.VersePosition == VersePosition.Right
||
v.VersePosition == VersePosition.CenteredVerse1
||
v.VersePosition == VersePosition.Paragraph
)
&&
v.CoupletIndex != null
&&
string.IsNullOrEmpty(v.CoupletSummary)
)
.Skip(startFrom)
.ToListAsync()
:
await context.GanjoorVerses.AsNoTracking()
.Where(v =>
(
v.VersePosition == VersePosition.Right
||
v.VersePosition == VersePosition.CenteredVerse1
||
v.VersePosition == VersePosition.Paragraph
)
&&
v.CoupletIndex != null
&&
string.IsNullOrEmpty(v.CoupletSummary)
)
.Skip(startFrom).Take(count)
.ToListAsync()
;
await jobProgressServiceEF.UpdateJob(job.Id, 0, $"Verse count = {verses.Count}");
for ( var i = 0; i < verses.Count; i++ )
@ -128,7 +168,9 @@ namespace RMuseum.Services.Implementation
/// <summary>
/// fill poem summaries using open ai
/// </summary>
public void OpenAIStartFillingPoemSummaries()
/// <param name="startFrom"></param>
/// <param name="count"></param>
public void OpenAIStartFillingPoemSummaries(int startFrom, int count)
{
_backgroundTaskQueue.QueueBackgroundWorkItem
(
@ -137,7 +179,7 @@ namespace RMuseum.Services.Implementation
using (RMuseumDbContext context = new RMuseumDbContext(new DbContextOptions<RMuseumDbContext>())) //this is long running job, so _context might be already been freed/collected by GC
{
LongRunningJobProgressServiceEF jobProgressServiceEF = new LongRunningJobProgressServiceEF(context);
var job = (await jobProgressServiceEF.NewJob($"OpenAIStartFillingPoemSummaries", "Open AI initialization")).Result;
var job = (await jobProgressServiceEF.NewJob($"OpenAIStartFillingPoemSummaries - start: {startFrom} - count: {count}", "Open AI initialization")).Result;
try
{
@ -149,10 +191,28 @@ namespace RMuseum.Services.Implementation
await jobProgressServiceEF.UpdateJob(job.Id, 0, "Query data");
var poems = await context.GanjoorPoems.AsNoTracking()
var poems =
count == 0 ?
startFrom == 0 ?
await context.GanjoorPoems.AsNoTracking()
.Where(
p => string.IsNullOrEmpty(p.PoemSummary)
)
.ToListAsync()
:
await context.GanjoorPoems.AsNoTracking()
.Where(
p => string.IsNullOrEmpty(p.PoemSummary)
)
.Skip(startFrom)
.ToListAsync()
:
await context.GanjoorPoems.AsNoTracking()
.Where(
p => string.IsNullOrEmpty(p.PoemSummary)
)
.Skip(startFrom)
.Take(count)
.ToListAsync();
await jobProgressServiceEF.UpdateJob(job.Id, 0, $"Poems count = {poems.Count}");
@ -162,7 +222,7 @@ namespace RMuseum.Services.Implementation
if (!string.IsNullOrEmpty(poem.PlainText))
{
await jobProgressServiceEF.UpdateJob(job.Id, i, $"{i} از {poems.Count}");
await jobProgressServiceEF.UpdateJob(job.Id, i, $"{i} از {poems.Count} - {poem.FullTitle}");
string command = "به فارسی روان خلاصه کن:";
var completionResult = await openAiService.ChatCompletion.CreateCompletion(new ChatCompletionCreateRequest
{