diff --git a/RMuseum/RMuseum.xml b/RMuseum/RMuseum.xml index 086f9b0a..93d37169 100644 --- a/RMuseum/RMuseum.xml +++ b/RMuseum/RMuseum.xml @@ -21211,6 +21211,24 @@ current poem count. + + + Guards against two runs of the same export job (keyed by job name — "PublicDataExport" + / "TajikPublicDataExport") ever executing concurrently against the same git working + copy. This exists because deleting a job's row on the admin Jobs page only removes its + database record — it does not stop the background Task (or its child git.exe process) + that's still actually running. Without this guard, triggering the job again while a + previous run hadn't finished starts a second concurrent git process against the same + folder, which is exactly what produces a ".git/index.lock: File exists" failure. + + + + + Returns true (and marks as running) if no run of this job + was already in progress; false if one was, in which case the caller should refuse to + start a second one rather than queueing a background task that would race the first. + + start exporting all Ganjoor data (poets/categories/poems/verses) belonging to @@ -24160,6 +24178,14 @@ any reason (e.g. a Windows service running under a service account with its own PATH). + + + how long a single git command is allowed to run before it's killed and the job fails + with a clear timeout error, instead of hanging forever (e.g. if git ever ends up + waiting on an interactive prompt with no terminal to answer it — see + GIT_TERMINAL_PROMPT in GitRepoPublisher) + + Thin wrapper around the native git CLI (not LibGit2Sharp) for the export job: sync a @@ -24187,6 +24213,21 @@ be hand-editing it, so a hard reset is safe and keeps the job idempotent). + + + A .git/index.lock left behind by an abruptly-terminated git process (e.g. an app pool + recycle, or a previous run of this same job that got killed mid-command rather than + finishing) blocks every future git command in this working copy with a confusing + "Another git process seems to be running" error, even when nothing actually is. + GanjoorService's TryStartExclusiveExportJob already guarantees only one run of a given + export job (main or Tajik) executes at a time *within this process* — so reaching this + method at all means the current call is the sole legitimate owner of this working copy + right now, and any lock file found here can only be a leftover from something that is + no longer running (that in-process guard doesn't survive an app pool/process restart, + which is exactly the scenario that leaves an orphaned lock file in the first place). + Safe to remove unconditionally on that basis. + + Stages every change under the working copy and, if anything actually changed, commits diff --git a/RMuseum/Services/Implementation/GanjoorService-Partials/GanjoorService-PublicDataExport.cs b/RMuseum/Services/Implementation/GanjoorService-Partials/GanjoorService-PublicDataExport.cs index f70e518f..43c2d793 100644 --- a/RMuseum/Services/Implementation/GanjoorService-Partials/GanjoorService-PublicDataExport.cs +++ b/RMuseum/Services/Implementation/GanjoorService-Partials/GanjoorService-PublicDataExport.cs @@ -211,6 +211,7 @@ namespace RMuseum.Services.Implementation GitUserName = section["GitUserName"], GitToken = section["GitToken"], GitExecutablePath = section["GitExecutablePath"], + CommandTimeoutMinutes = int.TryParse(section["CommandTimeoutMinutes"], out var timeout) ? timeout : 45, }; } @@ -488,6 +489,9 @@ Not available as a static endpoint in this data set yet. return $@"# ganjoor-data +**[▶ Live demo](https://ganjoor.github.io/mini/)** — مین‌گنجور, a minimal reading app built +entirely on this data, running client-side in your browser with no server of its own. + Public, git-tracked export of [Ganjoor](https://ganjoor.net)'s poetry content — poets, categories, and poems, allowlisted to contain none of the site's user-account-linked data (comments, bookmarks, edit history, etc.). Generated from diff --git a/RMuseum/Services/Implementation/GanjoorService-Partials/GanjoorService-TajikPublicDataExport.cs b/RMuseum/Services/Implementation/GanjoorService-Partials/GanjoorService-TajikPublicDataExport.cs index a9fc10b1..04f21a5e 100644 --- a/RMuseum/Services/Implementation/GanjoorService-Partials/GanjoorService-TajikPublicDataExport.cs +++ b/RMuseum/Services/Implementation/GanjoorService-Partials/GanjoorService-TajikPublicDataExport.cs @@ -159,6 +159,7 @@ namespace RMuseum.Services.Implementation GitUserName = section["GitUserName"], GitToken = section["GitToken"], GitExecutablePath = section["GitExecutablePath"], + CommandTimeoutMinutes = int.TryParse(section["CommandTimeoutMinutes"], out var timeout) ? timeout : 45, }; } diff --git a/RMuseum/Utils/PublicDataExport/GitRepoPublisher.cs b/RMuseum/Utils/PublicDataExport/GitRepoPublisher.cs index 7bcb55e7..4dcfa9ee 100644 --- a/RMuseum/Utils/PublicDataExport/GitRepoPublisher.cs +++ b/RMuseum/Utils/PublicDataExport/GitRepoPublisher.cs @@ -54,6 +54,14 @@ namespace RMuseum.Utils.PublicDataExport /// any reason (e.g. a Windows service running under a service account with its own PATH). /// public string GitExecutablePath { get; set; } + + /// + /// how long a single git command is allowed to run before it's killed and the job fails + /// with a clear timeout error, instead of hanging forever (e.g. if git ever ends up + /// waiting on an interactive prompt with no terminal to answer it — see + /// GIT_TERMINAL_PROMPT in GitRepoPublisher) + /// + public int CommandTimeoutMinutes { get; set; } = 45; } /// @@ -209,12 +217,34 @@ namespace RMuseum.Utils.PublicDataExport WorkingDirectory = workingDirectory, RedirectStandardOutput = true, RedirectStandardError = true, + RedirectStandardInput = true, UseShellExecute = false, CreateNoWindow = true, }; - using (var process = new Process { StartInfo = psi }) + // Without this, git can end up trying to prompt interactively for credentials or a + // host-key confirmation. There is no interactive terminal in this process (it runs + // under IIS/a background job) — an unanswered prompt just hangs forever with no error + // and no timeout, which is exactly what happened here. This makes git fail fast with + // a real error instead. + psi.EnvironmentVariables["GIT_TERMINAL_PROMPT"] = "0"; + + var stdout = new StringBuilder(); + var stderr = new StringBuilder(); + + using (var process = new Process { StartInfo = psi, EnableRaisingEvents = true }) { + // Reading both redirected streams asynchronously (rather than e.g. + // StandardOutput.ReadToEnd() before WaitForExit()) is required here, not optional: + // git writes most of its push/fetch progress to stderr, and synchronously draining + // stdout first while nobody drains stderr is a classic .NET Process deadlock if + // stderr's OS pipe buffer fills up — the child blocks writing to stderr, the parent + // blocks reading stdout, and both wait on each other forever. That deadlock (not a + // slow network) is almost certainly what actually produced the multi-day hang this + // was fixed after. + process.OutputDataReceived += (s, e) => { if (e.Data != null) stdout.AppendLine(e.Data); }; + process.ErrorDataReceived += (s, e) => { if (e.Data != null) stderr.AppendLine(e.Data); }; + try { process.Start(); @@ -230,11 +260,37 @@ namespace RMuseum.Utils.PublicDataExport "the full path of git.exe.", exp); } - string stdout = process.StandardOutput.ReadToEnd(); - string stderr = process.StandardError.ReadToEnd(); + process.StandardInput.Close(); // nothing will ever be typed to it + process.BeginOutputReadLine(); + process.BeginErrorReadLine(); + + bool exited = process.WaitForExit(_options.CommandTimeoutMinutes * 60 * 1000); + if (!exited) + { + TryKill(process); + throw new TimeoutException( + $"git {RedactAuth(arguments)} did not finish within {_options.CommandTimeoutMinutes} minutes and was killed. " + + "This is a hang, not normal slowness for this job — most likely git was waiting on a prompt " + + "with no terminal to answer it (should no longer happen with GIT_TERMINAL_PROMPT=0 set above) " + + "or a genuine network stall."); + } + + // let the async read handlers finish flushing after the process has exited process.WaitForExit(); - return new GitProcessResult { ExitCode = process.ExitCode, StandardOutput = stdout, StandardError = stderr }; + return new GitProcessResult { ExitCode = process.ExitCode, StandardOutput = stdout.ToString(), StandardError = stderr.ToString() }; + } + } + + private static void TryKill(Process process) + { + try + { + process.Kill(entireProcessTree: true); + } + catch + { + // best-effort - if it's already gone, or can't be killed, there's nothing more to do } } diff --git a/RMuseum/appsettings.json b/RMuseum/appsettings.json index 43f5bfb9..40fe92dc 100644 --- a/RMuseum/appsettings.json +++ b/RMuseum/appsettings.json @@ -76,7 +76,8 @@ "PushEnabled": "False", "GitUserName": "", "GitToken": "", - "GitExecutablePath": "" + "GitExecutablePath": "", + "CommandTimeoutMinutes": "45" }, "TajikPublicDataExport": { "LocalWorkingCopyPath": "C:\\ganjoor-tajik-public-data", @@ -87,7 +88,8 @@ "PushEnabled": "False", "GitUserName": "", "GitToken": "", - "GitExecutablePath": "" + "GitExecutablePath": "", + "CommandTimeoutMinutes": "45" }, "ExternalFTPServer": { "Host": "localhost",