#129 parsing donation page html and filling records

RMuseum/DbContext/RMuseumDbContext.cs

@@ -13,6 +13,7 @@ using System;
 using RMuseum.Models.Ganjoor;
 using RMuseum.Models.MusicCatalogue;
 using System.IO;
+using RMuseum.Models.Accounting;
 
 namespace RMuseum.DbContext
 {
@@ -277,6 +278,16 @@ namespace RMuseum.DbContext
         /// </summary>
         public DbSet<GanjoorHealthCheckError> GanjoorHealthCheckErrors { get; set; }
 
+        /// <summary>
+        /// donations
+        /// </summary>
+        public DbSet<GanjoorDonation> GanjoorDonations { get; set; }
+
+        /// <summary>
+        /// expenses
+        /// </summary>
+        public DbSet<GanjoorExpense> GanjoorExpenses { get; set; }
+
 
     }
 }

RMuseum/Models/Accounting/DonationExpenditure.cs

@@ -0,0 +1,23 @@
+namespace RMuseum.Models.Accounting
+{
+    /// <summary>
+    /// donation expenditures
+    /// </summary>
+    public class DonationExpenditure
+    {
+        /// <summary>
+        /// GanjoorDonation Id
+        /// </summary>
+        public int GanjoorDonationId { get; set; }
+
+        /// <summary>
+        /// GanjoorDonation
+        /// </summary>
+        public GanjoorDonation GanjoorDonation { get; set; }
+
+        /// <summary>
+        /// amount
+        /// </summary>
+        public decimal Amount { get; set; }
+    }
+}

RMuseum/Models/Accounting/GanjoorDonation.cs

@@ -48,7 +48,7 @@ namespace RMuseum.Models.Accounting
         public string DonorLink { get; set; }
 
         /// <summary>
-        /// remaining
+        /// remaining (it could be ignored later when our data has been normalized enough and be calculated using related data)
         /// </summary>
         public decimal Remaining { get; set; }
 

RMuseum/Models/Accounting/GanjoorExpense.cs

@@ -0,0 +1,41 @@
+using System;
+using System.Collections.Generic;
+
+namespace RMuseum.Models.Accounting
+{
+    /// <summary>
+    /// Ganjoor Expenses
+    /// </summary>
+    public class GanjoorExpense
+    {
+        /// <summary>
+        /// Id
+        /// </summary>
+        public int Id { get; set; }
+
+        /// <summary>
+        /// date
+        /// </summary>
+        public DateTime ExpenseDate { get; set; }
+
+        /// <summary>
+        /// amount
+        /// </summary>
+        public decimal Amount { get; set; }
+
+        /// <summary>
+        /// unit
+        /// </summary>
+        public string Unit { get; set; }
+
+        /// <summary>
+        /// description
+        /// </summary>
+        public string Description { get; set; }
+
+        /// <summary>
+        /// donation expenitures
+        /// </summary>
+        public ICollection<DonationExpenditure> DonationExpenditures { get; set; }
+    }
+}

RMuseum/RMuseum.xml

@@ -1484,6 +1484,36 @@
             Ganjoor Health Check Errors
             </summary>
         </member>
+        <member name="P:RMuseum.DbContext.RMuseumDbContext.GanjoorDonations">
+            <summary>
+            donations
+            </summary>
+        </member>
+        <member name="P:RMuseum.DbContext.RMuseumDbContext.GanjoorExpenses">
+            <summary>
+            expenses
+            </summary>
+        </member>
+        <member name="T:RMuseum.Models.Accounting.DonationExpenditure">
+            <summary>
+            donation expenditures
+            </summary>
+        </member>
+        <member name="P:RMuseum.Models.Accounting.DonationExpenditure.GanjoorDonationId">
+            <summary>
+            GanjoorDonation Id
+            </summary>
+        </member>
+        <member name="P:RMuseum.Models.Accounting.DonationExpenditure.GanjoorDonation">
+            <summary>
+            GanjoorDonation
+            </summary>
+        </member>
+        <member name="P:RMuseum.Models.Accounting.DonationExpenditure.Amount">
+            <summary>
+            amount
+            </summary>
+        </member>
         <member name="T:RMuseum.Models.Accounting.GanjoorDonation">
             <summary>
             Ganjoor Donation (based on donation records saved in old html format of https://ganjoor.net/donate/)
@@ -1496,7 +1526,7 @@
         </member>
         <member name="P:RMuseum.Models.Accounting.GanjoorDonation.DateString">
             <summary>
-            date: (donation dates have been collected in different format, so instead of keeping their dates in a DateTime field I ought to use a string field)
+            date: (donation dates have been collected in different formats, so instead of keeping their dates in a DateTime field I ought to use a string field)
             </summary>
         </member>
         <member name="P:RMuseum.Models.Accounting.GanjoorDonation.RecordDate">
@@ -1531,7 +1561,7 @@
         </member>
         <member name="P:RMuseum.Models.Accounting.GanjoorDonation.Remaining">
             <summary>
-            remaining
+            remaining (it could be ignored later when our data has been normalized enough and be calculated using related data)
             </summary>
         </member>
         <member name="P:RMuseum.Models.Accounting.GanjoorDonation.ExpenditureDesc">
@@ -1544,6 +1574,41 @@
             record is imported from old HTML text of donation page
             </summary>
         </member>
+        <member name="T:RMuseum.Models.Accounting.GanjoorExpense">
+            <summary>
+            Ganjoor Expenses
+            </summary>
+        </member>
+        <member name="P:RMuseum.Models.Accounting.GanjoorExpense.Id">
+            <summary>
+            Id
+            </summary>
+        </member>
+        <member name="P:RMuseum.Models.Accounting.GanjoorExpense.ExpenseDate">
+            <summary>
+            date
+            </summary>
+        </member>
+        <member name="P:RMuseum.Models.Accounting.GanjoorExpense.Amount">
+            <summary>
+            amount
+            </summary>
+        </member>
+        <member name="P:RMuseum.Models.Accounting.GanjoorExpense.Unit">
+            <summary>
+            unit
+            </summary>
+        </member>
+        <member name="P:RMuseum.Models.Accounting.GanjoorExpense.Description">
+            <summary>
+            description
+            </summary>
+        </member>
+        <member name="P:RMuseum.Models.Accounting.GanjoorExpense.DonationExpenditures">
+            <summary>
+            donation expenitures
+            </summary>
+        </member>
         <member name="T:RMuseum.Models.Artifact.PublishStatus">
             <summary>
             Items or pictures status
@@ -7721,6 +7786,58 @@
             <param name="format"></param>
             <returns></returns>
         </member>
+        <member name="T:RMuseum.Services.Implementation.DonationService">
+            <summary>
+            donation service
+            </summary>
+        </member>
+        <member name="T:RMuseum.Services.Implementation.DonationService.DonationPageRow">
+            <summary>
+            donation page row
+            </summary>
+        </member>
+        <member name="P:RMuseum.Services.Implementation.DonationService.DonationPageRow.Date">
+            <summary>
+            date
+            </summary>
+        </member>
+        <member name="P:RMuseum.Services.Implementation.DonationService.DonationPageRow.Amount">
+            <summary>
+            amount
+            </summary>
+        </member>
+        <member name="P:RMuseum.Services.Implementation.DonationService.DonationPageRow.Donor">
+            <summary>
+            donor
+            </summary>
+        </member>
+        <member name="P:RMuseum.Services.Implementation.DonationService.DonationPageRow.Usage">
+            <summary>
+            usage
+            </summary>
+        </member>
+        <member name="P:RMuseum.Services.Implementation.DonationService.DonationPageRow.Remaining">
+            <summary>
+            remaining
+            </summary>
+        </member>
+        <member name="M:RMuseum.Services.Implementation.DonationService.InitializeRecords">
+            <summary>
+            parse html of https://ganjoor.net/donate/ and fill the records
+            </summary>
+            <returns></returns>
+        </member>
+        <member name="F:RMuseum.Services.Implementation.DonationService._context">
+            <summary>
+            Database Context
+            </summary>
+        </member>
+        <member name="M:RMuseum.Services.Implementation.DonationService.#ctor(RMuseum.DbContext.RMuseumDbContext)">
+            <summary>
+            constructor
+            </summary>
+            <param name="context"></param>
+        </member>
         <member name="T:RMuseum.Services.Implementation.GanjoorAppUserService">
             <summary>
             Ganjoor User Service

RMuseum/Services/Implementation/DonationService.cs

@@ -0,0 +1,177 @@
+using Microsoft.EntityFrameworkCore;
+using RMuseum.DbContext;
+using RMuseum.Models.Accounting;
+using RSecurityBackend.Models.Generic;
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Threading.Tasks;
+using DNTPersianUtils.Core;
+
+namespace RMuseum.Services.Implementation
+{
+    /// <summary>
+    /// donation service
+    /// </summary>
+    public class DonationService
+    {
+
+        /// <summary>
+        /// donation page row
+        /// </summary>
+        private class DonationPageRow
+        {
+            /// <summary>
+            /// date
+            /// </summary>
+            public string Date { get; set; }
+
+            /// <summary>
+            /// amount
+            /// </summary>
+            public string Amount { get; set; }
+
+            /// <summary>
+            /// donor
+            /// </summary>
+            public string Donor { get; set; }
+
+            /// <summary>
+            /// usage
+            /// </summary>
+            public string Usage { get; set; }
+
+            /// <summary>
+            /// remaining
+            /// </summary>
+            public string Remaining { get; set; }
+        }
+
+        /// <summary>
+        /// parse html of https://ganjoor.net/donate/ and fill the records
+        /// </summary>
+        /// <returns></returns>
+        public async Task<RServiceResult<bool>> InitializeRecords()
+        {
+            if (await _context.GanjoorDonations.AnyAsync())
+                return new RServiceResult<bool>(true);
+
+            string htmlText = await _context.GanjoorPages.Where(p => p.UrlSlug == "donate").Select(p => p.HtmlText).AsNoTracking().SingleAsync();
+
+            List<DonationPageRow> rows = new List<DonationPageRow>();
+
+            int nStartIndex = htmlText.IndexOf("<td class=\"ddate\">");
+            int rowNumber = 0;
+            while(nStartIndex != -1)
+            {
+                rowNumber++;
+                DonationPageRow row = new DonationPageRow();
+
+                nStartIndex += "<td class=\"ddate\">".Length;
+                row.Date = htmlText.Substring(nStartIndex, htmlText.IndexOf("</td>", nStartIndex));
+
+                nStartIndex = htmlText.IndexOf("<td class=\"damount\">", nStartIndex);
+                if (nStartIndex == -1)
+                    return new RServiceResult<bool>(false, $"{rowNumber} : damount");
+                nStartIndex += "<td class=\"damount\">".Length;
+                row.Amount = htmlText.Substring(nStartIndex, htmlText.IndexOf("</td>", nStartIndex));
+
+                nStartIndex = htmlText.IndexOf("<td class=\"ddonator\">", nStartIndex);
+                if (nStartIndex == -1)
+                    return new RServiceResult<bool>(false, $"{rowNumber} : ddonator");
+                nStartIndex += "<td class=\"ddonator\">".Length;
+                row.Donor = htmlText.Substring(nStartIndex, htmlText.IndexOf("</td>", nStartIndex));
+
+                nStartIndex = htmlText.IndexOf("<td class=\"dusage\">", nStartIndex);
+                if (nStartIndex == -1)
+                    return new RServiceResult<bool>(false, $"{rowNumber} : dusage");
+                nStartIndex += "<td class=\"dusage\">".Length;
+                row.Usage = htmlText.Substring(nStartIndex, htmlText.IndexOf("</td>", nStartIndex));
+
+                nStartIndex = htmlText.IndexOf("<td class=\"drem\">", nStartIndex);
+                if (nStartIndex == -1)
+                    return new RServiceResult<bool>(false, $"{rowNumber} : drem");
+                nStartIndex += "<td class=\"drem\">".Length;
+                row.Remaining = htmlText.Substring(nStartIndex, htmlText.IndexOf("</td>", nStartIndex));
+
+
+                rows.Add(row);
+
+                nStartIndex = htmlText.IndexOf("<td class=\"ddate\">", nStartIndex);
+            }
+
+            DateTime recordDate = DateTime.Now.AddDays(-2);
+
+            for(int i = rows.Count - 1; i>=0; i--)
+            {
+                GanjoorDonation donation = new GanjoorDonation()
+                {
+                    ImportedRecord = true,
+                    DateString = rows[i].Date,
+                    RecordDate = recordDate,
+                    AmountString = rows[i].Amount,
+                    DonorName = rows[i].Donor, //needs to be cleaned from href values
+                    ExpenditureDesc = rows[i].Usage,
+                    Remaining = 0
+                };
+
+                if(
+                    rows[i].Remaining != "۲۰ دلار" //one record
+                    &&
+                    rows[i].Remaining.ToEnglishNumbers() != "0"
+                    )
+                {
+                    donation.Remaining = decimal.Parse(rows[i].Remaining.Replace("٬", "").Replace("تومان", "").Trim());
+                }
+
+                if(donation.DonorName.IndexOf("href") != -1)
+                {
+                    nStartIndex = donation.DonorName.IndexOf("href") + "href".Length + 1;
+                    donation.DonorLink = donation.DonorName.Substring(nStartIndex, donation.DonorName.IndexOf("</a>") - nStartIndex);
+                    donation.DonorName = donation.DonorName.Substring(0, donation.DonorName.IndexOf("<a")) + donation.DonorName.Replace("</a>", "").Substring(donation.DonorName.IndexOf(">") + 1);
+                }
+
+                if(donation.AmountString.Contains("تومان"))
+                {
+                    donation.Amount = decimal.Parse(donation.AmountString.Replace("٬", "").Replace("تومان", "").Trim());
+                    donation.Unit = "تومان";
+                }
+
+                if (donation.AmountString.Contains("دلار"))
+                {
+                    donation.Amount = decimal.Parse(donation.AmountString.Replace("٬", "").Replace("دلار", "").Trim());
+                    donation.Unit = "دلار";
+                }
+
+                if(donation.ExpenditureDesc == "هنوز هزینه نشده.")
+                {
+                    donation.ExpenditureDesc = "";
+                }
+
+                _context.GanjoorDonations.Add(donation);
+
+                await _context.SaveChangesAsync(); //in order to make Id columns filled in desired order
+
+
+            }
+
+            return new RServiceResult<bool>(true);
+        }
+
+
+        /// <summary>
+        /// Database Context
+        /// </summary>
+        private readonly RMuseumDbContext _context;
+
+        /// <summary>
+        /// constructor
+        /// </summary>
+        /// <param name="context"></param>
+        public DonationService(RMuseumDbContext context)
+        {
+            _context = context;
+            
+        }
+    }
+}