< Summary

Information
Class: GistBackend.Services.GistService
Assembly: GistBackend
File(s): /home/runner/work/the-gist-of-it-sec/the-gist-of-it-sec/backend/GistBackend/Services/GistService.cs
Line coverage
78%
Covered lines: 96
Uncovered lines: 27
Coverable lines: 123
Total lines: 227
Line coverage: 78%
Branch coverage
63%
Covered branches: 33
Total branches: 52
Branch coverage: 63.4%
Method coverage

Feature is only available for sponsors

Upgrade to PRO version

Metrics

MethodBranch coverage Crap Score Cyclomatic complexity Line coverage
.ctor(...)100%11100%
.cctor()100%11100%
ExecuteAsync()75%4483.33%
ProcessFeedsAsync()100%22100%
ProcessFeedAsync()0%2260%
EnsureCurrentFeedInfoInDbAsync()100%44100%
ProcessEntriesAsync()100%22100%
ProcessEntryAsync()58.33%412469.04%
GenerateSummaryAIResponse()100%11100%
InsertDataIntoDatabaseAsync()50%7666.66%
UpdateDataInDatabaseAsync()50%7671.42%
CreateSummary(...)100%44100%

File(s)

/home/runner/work/the-gist-of-it-sec/the-gist-of-it-sec/backend/GistBackend/Services/GistService.cs

#LineLine coverage
 1using System.Diagnostics;
 2using GistBackend.Exceptions;
 3using GistBackend.Handlers.AIHandler;
 4using GistBackend.Handlers.ChromaDbHandler;
 5using GistBackend.Handlers.MariaDbHandler;
 6using GistBackend.Handlers.RssFeedHandler;
 7using GistBackend.Handlers.WebCrawlHandler;
 8using GistBackend.Types;
 9using GistBackend.Utils;
 10using Microsoft.Extensions.Hosting;
 11using Microsoft.Extensions.Logging;
 12using Prometheus;
 13using static GistBackend.Utils.LogEvents;
 14using static GistBackend.Utils.ServiceUtils;
 15using PrometheusSummary = Prometheus.Summary;
 16using Summary = GistBackend.Types.Summary;
 17
 18namespace GistBackend.Services;
 19
 820public class GistService(
 821    IRssFeedHandler rssFeedHandler,
 822    IWebCrawlHandler webCrawlHandler,
 823    IMariaDbHandler mariaDbHandler,
 824    IAIHandler aiHandler,
 825    IChromaDbHandler chromaDbHandler,
 826    ILogger<GistService>? logger = null
 827) : BackgroundService
 28{
 129    private static readonly Gauge ProcessFeedsGauge =
 130        Metrics.CreateGauge("process_feeds_seconds", "Time spent to process all feeds");
 131    private static readonly PrometheusSummary ProcessEntrySummary =
 132        Metrics.CreateSummary("process_entry_seconds", "Time spent to process an entry", "feed_title");
 133    private static readonly PrometheusSummary SummarizeEntrySummary =
 134        Metrics.CreateSummary("summarize_entry_seconds", "Time spent to summarize an entry");
 35
 836    private readonly Dictionary<int, RssFeed> _feedsByFeedId = new();
 37
 38    protected override async Task ExecuteAsync(CancellationToken ct)
 39    {
 840        while (!ct.IsCancellationRequested)
 41        {
 842            var startTime = DateTime.UtcNow;
 1643            using (new SelfReportingStopwatch(elapsed => ProcessFeedsGauge.Set(elapsed)))
 44            {
 845                await ProcessFeedsAsync(ct);
 4846                await Task.WhenAll(_feedsByFeedId.Values.Select(feed => ProcessEntriesAsync(feed.Entries!.OrderBy(e => e
 847            }
 848            await DelayUntilNextExecutionAsync(startTime, 5, logger, ct);
 49        }
 050    }
 51
 52    private async Task ProcessFeedsAsync(CancellationToken ct)
 53    {
 5554        foreach (var feed in rssFeedHandler.Definitions) await ProcessFeedAsync(feed, ct);
 855    }
 56
 57    private async Task ProcessFeedAsync(RssFeed feed, CancellationToken ct)
 58    {
 59        try
 60        {
 1361            await rssFeedHandler.ParseFeedAsync(feed, ct);
 1362            var feedId = await EnsureCurrentFeedInfoInDbAsync(feed, ct);
 1363            feed.ParseEntries(feedId);
 1364            _feedsByFeedId[feedId] = feed;
 1365        }
 066        catch (ParsingFeedException e)
 67        {
 068            logger?.LogWarning(ParsingFeedFailed, e, "Skipping feed, failed to parse RSS feed from {RssUrl}",
 069                feed.RssUrl);
 070        }
 1371    }
 72
 73    private async Task<int> EnsureCurrentFeedInfoInDbAsync(RssFeed feed, CancellationToken ct)
 74    {
 1375        var existingFeedInfo = await mariaDbHandler.GetFeedInfoByRssUrlAsync(feed.RssUrl, ct);
 1376        var parsedFeedInfo = feed.ToRssFeedInfo();
 77
 1378        if (existingFeedInfo is null)
 79        {
 1280            var feedId = await mariaDbHandler.InsertFeedInfoAsync(parsedFeedInfo, ct);
 1281            return feedId;
 82        }
 183        if (parsedFeedInfo with { Id = existingFeedInfo.Id } != existingFeedInfo)
 84        {
 185            await mariaDbHandler.UpdateFeedInfoAsync(parsedFeedInfo, ct);
 86        }
 187        return existingFeedInfo.Id!.Value;
 1388    }
 89
 90    private async Task ProcessEntriesAsync(IEnumerable<RssEntry> entries, CancellationToken ct)
 91    {
 11292        foreach (var entry in entries) await ProcessEntryAsync(entry, ct);
 893    }
 94
 95    private async Task ProcessEntryAsync(RssEntry entry, CancellationToken ct)
 96    {
 3297        var stopwatch = Stopwatch.StartNew();
 3298        using var loggingScope = logger?.BeginScope("Processing entry with reference {Reference}", entry.Reference);
 99
 32100        var existingGist = await mariaDbHandler.GetGistByReferenceAsync(entry.Reference, ct);
 32101        var currentVersionAlreadyExists = existingGist is not null && existingGist.Updated == entry.Updated;
 32102        if (currentVersionAlreadyExists) return;
 103
 32104        var feed = _feedsByFeedId[entry.FeedId];
 105
 106        try
 107        {
 32108            var fetchResponse = await webCrawlHandler.FetchAsync(entry.Url.AbsoluteUri, ct);
 32109            if (fetchResponse.Status != 200)
 110            {
 0111                logger?.LogWarning(FetchingPageContentFailed,
 0112                    "Skipping entry, fetched page content returned status {Status} for {Url}", fetchResponse.Status,
 0113                    entry.Url.AbsoluteUri);
 0114                return;
 115            }
 116
 32117            if (feed.CheckForPaywall(fetchResponse.Content))
 118            {
 2119                var disabledGist = new DisabledGist(entry);
 3120                if (existingGist is not null) await mariaDbHandler.UpdateDisabledGistAsync(disabledGist, ct);
 1121                else await mariaDbHandler.InsertDisabledGistAsync(disabledGist, ct);
 2122                logger?.LogInformation(PaywallDetected,
 2123                    "Disabled gist for entry with reference {Reference} due to detected paywall",
 2124                    entry.Reference);
 2125                return;
 126            }
 127
 30128            var entryText = feed.ExtractText(fetchResponse.Content);
 30129            var summaryAIResponse = await GenerateSummaryAIResponse(feed.Language, entry.Title, entryText, ct);
 30130            var isSponsoredContent = feed.CheckForSponsoredContent(fetchResponse.Content);
 131
 30132            var gist = new Gist(entry, summaryAIResponse, isSponsoredContent);
 133
 30134            await chromaDbHandler.UpsertEntryAsync(entry, summaryAIResponse.SummaryEnglish, ct);
 135
 30136            if (existingGist is null)
 137            {
 25138                await InsertDataIntoDatabaseAsync(entry, gist, summaryAIResponse, feed.Language, ct);
 139            }
 140            else
 141            {
 5142                await UpdateDataInDatabaseAsync(entry, existingGist.Id!.Value, gist, summaryAIResponse, feed.Language,
 5143                    ct);
 144            }
 30145        }
 0146        catch (ExtractingEntryTextException e)
 147        {
 0148            logger?.LogWarning(ExtractingPageContentFailed, e,
 0149                "Skipping entry, failed to extract text from page content for {Url}",
 0150                entry.Url.AbsoluteUri);
 0151        }
 0152        catch (Exception e) when (e is ExternalServiceException or HttpRequestException)
 153        {
 0154            logger?.LogWarning(FetchingPageContentFailed, e, "Skipping entry, failed to fetch page content for {Url}",
 0155                entry.Url.AbsoluteUri);
 0156        }
 157        finally
 158        {
 32159            stopwatch.Stop();
 32160            ProcessEntrySummary.WithLabels(feed.Title!).Observe(stopwatch.Elapsed.Seconds);
 161        }
 32162    }
 163
 164    private async Task<SummaryAIResponse> GenerateSummaryAIResponse(Language feedLanguage, string entryTitle,
 165        string text, CancellationToken ct)
 166    {
 60167        using var stopwatch = new SelfReportingStopwatch(elapsed => SummarizeEntrySummary.Observe(elapsed));
 30168        return await aiHandler.GenerateSummaryAIResponseAsync(feedLanguage, entryTitle, text, ct);
 30169    }
 170
 171    private async Task InsertDataIntoDatabaseAsync(RssEntry entry, Gist gist, SummaryAIResponse summaryAIResponse,
 172        Language feedLanguage, CancellationToken ct)
 173    {
 25174        await using var handle = await mariaDbHandler.OpenTransactionAsync(ct);
 175        try
 176        {
 25177            var gistId = await mariaDbHandler.InsertGistAsync(gist, handle, ct);
 178
 25179            var germanSummary = CreateSummary(gistId, entry, summaryAIResponse, feedLanguage, Language.De);
 25180            await mariaDbHandler.InsertSummaryAsync(germanSummary, handle, ct);
 181
 25182            var englishSummary = CreateSummary(gistId, entry, summaryAIResponse, feedLanguage, Language.En);
 25183            await mariaDbHandler.InsertSummaryAsync(englishSummary, handle, ct);
 184
 25185            await handle.Transaction.CommitAsync(ct);
 25186            logger?.LogInformation(GistInserted, "Gist inserted with ID {Id}", gistId);
 25187        }
 0188        catch (Exception e)
 189        {
 0190            logger?.LogError(InsertingGistFailed, e, "Failed to insert gist with Reference {Reference}",
 0191                gist.Reference);
 0192            await handle.Transaction.RollbackAsync(ct);
 0193            throw;
 194        }
 25195    }
 196
 197    private async Task UpdateDataInDatabaseAsync(RssEntry entry, int gistId, Gist gist,
 198        SummaryAIResponse summaryAIResponse, Language feedLanguage, CancellationToken ct)
 199    {
 5200        await using var handle = await mariaDbHandler.OpenTransactionAsync(ct);
 201        try
 202        {
 5203            await mariaDbHandler.UpdateGistAsync(gist, handle, ct);
 204
 5205            var germanSummary = CreateSummary(gistId, entry, summaryAIResponse, feedLanguage, Language.De);
 5206            await mariaDbHandler.UpdateSummaryAsync(germanSummary, handle, ct);
 207
 5208            var englishSummary = CreateSummary(gistId, entry, summaryAIResponse, feedLanguage, Language.En);
 5209            await mariaDbHandler.UpdateSummaryAsync(englishSummary, handle, ct);
 210
 5211            await handle.Transaction.CommitAsync(ct);
 5212            logger?.LogInformation(GistUpdated, "Gist updated at ID {Id}", gistId);
 5213        }
 0214        catch (Exception e)
 215        {
 0216            logger?.LogError(UpdatingGistFailed, e, "Failed to update gist with ID {Id}", gistId);
 0217            await handle.Transaction.RollbackAsync(ct);
 0218            throw;
 219        }
 5220    }
 221
 222    private static Summary CreateSummary(int gistId, RssEntry entry, SummaryAIResponse summaryAIResponse,
 223        Language feedLanguage, Language summaryLanguage) =>
 60224        new(gistId, summaryLanguage, feedLanguage != summaryLanguage,
 60225            feedLanguage == summaryLanguage ? entry.Title : summaryAIResponse.TitleTranslated,
 60226            summaryLanguage == Language.En ? summaryAIResponse.SummaryEnglish : summaryAIResponse.SummaryGerman);
 227}
Generated by: ReportGenerator 5.5.4.0
04/02/2026 - 18:13:56
GitHub | reportgenerator.io

Methods/Properties

.ctor(GistBackend.Handlers.RssFeedHandler.IRssFeedHandler,GistBackend.Handlers.WebCrawlHandler.IWebCrawlHandler,GistBackend.Handlers.MariaDbHandler.IMariaDbHandler,GistBackend.Handlers.AIHandler.IAIHandler,GistBackend.Handlers.ChromaDbHandler.IChromaDbHandler,Microsoft.Extensions.Logging.ILogger`1<GistBackend.Services.GistService>)
.cctor()
ExecuteAsync()
ProcessFeedsAsync()
ProcessFeedAsync()
EnsureCurrentFeedInfoInDbAsync()
ProcessEntriesAsync()
ProcessEntryAsync()
GenerateSummaryAIResponse()
InsertDataIntoDatabaseAsync()
UpdateDataInDatabaseAsync()
CreateSummary(System.Int32,GistBackend.Types.RssEntry,GistBackend.Types.SummaryAIResponse,GistBackend.Types.Language,GistBackend.Types.Language)