From b4cd3d1b33afb74d092c519d9c2d673f944dade4 Mon Sep 17 00:00:00 2001 From: PrashantUnity Date: Sat, 27 Jun 2026 16:38:37 +0530 Subject: [PATCH 1/8] Now No online commit --- docker-compose.prod.yml | 1 + docker-compose.yml | 1 + scripts/local-run-common.sh | 1 + .../.idea.WebsiteProfiling/.idea/.gitignore | 15 + .../.idea/.idea.WebsiteProfiling/.idea/.name | 1 + .../.idea/encodings.xml | 4 + .../.idea/indexLayout.xml | 10 + .../.idea.WebsiteProfiling/.idea/vcs.xml | 7 + .../Build/CategoryHelpers.cs | 28 + .../Build/KeywordOpportunitiesBuilder.cs | 420 +++++++++++++ .../Build/NativeReportBuilder.cs | 75 ++- .../Build/NativeReportPayloadAssembler.cs | 30 +- .../Build/OptionalAuditUrlsBuilder.cs | 54 ++ .../Build/OptionalAuditsBuilder.cs | 554 ++++++++++++++++++ .../Build/SpellCheckerFactory.cs | 57 ++ .../Build/TextHygieneHelper.cs | 40 +- .../DependencyInjection.cs | 10 + .../Integrations/AiServiceEnrichmentClient.cs | 89 +++ .../Options/ReportServiceOptions.cs | 2 + .../ReportService.Application.csproj | 1 + .../Repositories/CrawlPageHtmlReader.cs | 45 ++ .../KeywordOpportunitiesBuilderTests.cs | 84 +++ .../NativeReportPayloadAssemblerTests.cs | 70 +++ .../OptionalAuditUrlsBuilderTests.cs | 42 ++ .../OptionalAuditsBuilderTests.cs | 125 ++++ 25 files changed, 1756 insertions(+), 10 deletions(-) create mode 100644 services/.idea/.idea.WebsiteProfiling/.idea/.gitignore create mode 100644 services/.idea/.idea.WebsiteProfiling/.idea/.name create mode 100644 services/.idea/.idea.WebsiteProfiling/.idea/encodings.xml create mode 100644 services/.idea/.idea.WebsiteProfiling/.idea/indexLayout.xml create mode 100644 services/.idea/.idea.WebsiteProfiling/.idea/vcs.xml create mode 100644 services/ReportService/src/ReportService.Application/Build/KeywordOpportunitiesBuilder.cs create mode 100644 services/ReportService/src/ReportService.Application/Build/OptionalAuditUrlsBuilder.cs create mode 100644 services/ReportService/src/ReportService.Application/Build/OptionalAuditsBuilder.cs create mode 100644 services/ReportService/src/ReportService.Application/Build/SpellCheckerFactory.cs create mode 100644 services/ReportService/src/ReportService.Application/Integrations/AiServiceEnrichmentClient.cs create mode 100644 services/ReportService/src/ReportService.Application/Repositories/CrawlPageHtmlReader.cs create mode 100644 services/ReportService/tests/ReportService.Tests/KeywordOpportunitiesBuilderTests.cs create mode 100644 services/ReportService/tests/ReportService.Tests/OptionalAuditUrlsBuilderTests.cs create mode 100644 services/ReportService/tests/ReportService.Tests/OptionalAuditsBuilderTests.cs diff --git a/docker-compose.prod.yml b/docker-compose.prod.yml index f03835d..b8c7663 100644 --- a/docker-compose.prod.yml +++ b/docker-compose.prod.yml @@ -122,6 +122,7 @@ services: ASPNETCORE_URLS: http://+:8094 FASTAPI_URL: http://fastapi:8001 INTEGRATIONS_SERVICE_URL: http://integrations:8093 + AISERVICE_URL: http://ai:8092 REPORT_SERVICE_USE_PYTHON_BRIDGE: "0" USE_FASTAPI_PYTHON_BRIDGE: "1" REPORT_SERVICE_WORKER_ENABLED: "1" diff --git a/docker-compose.yml b/docker-compose.yml index fd54630..4cba43e 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -124,6 +124,7 @@ services: ASPNETCORE_URLS: http://+:8094 FASTAPI_URL: http://fastapi:8001 INTEGRATIONS_SERVICE_URL: http://integrations:8093 + AISERVICE_URL: http://ai:8092 REPORT_SERVICE_USE_PYTHON_BRIDGE: "0" USE_FASTAPI_PYTHON_BRIDGE: "1" REPORT_SERVICE_WORKER_ENABLED: "1" diff --git a/scripts/local-run-common.sh b/scripts/local-run-common.sh index f447441..c2d8abb 100644 --- a/scripts/local-run-common.sh +++ b/scripts/local-run-common.sh @@ -92,6 +92,7 @@ start_host_report_service() { DATABASE_URL="$DATABASE_URL" \ FASTAPI_URL="http://127.0.0.1:8001" \ INTEGRATIONS_SERVICE_URL="$INTEGRATIONS_SERVICE_URL" \ + AISERVICE_URL="${AISERVICE_URL:-${AI_SERVICE_URL:-http://127.0.0.1:8092}}" \ WEBSITE_PROFILING_ROOT="$root" \ DATA_DIR="${DATA_DIR:-$root/data}" \ PYTHON="${PYTHON:-$root/.venv/bin/python}" \ diff --git a/services/.idea/.idea.WebsiteProfiling/.idea/.gitignore b/services/.idea/.idea.WebsiteProfiling/.idea/.gitignore new file mode 100644 index 0000000..b115e5d --- /dev/null +++ b/services/.idea/.idea.WebsiteProfiling/.idea/.gitignore @@ -0,0 +1,15 @@ +# Default ignored files +/shelf/ +/workspace.xml +# Rider ignored files +/projectSettingsUpdater.xml +/modules.xml +/contentModel.xml +/.idea.WebsiteProfiling.iml +# Editor-based HTTP Client requests +/httpRequests/ +# Ignored default folder with query files +/queries/ +# Datasource local storage ignored files +/dataSources/ +/dataSources.local.xml diff --git a/services/.idea/.idea.WebsiteProfiling/.idea/.name b/services/.idea/.idea.WebsiteProfiling/.idea/.name new file mode 100644 index 0000000..f444313 --- /dev/null +++ b/services/.idea/.idea.WebsiteProfiling/.idea/.name @@ -0,0 +1 @@ +WebsiteProfiling \ No newline at end of file diff --git a/services/.idea/.idea.WebsiteProfiling/.idea/encodings.xml b/services/.idea/.idea.WebsiteProfiling/.idea/encodings.xml new file mode 100644 index 0000000..df87cf9 --- /dev/null +++ b/services/.idea/.idea.WebsiteProfiling/.idea/encodings.xml @@ -0,0 +1,4 @@ + + + + \ No newline at end of file diff --git a/services/.idea/.idea.WebsiteProfiling/.idea/indexLayout.xml b/services/.idea/.idea.WebsiteProfiling/.idea/indexLayout.xml new file mode 100644 index 0000000..1b9cdc9 --- /dev/null +++ b/services/.idea/.idea.WebsiteProfiling/.idea/indexLayout.xml @@ -0,0 +1,10 @@ + + + + + ../../WebsiteProfiling + + + + + \ No newline at end of file diff --git a/services/.idea/.idea.WebsiteProfiling/.idea/vcs.xml b/services/.idea/.idea.WebsiteProfiling/.idea/vcs.xml new file mode 100644 index 0000000..62bd7a0 --- /dev/null +++ b/services/.idea/.idea.WebsiteProfiling/.idea/vcs.xml @@ -0,0 +1,7 @@ + + + + + + + \ No newline at end of file diff --git a/services/ReportService/src/ReportService.Application/Build/CategoryHelpers.cs b/services/ReportService/src/ReportService.Application/Build/CategoryHelpers.cs index f0aa13e..9105157 100644 --- a/services/ReportService/src/ReportService.Application/Build/CategoryHelpers.cs +++ b/services/ReportService/src/ReportService.Application/Build/CategoryHelpers.cs @@ -249,6 +249,34 @@ public static List IndexationCoverageIssues( return issues; } + public static void MergeIssuesIntoCategory( + IList categories, + string categoryId, + IEnumerable extra) + { + var extraList = extra.ToList(); + if (extraList.Count == 0) + { + return; + } + + for (var i = 0; i < categories.Count; i++) + { + if (!string.Equals(categories[i].Id, categoryId, StringComparison.Ordinal)) + { + continue; + } + + var merged = SortIssues(categories[i].Issues.Concat(extraList)); + categories[i] = categories[i] with + { + Issues = merged, + Recommendations = RecommendationsFromIssues(merged), + }; + break; + } + } + public static void MergeIndexationIssues( IList categories, IReadOnlyList rows, diff --git a/services/ReportService/src/ReportService.Application/Build/KeywordOpportunitiesBuilder.cs b/services/ReportService/src/ReportService.Application/Build/KeywordOpportunitiesBuilder.cs new file mode 100644 index 0000000..a250d7c --- /dev/null +++ b/services/ReportService/src/ReportService.Application/Build/KeywordOpportunitiesBuilder.cs @@ -0,0 +1,420 @@ +using System.Text.Json; +using System.Text.RegularExpressions; +using ReportService.Application.Repositories; + +namespace ReportService.Application.Build; + +/// Port of Python tools/keywords.py + content_analytics._build_keyword_opportunities. +public static class KeywordOpportunitiesBuilder +{ + private static readonly Dictionary DefaultWeights = new(StringComparer.Ordinal) + { + ["volume"] = 0.40, + ["relevance"] = 0.30, + ["ctr_est"] = 0.15, + ["ease"] = 0.15, + }; + + private static readonly Regex TokenRegex = new(@"\b[\w']+\b", RegexOptions.Compiled); + private static readonly Regex SlugWordRegex = new(@"\b[\w']+\b", RegexOptions.Compiled); + private static readonly Regex Status2Xx = new(@"^2\d{2}$", RegexOptions.Compiled); + + public static Dictionary Build( + IReadOnlyList rows, + IReadOnlyDictionary? config) + { + if (!ParseBool(config, "include_keyword_opportunities", defaultValue: true)) + { + return new Dictionary(); + } + + var empty = new Dictionary + { + ["quick_wins"] = Array.Empty(), + ["high_value"] = Array.Empty(), + ["token_topic_clusters"] = Array.Empty(), + }; + + if (rows.Count == 0) + { + return empty; + } + + var success = rows.Where(r => Status2Xx.IsMatch(r.Status?.Trim() ?? "")).ToList(); + if (success.Count == 0) + { + return empty; + } + + var candidates = ExtractCandidates(success); + if (candidates.Count == 0) + { + return empty; + } + + var corpusSize = success.Count; + var scored = ScoreKeywords(candidates, corpusSize); + var clusters = TextHygieneHelper.FilterTopicClusters(ClusterKeywords(scored)).Take(50).ToList(); + var quickWins = scored.Where(s => GetDouble(s, "difficulty") < 60).Take(10).ToList(); + var highValue = scored.Where(s => GetDouble(s, "volume") >= 0.5).Take(10).ToList(); + if (highValue.Count == 0) + { + highValue = scored.Take(10).ToList(); + } + + return new Dictionary + { + ["quick_wins"] = quickWins, + ["high_value"] = highValue, + ["token_topic_clusters"] = clusters, + }; + } + + private static Dictionary ExtractCandidates(IReadOnlyList rows) + { + var candidates = new Dictionary(StringComparer.Ordinal); + foreach (var row in rows) + { + var url = row.Url?.Trim() ?? ""; + if (url.Length == 0 || row.Status?.StartsWith('4') == true || row.Status?.StartsWith('5') == true) + { + continue; + } + + var allTokens = new List(); + foreach (var text in new[] { row.Title, row.MetaDescription, row.H1 }) + { + if (!string.IsNullOrWhiteSpace(text)) + { + allTokens.AddRange(Tokenize(text)); + } + } + + allTokens.AddRange(SlugTokens(url)); + foreach (var (word, count) in ParseTopKeywords(row.TopKeywords)) + { + AddCandidate(candidates, word, url, count); + } + + if (allTokens.Count == 0) + { + continue; + } + + for (var n = 1; n <= 4; n++) + { + foreach (var ng in Ngrams(allTokens, n)) + { + if (ng.Length < 2) + { + continue; + } + + AddCandidate(candidates, ng, url); + } + } + } + + return candidates; + } + + private static List> ScoreKeywords( + Dictionary candidates, + int corpusSize, + IReadOnlyDictionary? weights = null) + { + weights ??= DefaultWeights; + var relevanceScores = RelevanceTfIdf(candidates, corpusSize); + var results = new List>(); + + foreach (var (kw, data) in candidates) + { + if (TextHygieneHelper.IsJunkSemanticTerm(kw)) + { + continue; + } + + var rawVol = (data.Count / (double)Math.Max(corpusSize, 1)) * 100; + var volume = Math.Min(1.0, rawVol); + const double difficulty = 50.0; + var ease = 1.0 - (difficulty / 100.0); + var relevance = relevanceScores.GetValueOrDefault(kw, 0.5); + const double ctrEst = 0.1; + var score = + weights.GetValueOrDefault("volume", 0.4) * volume + + weights.GetValueOrDefault("relevance", 0.3) * relevance + + weights.GetValueOrDefault("ctr_est", 0.15) * ctrEst + + weights.GetValueOrDefault("ease", 0.15) * ease; + + var action = data.Sources.Count > 1 + ? "internal link" + : relevance > 0.7 + ? "optimize page" + : "create content"; + + results.Add(new Dictionary + { + ["keyword"] = kw, + ["score"] = Math.Round(score, 4), + ["volume"] = Math.Round(volume, 4), + ["difficulty"] = difficulty, + ["difficulty_estimated"] = true, + ["relevance"] = Math.Round(relevance, 4), + ["ctr_est"] = Math.Round(ctrEst, 4), + ["current_rank"] = null, + ["recommended_action"] = action, + ["source"] = "site", + ["data_source"] = "crawl_heuristic", + ["sources_count"] = data.Sources.Count, + }); + } + + return results.OrderByDescending(r => GetDouble(r, "score")).ToList(); + } + + private static List> ClusterKeywords( + IReadOnlyList> scored) + { + if (scored.Count == 0) + { + return []; + } + + var kwToTokens = scored.ToDictionary( + s => s.GetValueOrDefault("keyword")?.ToString() ?? "", + s => new HashSet(Tokenize(s.GetValueOrDefault("keyword")?.ToString() ?? ""), StringComparer.Ordinal), + StringComparer.Ordinal); + + var used = new HashSet(StringComparer.Ordinal); + var kwList = scored.Select(s => s.GetValueOrDefault("keyword")?.ToString() ?? "").ToList(); + var clusters = new List>(); + + foreach (var s in scored) + { + var kw = s.GetValueOrDefault("keyword")?.ToString() ?? ""; + if (TextHygieneHelper.IsJunkSemanticTerm(kw) || used.Contains(kw)) + { + continue; + } + + var cluster = new HashSet(StringComparer.Ordinal) { kw }; + used.Add(kw); + var tokens = new HashSet(kwToTokens.GetValueOrDefault(kw, []), StringComparer.Ordinal); + + foreach (var other in kwList) + { + if (used.Contains(other)) + { + continue; + } + + if (!tokens.Overlaps(kwToTokens.GetValueOrDefault(other, []))) + { + continue; + } + + cluster.Add(other); + used.Add(other); + foreach (var t in kwToTokens.GetValueOrDefault(other, [])) + { + tokens.Add(t); + } + } + + clusters.Add(cluster); + } + + var scoreByKw = scored.ToDictionary( + s => s.GetValueOrDefault("keyword")?.ToString() ?? "", + s => GetDouble(s, "score"), + StringComparer.Ordinal); + + var output = new List>(); + foreach (var cluster in clusters) + { + if (cluster.Count == 0) + { + continue; + } + + var topKw = cluster.MaxBy(k => scoreByKw.GetValueOrDefault(k, 0)) ?? cluster.First(); + var scoresIn = cluster.Select(k => scoreByKw.GetValueOrDefault(k, 0)).ToList(); + var clusterScore = scoresIn.Count > 0 ? scoresIn.Sum() / scoresIn.Count : 0; + output.Add(new Dictionary + { + ["top_keyword"] = topKw, + ["cluster_score"] = Math.Round(clusterScore, 4), + ["keywords"] = cluster.OrderBy(k => k, StringComparer.Ordinal).ToList(), + }); + } + + return output.OrderByDescending(c => GetDouble(c, "cluster_score")).ToList(); + } + + private sealed class CandidateData + { + public HashSet Sources { get; } = new(StringComparer.Ordinal); + public int Count { get; set; } + } + + private static void AddCandidate( + Dictionary candidates, + string keyword, + string url, + int weight = 1) + { + var kw = keyword.Trim().ToLowerInvariant(); + if (kw.Length < 2 || TextHygieneHelper.IsJunkSemanticTerm(kw)) + { + return; + } + + if (!candidates.TryGetValue(kw, out var data)) + { + data = new CandidateData(); + candidates[kw] = data; + } + + data.Sources.Add(url); + data.Count += Math.Max(1, weight); + } + + private static Dictionary RelevanceTfIdf( + Dictionary candidates, + int corpusSize) + { + var totalDocs = corpusSize > 0 ? corpusSize : 1; + var scores = new Dictionary(StringComparer.Ordinal); + foreach (var (kw, data) in candidates) + { + var docFreq = Math.Max(data.Sources.Count, 1); + var idf = 1.0 + Math.Pow(totalDocs / (double)docFreq, 0.5); + var tf = Math.Min(1.0, data.Count / (double)Math.Max(totalDocs, 1)); + scores[kw] = Math.Min(1.0, (tf * idf) / 10.0); + } + + return scores; + } + + private static List<(string Word, int Count)> ParseTopKeywords(string? raw) + { + var output = new List<(string Word, int Count)>(); + if (string.IsNullOrWhiteSpace(raw)) + { + return output; + } + + try + { + using var doc = JsonDocument.Parse(raw); + if (doc.RootElement.ValueKind != JsonValueKind.Array) + { + return output; + } + + foreach (var item in doc.RootElement.EnumerateArray()) + { + if (item.ValueKind != JsonValueKind.Object) + { + continue; + } + + var word = item.TryGetProperty("word", out var w) ? w.GetString()?.Trim().ToLowerInvariant() : null; + if (string.IsNullOrEmpty(word) || word.Length < 3 || TextHygieneHelper.IsJunkSemanticTerm(word)) + { + continue; + } + + var count = item.TryGetProperty("count", out var c) && c.TryGetInt32(out var n) ? Math.Max(1, n) : 1; + output.Add((word, count)); + } + } + catch (JsonException) + { + // ignore malformed payload + } + + return output; + } + + private static List SlugTokens(string url) + { + if (!Uri.TryCreate(url, UriKind.Absolute, out var uri)) + { + return []; + } + + var path = uri.AbsolutePath.Trim('/'); + if (path.Length == 0) + { + return []; + } + + var skip = new HashSet(StringComparer.OrdinalIgnoreCase) + { + "html", "php", "asp", "aspx", "jsp", + }; + + var output = new List(); + foreach (var seg in path.Split('/', StringSplitOptions.RemoveEmptyEntries)) + { + if (skip.Contains(seg)) + { + continue; + } + + var normalized = seg.Replace('-', ' ').Replace('_', ' '); + foreach (Match m in SlugWordRegex.Matches(normalized.ToLowerInvariant())) + { + output.Add(NormalizeToken(m.Value)); + } + } + + return output.Where(t => t.Length > 0).ToList(); + } + + private static List Ngrams(IReadOnlyList tokens, int n) + { + if (n <= 0 || n > tokens.Count) + { + return []; + } + + var output = new List(); + for (var i = 0; i <= tokens.Count - n; i++) + { + output.Add(string.Join(' ', tokens.Skip(i).Take(n))); + } + + return output; + } + + private static List Tokenize(string text) + { + return TokenRegex.Matches(text.ToLowerInvariant()) + .Select(m => NormalizeToken(m.Value)) + .Where(t => t.Length > 0) + .ToList(); + } + + private static string NormalizeToken(string token) => + Regex.Replace(token.ToLowerInvariant().Trim(), @"[^\w\s]", "").Trim(); + + private static double GetDouble(IReadOnlyDictionary dict, string key) => + dict.TryGetValue(key, out var val) && val is not null && double.TryParse(val.ToString(), out var d) ? d : 0; + + private static bool ParseBool(IReadOnlyDictionary? config, string key, bool defaultValue) + { + if (config is null || !config.TryGetValue(key, out var raw)) + { + return defaultValue; + } + + return raw.Trim().ToLowerInvariant() switch + { + "0" or "false" or "no" or "off" => false, + "1" or "true" or "yes" or "on" => true, + _ => defaultValue, + }; + } +} diff --git a/services/ReportService/src/ReportService.Application/Build/NativeReportBuilder.cs b/services/ReportService/src/ReportService.Application/Build/NativeReportBuilder.cs index d2334c1..99befbb 100644 --- a/services/ReportService/src/ReportService.Application/Build/NativeReportBuilder.cs +++ b/services/ReportService/src/ReportService.Application/Build/NativeReportBuilder.cs @@ -14,6 +14,9 @@ public sealed class NativeReportBuilder( LighthouseDbReader lighthouseDbReader, LinkEdgesReader linkEdgesReader, IntegrationsReportDataClient integrationsReportData, + AiServiceEnrichmentClient aiServiceEnrichment, + CrawlPageHtmlReader crawlPageHtmlReader, + IHttpClientFactory httpClientFactory, SitemapDiscoveryService sitemapDiscovery, SiteLevelBuilder siteLevelBuilder, SubdomainInventoryBuilder subdomainInventoryBuilder, @@ -110,6 +113,16 @@ public async Task BuildNativeSliceAsync( var categoryList = categories.ToList(); + var (auditedCategories, optionalAuditMeta) = await OptionalAuditsBuilder.ApplyAsync( + categoryList, + rows, + config, + crawlRunId, + crawlPageHtmlReader, + httpClientFactory, + cancellationToken); + categoryList = auditedCategories.ToList(); + var gapLimit = int.TryParse(config?.GetValueOrDefault("google_url_gap_list_limit"), out var gl) ? gl : 200; var indexation = await IndexationCoverageBuilder.BuildAsync( rows, @@ -159,6 +172,12 @@ public async Task BuildNativeSliceAsync( var successRows = CategoryHelpers.SuccessRows(rows); var contentUrls = ContentUrlListsBuilder.Build(rows, successRows); var contentAnalytics = ContentAnalyticsBuilder.BuildContentAnalytics(rows); + var keywordOpportunities = KeywordOpportunitiesBuilder.Build(rows, config); + var semanticKeywordClusters = await BuildSemanticKeywordClustersAsync( + contentAnalytics, + mlBundle, + cancellationToken); + var optionalAuditUrls = OptionalAuditUrlsBuilder.Build(categoryList); var responseTimeStats = ContentAnalyticsBuilder.BuildResponseTimeStats(rows); var depthDistribution = ContentAnalyticsBuilder.BuildDepthDistribution(rows); var chartData = ReportChartDataBuilder.Build(rows); @@ -218,7 +237,11 @@ public async Task BuildNativeSliceAsync( ImageInventorySummary: imageInventorySummary, Subdomains: subdomains, CrawlSegments: crawlSegments, - LighthouseFailureUrls: lighthouseFailureUrls); + LighthouseFailureUrls: lighthouseFailureUrls, + KeywordOpportunities: keywordOpportunities, + SemanticKeywordClusters: semanticKeywordClusters, + OptionalAuditUrls: optionalAuditUrls, + OptionalAuditMeta: optionalAuditMeta); var corePayload = NativeReportPayloadAssembler.AssembleCore( slice, @@ -279,6 +302,52 @@ public async Task BuildAsync( } } + private async Task>> BuildSemanticKeywordClustersAsync( + Dictionary contentAnalytics, + IReadOnlyDictionary? mlBundle, + CancellationToken cancellationToken) + { + var words = new List(); + if (contentAnalytics.TryGetValue("top_keywords_site", out var topObj) + && topObj is IEnumerable> topKeywords) + { + foreach (var item in topKeywords) + { + var word = item.GetValueOrDefault("word")?.ToString()?.Trim(); + if (!string.IsNullOrEmpty(word) && !TextHygieneHelper.IsJunkSemanticTerm(word)) + { + words.Add(word); + } + } + } + + if (words.Count < 2) + { + return []; + } + + try + { + return await aiServiceEnrichment.TryClusterKeywordsAsync(words, cancellationToken); + } + catch (Exception ex) + { + if (mlBundle is Dictionary mutable) + { + if (!mutable.TryGetValue("ml_errors", out var errorsObj) + || errorsObj is not List errors) + { + errors = []; + mutable["ml_errors"] = errors; + } + + errors.Add(ex.Message); + } + + return []; + } + } + private static Dictionary BuildSummarySeoPayload( Dictionary>> issues) { @@ -397,6 +466,10 @@ public sealed record NativeReportSlice( Dictionary? Subdomains = null, Dictionary? CrawlSegments = null, Dictionary? LighthouseFailureUrls = null, + Dictionary? KeywordOpportunities = null, + List>? SemanticKeywordClusters = null, + Dictionary? OptionalAuditUrls = null, + Dictionary? OptionalAuditMeta = null, IReadOnlyDictionary? MlBundle = null) { public Dictionary? CorePayload { get; init; } diff --git a/services/ReportService/src/ReportService.Application/Build/NativeReportPayloadAssembler.cs b/services/ReportService/src/ReportService.Application/Build/NativeReportPayloadAssembler.cs index ae2333e..c663259 100644 --- a/services/ReportService/src/ReportService.Application/Build/NativeReportPayloadAssembler.cs +++ b/services/ReportService/src/ReportService.Application/Build/NativeReportPayloadAssembler.cs @@ -68,8 +68,14 @@ public static class NativeReportPayloadAssembler } payload["security_findings"] = slice.SecurityFindings ?? []; - payload["keyword_opportunities"] = new Dictionary(); - payload["semantic_keyword_clusters"] = Array.Empty(); + payload["keyword_opportunities"] = slice.KeywordOpportunities + ?? new Dictionary + { + ["quick_wins"] = Array.Empty(), + ["high_value"] = Array.Empty(), + ["token_topic_clusters"] = Array.Empty(), + }; + payload["semantic_keyword_clusters"] = slice.SemanticKeywordClusters ?? []; payload["image_inventory"] = slice.ImageInventory ?? []; payload["image_inventory_summary"] = slice.ImageInventorySummary ?? new Dictionary @@ -81,13 +87,21 @@ public static class NativeReportPayloadAssembler ["unoptimized_min_kb"] = 200, ["inventory_available"] = false, }; - payload["optional_audit_urls"] = new Dictionary + payload["optional_audit_urls"] = slice.OptionalAuditUrls + ?? new Dictionary + { + ["spell"] = Array.Empty(), + ["html"] = Array.Empty(), + ["amp"] = Array.Empty(), + ["pagination"] = Array.Empty(), + }; + if (slice.OptionalAuditMeta is not null) { - ["spell"] = Array.Empty(), - ["html"] = Array.Empty(), - ["amp"] = Array.Empty(), - ["pagination"] = Array.Empty(), - }; + foreach (var (key, value) in slice.OptionalAuditMeta) + { + payload[key] = value; + } + } payload["lighthouse_failure_urls"] = slice.LighthouseFailureUrls ?? new Dictionary { diff --git a/services/ReportService/src/ReportService.Application/Build/OptionalAuditUrlsBuilder.cs b/services/ReportService/src/ReportService.Application/Build/OptionalAuditUrlsBuilder.cs new file mode 100644 index 0000000..d85525c --- /dev/null +++ b/services/ReportService/src/ReportService.Application/Build/OptionalAuditUrlsBuilder.cs @@ -0,0 +1,54 @@ +namespace ReportService.Application.Build; + +/// Bucket category issues into optional_audit_urls lists (Python builder.py message heuristics). +public static class OptionalAuditUrlsBuilder +{ + public static Dictionary Build(IReadOnlyList categories) + { + var spell = new List>(); + var html = new List>(); + var amp = new List>(); + var pagination = new List>(); + + foreach (var cat in categories) + { + foreach (var issue in cat.Issues) + { + var msg = issue.Message.ToLowerInvariant(); + var rec = new Dictionary + { + ["url"] = issue.Url ?? "", + ["message"] = issue.Message, + }; + + if (msg.Contains("spell", StringComparison.Ordinal)) + { + spell.Add(rec); + } + else if (msg.Contains("html", StringComparison.Ordinal) + && msg.Contains("validation", StringComparison.Ordinal)) + { + html.Add(rec); + } + else if (msg.Contains("amp", StringComparison.Ordinal)) + { + amp.Add(rec); + } + else if (msg.Contains("pagination", StringComparison.Ordinal) + || msg.Contains("rel=prev", StringComparison.Ordinal) + || msg.Contains("rel=next", StringComparison.Ordinal)) + { + pagination.Add(rec); + } + } + } + + return new Dictionary + { + ["spell"] = spell, + ["html"] = html, + ["amp"] = amp, + ["pagination"] = pagination, + }; + } +} diff --git a/services/ReportService/src/ReportService.Application/Build/OptionalAuditsBuilder.cs b/services/ReportService/src/ReportService.Application/Build/OptionalAuditsBuilder.cs new file mode 100644 index 0000000..5053ce6 --- /dev/null +++ b/services/ReportService/src/ReportService.Application/Build/OptionalAuditsBuilder.cs @@ -0,0 +1,554 @@ +using System.Text.Json; +using System.Text.RegularExpressions; +using ReportService.Application.Repositories; + +namespace ReportService.Application.Build; + +/// Port of Python reporting/optional_audits.py — pagination, spell, HTML, AMP, Wayback, axe. +public static class OptionalAuditsBuilder +{ + private static readonly Regex WordRegex = new(@"[a-zA-Z']{4,}", RegexOptions.Compiled); + private static readonly Regex TitleTagRegex = new("]*>", RegexOptions.IgnoreCase | RegexOptions.Compiled); + private static readonly Regex HtmlCloseRegex = new("", RegexOptions.IgnoreCase | RegexOptions.Compiled); + private static readonly Regex IdAttrRegex = new("""\bid=["']([^"']+)["']""", RegexOptions.IgnoreCase | RegexOptions.Compiled); + + public static async Task<(IReadOnlyList Categories, Dictionary Meta)> ApplyAsync( + IList categories, + IReadOnlyList rows, + IReadOnlyDictionary? config, + long? crawlRunId, + CrawlPageHtmlReader? htmlReader, + IHttpClientFactory? httpClientFactory, + CancellationToken cancellationToken = default) + { + var meta = new Dictionary(); + var cfg = config ?? new Dictionary(); + + var pagination = PaginationIssues(rows); + if (pagination.Count > 0) + { + CategoryHelpers.MergeIssuesIntoCategory(categories, "technical_seo", pagination); + meta["pagination_issues"] = pagination.Count; + } + + if (ParseBool(cfg, "enable_spell_check", defaultValue: false)) + { + var (spellIssues, skipReason) = SpellCheckIssues(rows); + if (!string.IsNullOrEmpty(skipReason)) + { + meta["spell_check_skipped"] = skipReason; + } + + if (spellIssues.Count > 0) + { + CategoryHelpers.MergeIssuesIntoCategory(categories, "intelligence", spellIssues); + meta["spell_check_pages"] = spellIssues.Count; + } + } + + if (ParseBool(cfg, "enable_html_validation", defaultValue: false)) + { + var (htmlIssues, usedParser) = await HtmlValidationIssuesAsync( + rows, + crawlRunId, + htmlReader, + cancellationToken); + meta["html_validation_parser"] = usedParser ? "html5lib" : "regex"; + if (htmlIssues.Count > 0) + { + CategoryHelpers.MergeIssuesIntoCategory(categories, "technical_seo", htmlIssues); + meta["html_validation_pages"] = htmlIssues.Count; + } + } + + if (ParseBool(cfg, "enable_amp_audit", defaultValue: false)) + { + var amp = AmpAuditIssues(rows); + if (amp.Count > 0) + { + CategoryHelpers.MergeIssuesIntoCategory(categories, "technical_seo", amp); + meta["amp_audit_issues"] = amp.Count; + } + } + + if (ParseBool(cfg, "enable_wayback_lookup", defaultValue: false)) + { + var wb = await WaybackIssuesAsync(rows, httpClientFactory, cancellationToken); + if (wb.Count > 0) + { + CategoryHelpers.MergeIssuesIntoCategory(categories, "technical_seo", wb); + meta["wayback_404_checked"] = wb.Count; + } + } + + if (ParseBool(cfg, "enable_axe", defaultValue: false)) + { + var renderMode = cfg.GetValueOrDefault("crawl_render_mode")?.Trim().ToLowerInvariant() ?? "static"; + if (renderMode == "static") + { + meta["axe_skipped"] = "enable_axe requires javascript or auto crawl rendering"; + } + else + { + var axe = AxeIssuesFromRows(rows); + if (axe.Count > 0) + { + CategoryHelpers.MergeIssuesIntoCategory(categories, "html_accessibility", axe); + meta["axe_violation_count"] = axe.Count; + } + } + } + + return (categories.ToList(), meta); + } + + internal static List PaginationIssues(IReadOnlyList rows) + { + var issues = new List(); + if (rows.Count == 0) + { + return issues; + } + + var orphanPrev = 0; + var ampMismatch = 0; + foreach (var row in rows) + { + var url = row.Url?.Trim() ?? ""; + if (url.Length == 0) + { + continue; + } + + var (relNext, relPrev, amphtml) = ParsePagination(row.PageAnalysisJson); + if (!string.IsNullOrWhiteSpace(relPrev) && string.IsNullOrWhiteSpace(relNext)) + { + orphanPrev++; + } + + if (!string.IsNullOrWhiteSpace(amphtml) + && !string.IsNullOrWhiteSpace(row.CanonicalUrl) + && !string.Equals(amphtml.Trim(), row.CanonicalUrl.Trim(), StringComparison.Ordinal)) + { + ampMismatch++; + } + } + + if (orphanPrev > 0) + { + issues.Add(CategoryHelpers.Issue( + $"{orphanPrev} page(s) have rel=prev without rel=next (pagination chain may be broken).", + priority: "Medium", + recommendation: "Ensure paginated series use paired rel=next and rel=prev links.")); + } + + if (ampMismatch > 0) + { + issues.Add(CategoryHelpers.Issue( + $"{ampMismatch} page(s) have amphtml link that does not match canonical.", + priority: "Medium", + recommendation: "Pair AMP and canonical URLs correctly for mobile/desktop variants.")); + } + + return issues; + } + + internal static (List Issues, string? SkipReason) SpellCheckIssues( + IReadOnlyList rows, + int maxPages = 50) + { + var issues = new List(); + var (checker, skipReason) = SpellCheckerFactory.GetOrCreate(); + if (checker is null) + { + return (issues, skipReason); + } + + var checkedCount = 0; + foreach (var row in rows) + { + if (checkedCount >= maxPages) + { + break; + } + + if (!CategoryHelpers.IsSuccessStatus(row.Status)) + { + continue; + } + + var excerpt = SpellTextParts(row).Trim(); + if (excerpt.Length < 40) + { + continue; + } + + var words = WordRegex.Matches(excerpt.ToLowerInvariant()) + .Select(m => m.Value) + .Take(120) + .ToList(); + if (words.Count == 0) + { + continue; + } + + checkedCount++; + var unknown = words.Where(w => !checker.Check(w)).Distinct(StringComparer.OrdinalIgnoreCase).ToList(); + if (unknown.Count >= 3) + { + var sample = string.Join(", ", unknown.OrderBy(w => w, StringComparer.Ordinal).Take(5)); + issues.Add(CategoryHelpers.Issue( + $"Possible spelling issues ({sample}).", + row.Url, + "Low", + "Review title, H1, and visible copy for typos.")); + } + } + + return (issues.Take(20).ToList(), null); + } + + internal static async Task<(List Issues, bool UsedParser)> HtmlValidationIssuesAsync( + IReadOnlyList rows, + long? crawlRunId, + CrawlPageHtmlReader? htmlReader, + CancellationToken cancellationToken, + int maxPages = 30) + { + var issues = new List(); + var checkedCount = 0; + + if (htmlReader is not null && crawlRunId is > 0) + { + var htmlRows = await htmlReader.ReadBatchAsync(crawlRunId.Value, maxPages, cancellationToken); + foreach (var (url, html) in htmlRows) + { + if (checkedCount >= maxPages) + { + break; + } + + checkedCount++; + var warnings = CollectHtmlWarnings(html); + if (warnings.Count > 0) + { + issues.Add(CategoryHelpers.Issue( + $"HTML structure warnings: {string.Join(", ", warnings)}.", + url, + "Low", + "Fix markup validation issues that may affect parsing or accessibility.")); + } + } + + return (issues, false); + } + + foreach (var row in rows) + { + if (checkedCount >= maxPages) + { + break; + } + + checkedCount++; + var warnings = CollectHtmlWarningsFromMetadata(row); + if (warnings.Count == 0) + { + continue; + } + + issues.Add(CategoryHelpers.Issue( + $"HTML structure warnings: {string.Join(", ", warnings)}.", + row.Url, + "Low", + "Fix markup validation issues that may affect parsing or accessibility.")); + } + + return (issues, false); + } + + internal static List AmpAuditIssues(IReadOnlyList rows) + { + var issues = new List(); + foreach (var row in rows) + { + var url = row.Url ?? ""; + var (_, _, amphtml) = ParsePagination(row.PageAnalysisJson); + var path = Uri.TryCreate(url, UriKind.Absolute, out var uri) ? uri.AbsolutePath.ToLowerInvariant() : ""; + var isAmp = path.Contains("/amp", StringComparison.Ordinal) + || Regex.IsMatch(row.ContentType ?? "", @"\bamp\b", RegexOptions.IgnoreCase); + + if (string.IsNullOrWhiteSpace(amphtml) && !isAmp) + { + continue; + } + + var canon = row.CanonicalUrl?.Trim() ?? ""; + if (string.IsNullOrEmpty(canon)) + { + issues.Add(CategoryHelpers.Issue( + "AMP or amphtml variant missing canonical URL.", + url, + "Medium", + "Add canonical link pointing to the preferred non-AMP URL.")); + } + else if (!string.IsNullOrWhiteSpace(amphtml) + && isAmp + && !string.Equals(amphtml.Trim().TrimEnd('/'), canon.TrimEnd('/'), StringComparison.OrdinalIgnoreCase)) + { + issues.Add(CategoryHelpers.Issue( + "AMP page canonical does not match linked amphtml href.", + url, + "Medium", + "Align canonical URL with amphtml pairing for AMP variants.")); + } + } + + return issues.Take(25).ToList(); + } + + internal static async Task> WaybackIssuesAsync( + IReadOnlyList rows, + IHttpClientFactory? httpClientFactory, + CancellationToken cancellationToken, + int maxLookups = 15) + { + var issues = new List(); + if (httpClientFactory is null) + { + return issues; + } + + var cache = new Dictionary(StringComparer.OrdinalIgnoreCase); + var looked = 0; + var client = httpClientFactory.CreateClient(nameof(OptionalAuditsBuilder)); + client.Timeout = TimeSpan.FromSeconds(10); + + foreach (var row in rows) + { + if (looked >= maxLookups) + { + break; + } + + var status = row.Status?.Trim() ?? ""; + if (!status.StartsWith("404", StringComparison.Ordinal)) + { + continue; + } + + var url = row.Url?.Trim() ?? ""; + if (url.Length == 0) + { + continue; + } + + var cacheKey = url.TrimEnd('/'); + if (cache.TryGetValue(cacheKey, out var cachedAvailable)) + { + if (cachedAvailable) + { + issues.Add(CategoryHelpers.Issue( + "404 URL has Wayback snapshot (Estimated).", + url, + "Low", + "Review whether redirect or content restoration is appropriate.")); + looked++; + } + + continue; + } + + looked++; + try + { + using var response = await client.GetAsync( + $"https://archive.org/wayback/available?url={Uri.EscapeDataString(url)}", + cancellationToken); + if (!response.IsSuccessStatusCode) + { + cache[cacheKey] = false; + continue; + } + + await using var stream = await response.Content.ReadAsStreamAsync(cancellationToken); + using var doc = await JsonDocument.ParseAsync(stream, cancellationToken: cancellationToken); + var available = false; + string? timestamp = null; + if (doc.RootElement.TryGetProperty("archived_snapshots", out var snapshots) + && snapshots.TryGetProperty("closest", out var closest)) + { + available = closest.TryGetProperty("available", out var availEl) + && availEl.ValueKind == JsonValueKind.True; + if (closest.TryGetProperty("timestamp", out var tsEl)) + { + timestamp = tsEl.GetString(); + } + } + + cache[cacheKey] = available; + if (available) + { + issues.Add(CategoryHelpers.Issue( + $"404 URL has Wayback snapshot (Estimated, captured {timestamp ?? "unknown"}).", + url, + "Low", + "Review whether redirect or content restoration is appropriate.")); + } + } + catch (Exception) + { + cache[cacheKey] = false; + } + } + + return issues; + } + + internal static List AxeIssuesFromRows(IReadOnlyList rows) + { + var issues = new List(); + foreach (var row in rows) + { + var pa = CategoryHelpers.ParsePageAnalysisCell(row.PageAnalysisJson); + if (!pa.TryGetValue("axe_violations", out var axeRaw)) + { + continue; + } + + var axeJson = axeRaw?.ToString(); + if (string.IsNullOrWhiteSpace(axeJson)) + { + continue; + } + + try + { + using var doc = JsonDocument.Parse(axeJson); + if (doc.RootElement.ValueKind != JsonValueKind.Array) + { + continue; + } + + var count = 0; + foreach (var v in doc.RootElement.EnumerateArray()) + { + if (count >= 5) + { + break; + } + + if (v.ValueKind != JsonValueKind.Object) + { + continue; + } + + var msg = v.TryGetProperty("description", out var desc) ? desc.GetString() + : v.TryGetProperty("id", out var idEl) ? idEl.GetString() : "axe violation"; + var rec = v.TryGetProperty("help", out var help) ? help.GetString() + : "Fix accessibility violation reported by axe-core."; + issues.Add(CategoryHelpers.Issue( + $"axe: {msg}", + row.Url, + "Medium", + rec ?? "Fix accessibility violation reported by axe-core.")); + count++; + } + } + catch (JsonException) + { + // ignore malformed payload + } + } + + return issues.Take(40).ToList(); + } + + private static (string? RelNext, string? RelPrev, string? Amphtml) ParsePagination(string? pageAnalysisJson) + { + var pa = CategoryHelpers.ParsePageAnalysisCell(pageAnalysisJson); + if (!pa.TryGetValue("pagination", out var pagRaw) || pagRaw is not string pagJson) + { + return (null, null, null); + } + + try + { + using var doc = JsonDocument.Parse(pagJson); + if (doc.RootElement.ValueKind != JsonValueKind.Object) + { + return (null, null, null); + } + + string? GetStr(string name) => + doc.RootElement.TryGetProperty(name, out var el) && el.ValueKind == JsonValueKind.String + ? el.GetString() + : null; + + return (GetStr("rel_next"), GetStr("rel_prev"), GetStr("amphtml")); + } + catch (JsonException) + { + return (null, null, null); + } + } + + private static string SpellTextParts(CrawlRow row) => + string.Join(' ', new[] + { + row.Title ?? "", + row.H1 ?? "", + row.ContentExcerpt ?? "", + row.MetaDescription ?? "", + }.Where(p => !string.IsNullOrWhiteSpace(p))); + + private static List CollectHtmlWarnings(string html) + { + var warnings = new List(); + if (TitleTagRegex.Matches(html).Count > 1) + { + warnings.Add("multiple title tags"); + } + + if (HtmlOpenRegex.IsMatch(html) && !HtmlCloseRegex.IsMatch(html)) + { + warnings.Add("missing closing html tag"); + } + + var ids = IdAttrRegex.Matches(html).Select(m => m.Groups[1].Value).ToList(); + if (ids.Count != ids.Distinct(StringComparer.OrdinalIgnoreCase).Count()) + { + warnings.Add("duplicate id attributes"); + } + + return warnings; + } + + private static List CollectHtmlWarningsFromMetadata(CrawlRow row) + { + // Without stored HTML, only flag obvious metadata gaps. + var warnings = new List(); + if (row.H1Count is > 1) + { + warnings.Add("multiple h1 tags"); + } + + return warnings; + } + + private static bool ParseBool(IReadOnlyDictionary config, string key, bool defaultValue) + { + if (!config.TryGetValue(key, out var raw)) + { + return defaultValue; + } + + return raw.Trim().ToLowerInvariant() switch + { + "0" or "false" or "no" or "off" => false, + "1" or "true" or "yes" or "on" => true, + _ => defaultValue, + }; + } +} diff --git a/services/ReportService/src/ReportService.Application/Build/SpellCheckerFactory.cs b/services/ReportService/src/ReportService.Application/Build/SpellCheckerFactory.cs new file mode 100644 index 0000000..39cf780 --- /dev/null +++ b/services/ReportService/src/ReportService.Application/Build/SpellCheckerFactory.cs @@ -0,0 +1,57 @@ +using System.Text.Json; +using System.Text.RegularExpressions; +using ReportService.Application.Repositories; +using WeCantSpell.Hunspell; + +namespace ReportService.Application.Build; + +/// Optional Hunspell spell checker for optional audits (mirrors pyspellchecker path). +internal static class SpellCheckerFactory +{ + private static WordList? _cached; + private static string? _skipReason; + + public static (WordList? Checker, string? SkipReason) GetOrCreate() + { + if (_cached is not null) + { + return (_cached, null); + } + + if (_skipReason is not null) + { + return (null, _skipReason); + } + + var candidates = new[] + { + Path.Combine(AppContext.BaseDirectory, "Dictionaries", "en_US"), + "/usr/share/hunspell/en_US", + "/usr/local/share/hunspell/en_US", + "/opt/homebrew/share/hunspell/en_US", + }; + + foreach (var basePath in candidates) + { + var dicPath = basePath + ".dic"; + var affPath = basePath + ".aff"; + if (!File.Exists(dicPath) || !File.Exists(affPath)) + { + continue; + } + + try + { + _cached = WordList.CreateFromFiles(dicPath, affPath); + return (_cached, null); + } + catch (Exception) + { + // try next path + } + } + + _skipReason = "Hunspell dictionary not installed"; + return (null, _skipReason); + } +} diff --git a/services/ReportService/src/ReportService.Application/Build/TextHygieneHelper.cs b/services/ReportService/src/ReportService.Application/Build/TextHygieneHelper.cs index 127c439..e410ccf 100644 --- a/services/ReportService/src/ReportService.Application/Build/TextHygieneHelper.cs +++ b/services/ReportService/src/ReportService.Application/Build/TextHygieneHelper.cs @@ -10,6 +10,11 @@ public static class TextHygieneHelper "http", "https", "www", "com", "org", "net", "null", "undefined", "nan", }; + private static readonly HashSet HeadingTokens = new(StringComparer.Ordinal) + { + "h1", "h2", "h3", "h4", "h5", "h6", + }; + public static bool IsJunkSemanticTerm(string? term) { var tokens = Tokenize(term); @@ -18,12 +23,45 @@ public static bool IsJunkSemanticTerm(string? term) return true; } + if (tokens.All(t => HeadingTokens.Contains(t))) + { + return true; + } + if (tokens.All(t => JunkTokens.Contains(t))) { return true; } - return tokens.Any(t => JunkTokens.Contains(t)); + return false; + } + + public static List FilterSemanticTerms(IEnumerable terms) => + terms.Where(t => !string.IsNullOrWhiteSpace(t) && !IsJunkSemanticTerm(t)).ToList(); + + public static List> FilterTopicClusters( + IReadOnlyList> clusters) + { + var output = new List>(); + foreach (var cluster in clusters) + { + var top = (cluster.GetValueOrDefault("top_keyword")?.ToString() + ?? cluster.GetValueOrDefault("representative")?.ToString() ?? "").Trim(); + if (string.IsNullOrEmpty(top) || IsJunkSemanticTerm(top)) + { + continue; + } + + var copy = new Dictionary(cluster); + if (copy.TryGetValue("keywords", out var keywordsObj) && keywordsObj is IEnumerable keywords) + { + copy["keywords"] = FilterSemanticTerms(keywords.Select(k => k?.ToString() ?? "")); + } + + output.Add(copy); + } + + return output; } private static List Tokenize(string? term) => diff --git a/services/ReportService/src/ReportService.Application/DependencyInjection.cs b/services/ReportService/src/ReportService.Application/DependencyInjection.cs index 233b490..8909a7b 100644 --- a/services/ReportService/src/ReportService.Application/DependencyInjection.cs +++ b/services/ReportService/src/ReportService.Application/DependencyInjection.cs @@ -93,12 +93,20 @@ public static IServiceCollection AddReportApplication(this IServiceCollection se { o.IntegrationsServiceUrl = integrations.Trim(); } + + var aiService = Environment.GetEnvironmentVariable("AISERVICE_URL"); + if (!string.IsNullOrWhiteSpace(aiService)) + { + o.AiServiceUrl = aiService.Trim(); + } }); services.AddHttpClient(); services.AddHttpClient(nameof(FastApiPythonBridge)); services.AddHttpClient(nameof(ReportBuildService)); services.AddHttpClient(nameof(IntegrationsReportDataClient)); + services.AddHttpClient(nameof(AiServiceEnrichmentClient)); + services.AddHttpClient(nameof(OptionalAuditsBuilder)); services.AddHttpClient(nameof(SitemapDiscoveryService)); services.AddHttpClient(nameof(SiteLevelBuilder)); services.AddHttpClient(nameof(SubdomainInventoryBuilder)); @@ -129,6 +137,8 @@ public static IServiceCollection AddReportApplication(this IServiceCollection se services.AddScoped(); services.AddScoped(); services.AddScoped(); + services.AddScoped(); + services.AddScoped(); services.AddScoped(); services.AddScoped(); services.AddScoped(); diff --git a/services/ReportService/src/ReportService.Application/Integrations/AiServiceEnrichmentClient.cs b/services/ReportService/src/ReportService.Application/Integrations/AiServiceEnrichmentClient.cs new file mode 100644 index 0000000..756e9ac --- /dev/null +++ b/services/ReportService/src/ReportService.Application/Integrations/AiServiceEnrichmentClient.cs @@ -0,0 +1,89 @@ +using System.Net.Http.Json; +using System.Text.Json; +using Microsoft.Extensions.Options; +using ReportService.Application.Options; + +namespace ReportService.Application.Integrations; + +/// Calls AiService enrichment endpoints for native report build. +public sealed class AiServiceEnrichmentClient( + IHttpClientFactory httpClientFactory, + IOptions options) +{ + private static readonly JsonSerializerOptions JsonOptions = new() + { + PropertyNameCaseInsensitive = true, + }; + + public async Task>> TryClusterKeywordsAsync( + IReadOnlyList keywords, + CancellationToken cancellationToken = default) + { + if (keywords.Count < 2) + { + return []; + } + + var baseUrl = (Environment.GetEnvironmentVariable("AISERVICE_URL") + ?? Environment.GetEnvironmentVariable("AI_SERVICE_URL") + ?? options.Value.AiServiceUrl).Trim().TrimEnd('/'); + if (string.IsNullOrEmpty(baseUrl)) + { + return []; + } + + var client = httpClientFactory.CreateClient(nameof(AiServiceEnrichmentClient)); + client.Timeout = TimeSpan.FromSeconds(60); + + try + { + using var response = await client.PostAsJsonAsync( + $"{baseUrl}/internal/enrichment/cluster-keywords", + new { keywords = keywords.Take(200).ToList() }, + cancellationToken); + if (!response.IsSuccessStatusCode) + { + return []; + } + + await using var stream = await response.Content.ReadAsStreamAsync(cancellationToken); + using var doc = await JsonDocument.ParseAsync(stream, cancellationToken: cancellationToken); + if (!doc.RootElement.TryGetProperty("clusters", out var clustersEl) + || clustersEl.ValueKind != JsonValueKind.Array) + { + return []; + } + + var clusters = new List>(); + foreach (var node in clustersEl.EnumerateArray()) + { + if (node.ValueKind != JsonValueKind.Object) + { + continue; + } + + var dict = JsonSerializer.Deserialize>( + node.GetRawText(), + JsonOptions); + if (dict is not null) + { + clusters.Add(dict); + } + } + + return clusters; + } + catch (HttpRequestException) + { + return []; + } + catch (TaskCanceledException) + { + return []; + } + catch (JsonException) + { + return []; + } + } +} diff --git a/services/ReportService/src/ReportService.Application/Options/ReportServiceOptions.cs b/services/ReportService/src/ReportService.Application/Options/ReportServiceOptions.cs index dd7d856..17e8993 100644 --- a/services/ReportService/src/ReportService.Application/Options/ReportServiceOptions.cs +++ b/services/ReportService/src/ReportService.Application/Options/ReportServiceOptions.cs @@ -11,4 +11,6 @@ public sealed class ReportServiceOptions public bool UsePythonBridge { get; set; } = true; public string IntegrationsServiceUrl { get; set; } = "http://127.0.0.1:8093"; + + public string AiServiceUrl { get; set; } = "http://127.0.0.1:8092"; } diff --git a/services/ReportService/src/ReportService.Application/ReportService.Application.csproj b/services/ReportService/src/ReportService.Application/ReportService.Application.csproj index 3cbb607..7b6abe6 100644 --- a/services/ReportService/src/ReportService.Application/ReportService.Application.csproj +++ b/services/ReportService/src/ReportService.Application/ReportService.Application.csproj @@ -20,6 +20,7 @@ + diff --git a/services/ReportService/src/ReportService.Application/Repositories/CrawlPageHtmlReader.cs b/services/ReportService/src/ReportService.Application/Repositories/CrawlPageHtmlReader.cs new file mode 100644 index 0000000..0e2dcc1 --- /dev/null +++ b/services/ReportService/src/ReportService.Application/Repositories/CrawlPageHtmlReader.cs @@ -0,0 +1,45 @@ +using Npgsql; + +namespace ReportService.Application.Repositories; + +/// Batch-read stored HTML from crawl_page_html for optional HTML validation audits. +public sealed class CrawlPageHtmlReader(NpgsqlDataSource dataSource) +{ + public async Task> ReadBatchAsync( + long crawlRunId, + int limit = 30, + CancellationToken cancellationToken = default) + { + if (crawlRunId <= 0 || limit <= 0) + { + return []; + } + + await using var conn = await dataSource.OpenConnectionAsync(cancellationToken); + await using var cmd = new NpgsqlCommand( + """ + SELECT url, html + FROM crawl_page_html + WHERE crawl_run_id = @runId + ORDER BY url + LIMIT @limit + """, + conn); + cmd.Parameters.AddWithValue("runId", crawlRunId); + cmd.Parameters.AddWithValue("limit", limit); + + var rows = new List<(string Url, string Html)>(); + await using var reader = await cmd.ExecuteReaderAsync(cancellationToken); + while (await reader.ReadAsync(cancellationToken)) + { + var url = reader.IsDBNull(0) ? "" : reader.GetString(0); + var html = reader.IsDBNull(1) ? "" : reader.GetString(1); + if (!string.IsNullOrWhiteSpace(url) && html.Length >= 100) + { + rows.Add((url.Trim(), html)); + } + } + + return rows; + } +} diff --git a/services/ReportService/tests/ReportService.Tests/KeywordOpportunitiesBuilderTests.cs b/services/ReportService/tests/ReportService.Tests/KeywordOpportunitiesBuilderTests.cs new file mode 100644 index 0000000..db1cf64 --- /dev/null +++ b/services/ReportService/tests/ReportService.Tests/KeywordOpportunitiesBuilderTests.cs @@ -0,0 +1,84 @@ +using ReportService.Application.Build; +using ReportService.Application.Repositories; + +namespace ReportService.Tests; + +public sealed class KeywordOpportunitiesBuilderTests +{ + [Fact] + public void Build_returns_empty_when_disabled() + { + var rows = new List + { + new() + { + Url = "https://example.com/seo-guide", + Status = "200", + Title = "SEO guide", + H1 = "SEO guide", + }, + }; + + var result = KeywordOpportunitiesBuilder.Build( + rows, + new Dictionary { ["include_keyword_opportunities"] = "false" }); + + Assert.Empty(result); + } + + [Fact] + public void Build_produces_quick_wins_and_clusters_for_success_rows() + { + var rows = new List + { + new() + { + Url = "https://example.com/seo-guide", + Status = "200", + Title = "SEO guide for beginners", + MetaDescription = "Learn SEO guide basics", + H1 = "SEO guide", + TopKeywords = """[{"word":"seo","count":4},{"word":"guide","count":3}]""", + }, + new() + { + Url = "https://example.com/seo-tools", + Status = "200", + Title = "SEO tools overview", + H1 = "SEO tools", + TopKeywords = """[{"word":"seo","count":2},{"word":"tools","count":2}]""", + }, + }; + + var result = KeywordOpportunitiesBuilder.Build(rows, new Dictionary()); + + var quickWins = Assert.IsType>>(result["quick_wins"]); + var highValue = Assert.IsType>>(result["high_value"]); + var clusters = Assert.IsType>>(result["token_topic_clusters"]); + + Assert.NotEmpty(quickWins); + Assert.NotEmpty(highValue); + Assert.NotEmpty(clusters); + Assert.Contains(quickWins, item => item["keyword"]?.ToString()?.Contains("seo", StringComparison.Ordinal) == true); + } + + [Fact] + public void Build_filters_junk_heading_tokens() + { + var rows = new List + { + new() + { + Url = "https://example.com/page", + Status = "200", + Title = "h1 h2 h3", + H1 = "h1", + }, + }; + + var result = KeywordOpportunitiesBuilder.Build(rows, new Dictionary()); + var quickWins = Assert.IsType>>(result["quick_wins"]); + + Assert.DoesNotContain(quickWins, item => item["keyword"]?.ToString() == "h1"); + } +} diff --git a/services/ReportService/tests/ReportService.Tests/NativeReportPayloadAssemblerTests.cs b/services/ReportService/tests/ReportService.Tests/NativeReportPayloadAssemblerTests.cs index de61e58..2e1968c 100644 --- a/services/ReportService/tests/ReportService.Tests/NativeReportPayloadAssemblerTests.cs +++ b/services/ReportService/tests/ReportService.Tests/NativeReportPayloadAssemblerTests.cs @@ -107,4 +107,74 @@ public void AssembleFull_includes_charts_graph_and_native_flag() Assert.Equal(false, nativeBuild["partial"]); Assert.Equal(true, nativeBuild["native"]); } + + [Fact] + public void AssembleFull_includes_keyword_and_optional_audit_fields_from_slice() + { + var keywordOpportunities = new Dictionary + { + ["quick_wins"] = new List> { new() { ["keyword"] = "seo" } }, + ["high_value"] = new List>(), + ["token_topic_clusters"] = new List>(), + }; + var optionalAuditUrls = new Dictionary + { + ["spell"] = Array.Empty(), + ["html"] = Array.Empty(), + ["amp"] = Array.Empty(), + ["pagination"] = new List> { new() { ["url"] = "https://example.com/p/2" } }, + }; + + var slice = new NativeReportSlice( + CrawlRowCount: 1, + Summary: new Dictionary { ["total_urls"] = 1 }, + SeoHealth: new Dictionary(), + Issues: new Dictionary>> + { + ["broken"] = [], + ["redirects"] = [], + }, + Recommendations: [], + ReportMeta: new Dictionary(), + UrlFingerprints: [], + HreflangSummary: new Dictionary(), + OutboundLinkDomains: [], + LighthouseByUrl: new Dictionary(), + Edges: [], + Categories: [], + Links: [], + OrphanUrls: [], + ContentUrls: ContentUrlListsBuilder.Build([], []), + ContentAnalytics: ContentAnalyticsBuilder.BuildContentAnalytics([]), + ResponseTimeStats: ContentAnalyticsBuilder.BuildResponseTimeStats([]), + DepthDistribution: ContentAnalyticsBuilder.BuildDepthDistribution([]), + ChartData: ReportChartDataBuilder.Build([]), + Graph: ReportGraphBuilder.Build([], []), + TextContentAnalysis: ContentAnalyticsBuilder.BuildTextContentAnalysis([]), + SocialCoverage: ContentAnalyticsBuilder.BuildSocialCoverage([]), + TechStackSummary: ContentAnalyticsBuilder.BuildTechStackSummary([]), + HreflangIssueUrls: [], + LinkEdges: [], + LinkRelSummary: new Dictionary(), + InlinkAnchorMatrix: [], + KeywordOpportunities: keywordOpportunities, + SemanticKeywordClusters: [new Dictionary { ["top_keyword"] = "seo" }], + OptionalAuditUrls: optionalAuditUrls, + OptionalAuditMeta: new Dictionary { ["pagination_issues"] = 1 }); + + var payload = NativeReportPayloadAssembler.AssembleFull(slice); + + var kw = Assert.IsType>(payload["keyword_opportunities"]); + var quickWins = Assert.IsType>>(kw["quick_wins"]); + Assert.Single(quickWins); + + var semantic = Assert.IsType>>(payload["semantic_keyword_clusters"]); + Assert.Single(semantic); + + var auditUrls = Assert.IsType>(payload["optional_audit_urls"]); + var pagination = Assert.IsType>>(auditUrls["pagination"]); + Assert.Single(pagination); + + Assert.Equal(1, payload["pagination_issues"]); + } } diff --git a/services/ReportService/tests/ReportService.Tests/OptionalAuditUrlsBuilderTests.cs b/services/ReportService/tests/ReportService.Tests/OptionalAuditUrlsBuilderTests.cs new file mode 100644 index 0000000..ab25a2e --- /dev/null +++ b/services/ReportService/tests/ReportService.Tests/OptionalAuditUrlsBuilderTests.cs @@ -0,0 +1,42 @@ +using ReportService.Application.Build; + +namespace ReportService.Tests; + +public sealed class OptionalAuditUrlsBuilderTests +{ + [Fact] + public void Build_buckets_issues_by_message_heuristics() + { + var categories = new List + { + new( + "technical_seo", + "Technical SEO", + 90, + [ + new("2 page(s) have rel=prev without rel=next (pagination chain may be broken).", "", "Medium", ""), + new("HTML structure validation warnings: multiple title tags.", "https://example.com/a", "Low", ""), + new("AMP or amphtml variant missing canonical URL.", "https://example.com/amp", "Medium", ""), + ], + []), + new( + "intelligence", + "Content quality", + 100, + [new("Possible spelling issues (teh, wrng).", "https://example.com/b", "Low", "")], + []), + }; + + var buckets = OptionalAuditUrlsBuilder.Build(categories); + + var pagination = Assert.IsType>>(buckets["pagination"]); + var html = Assert.IsType>>(buckets["html"]); + var amp = Assert.IsType>>(buckets["amp"]); + var spell = Assert.IsType>>(buckets["spell"]); + + Assert.Single(pagination); + Assert.Single(html); + Assert.Single(amp); + Assert.Single(spell); + } +} diff --git a/services/ReportService/tests/ReportService.Tests/OptionalAuditsBuilderTests.cs b/services/ReportService/tests/ReportService.Tests/OptionalAuditsBuilderTests.cs new file mode 100644 index 0000000..a6df320 --- /dev/null +++ b/services/ReportService/tests/ReportService.Tests/OptionalAuditsBuilderTests.cs @@ -0,0 +1,125 @@ +using ReportService.Application.Build; +using ReportService.Application.Repositories; + +namespace ReportService.Tests; + +public sealed class OptionalAuditsBuilderTests +{ + [Fact] + public void PaginationIssues_detects_orphan_prev_and_amp_mismatch() + { + var rows = new List + { + new() + { + Url = "https://example.com/page/2", + PageAnalysisJson = """{"pagination":{"rel_prev":"https://example.com/page/1"}}""", + }, + new() + { + Url = "https://example.com/amp/page", + CanonicalUrl = "https://example.com/page", + PageAnalysisJson = """{"pagination":{"amphtml":"https://example.com/other"}}""", + }, + }; + + var issues = OptionalAuditsBuilder.PaginationIssues(rows); + + Assert.Equal(2, issues.Count); + Assert.Contains(issues, i => i.Message.Contains("rel=prev", StringComparison.Ordinal)); + Assert.Contains(issues, i => i.Message.Contains("amphtml", StringComparison.Ordinal)); + } + + [Fact] + public void SpellCheckIssues_skips_when_dictionary_missing() + { + var rows = new List + { + new() + { + Url = "https://example.com/", + Status = "200", + Title = "This is a long enough title with zzqxzz zzqxzz zzqxzz for testing", + H1 = "Heading", + ContentExcerpt = "More content here for spell checking purposes.", + }, + }; + + var (_, skipReason) = OptionalAuditsBuilder.SpellCheckIssues(rows); + + Assert.NotNull(skipReason); + } + + [Fact] + public void AmpAuditIssues_flags_missing_canonical() + { + var rows = new List + { + new() + { + Url = "https://example.com/news/amp/", + ContentType = "text/html; amp", + PageAnalysisJson = """{"pagination":{"amphtml":"https://example.com/news/amp/"}}""", + }, + }; + + var issues = OptionalAuditsBuilder.AmpAuditIssues(rows); + + Assert.Single(issues); + Assert.Contains("missing canonical", issues[0].Message, StringComparison.OrdinalIgnoreCase); + } + + [Fact] + public void AxeIssuesFromRows_parses_violations() + { + var rows = new List + { + new() + { + Url = "https://example.com/", + PageAnalysisJson = """ + { + "axe_violations": [ + {"id":"color-contrast","description":"Contrast too low","help":"Fix contrast"} + ] + } + """, + }, + }; + + var issues = OptionalAuditsBuilder.AxeIssuesFromRows(rows); + + Assert.Single(issues); + Assert.StartsWith("axe:", issues[0].Message, StringComparison.Ordinal); + } + + [Fact] + public async Task ApplyAsync_merges_pagination_into_technical_seo() + { + var categories = new List + { + new("technical_seo", "Technical SEO", 90, [], []), + new("intelligence", "Content quality", 100, [], []), + }; + var rows = new List + { + new() + { + Url = "https://example.com/p/2", + PageAnalysisJson = """{"pagination":{"rel_prev":"https://example.com/p/1"}}""", + }, + }; + + var (updated, meta) = await OptionalAuditsBuilder.ApplyAsync( + categories, + rows, + new Dictionary(), + crawlRunId: null, + htmlReader: null, + httpClientFactory: null); + + var tech = updated.First(c => c.Id == "technical_seo"); + Assert.NotEmpty(tech.Issues); + Assert.Equal(1, meta.GetValueOrDefault("pagination_issues")); + } +} From 6f7aa2980b9ab563f63841dd22b7dd792123edd5 Mon Sep 17 00:00:00 2001 From: PrashantUnity Date: Sat, 27 Jun 2026 17:03:28 +0530 Subject: [PATCH 2/8] micro slop --- .../Handlers/Report/ReportToolHandlers.cs | 13 + .../Build/Categories/MobileCategoryBuilder.cs | 78 +++++- .../Categories/PerformanceCategoryBuilder.cs | 16 +- .../SearchPerformanceCategoryBuilder.cs | 52 ++-- .../Categories/TechnicalSeoCategoryBuilder.cs | 56 +++-- .../Build/CategoryBuilder.cs | 2 +- .../Build/CategoryHelpers.cs | 15 +- .../Build/ContentAnalyticsBuilder.cs | 12 +- .../Build/ContentUrlListsBuilder.cs | 21 +- .../Build/CtrCurve.cs | 30 +++ .../Build/IssueImpactEnricher.cs | 2 +- .../Build/KeywordOpportunitiesBuilder.cs | 108 +++++++- .../Build/NativeReportBuilder.cs | 17 +- .../Build/NativeReportPayloadAssembler.cs | 10 +- .../Build/SeoSummaryBuilder.cs | 233 +++++++++--------- .../Build/SiteHealthScoreBuilder.cs | 77 ++++++ .../Build/ThinContentHelper.cs | 28 +++ .../Repositories/ReportPayloadWriter.cs | 25 +- .../CategoryBuilderGoldenTests.cs | 4 +- .../CategoryHelpersHreflangTests.cs | 27 ++ .../ReportService.Tests/CtrCurveTests.cs | 22 ++ .../KeywordOpportunitiesBuilderTests.cs | 56 +++++ .../NativeReportPayloadAssemblerTests.cs | 2 + .../SearchPerformanceCategoryBuilderTests.cs | 63 +++++ .../SeoSummaryBuilderTests.cs | 120 ++++----- .../SiteHealthScoreBuilderTests.cs | 42 ++++ .../TechnicalSeoCategoryBuilderTests.cs | 18 ++ .../integrations/google/gsc.py | 7 +- .../reporting/content_analytics.py | 7 +- tests/integrations/test_gsc_summary.py | 23 ++ .../reporting/test_content_analytics_bins.py | 26 ++ web/src/_linkutils_extract.json | 4 +- web/src/lib/dashboard/engine/datasets.ts | 6 +- web/src/strings.json | 4 +- web/src/types/report.ts | 3 + 35 files changed, 927 insertions(+), 302 deletions(-) create mode 100644 services/ReportService/src/ReportService.Application/Build/CtrCurve.cs create mode 100644 services/ReportService/src/ReportService.Application/Build/SiteHealthScoreBuilder.cs create mode 100644 services/ReportService/src/ReportService.Application/Build/ThinContentHelper.cs create mode 100644 services/ReportService/tests/ReportService.Tests/CategoryHelpersHreflangTests.cs create mode 100644 services/ReportService/tests/ReportService.Tests/CtrCurveTests.cs create mode 100644 services/ReportService/tests/ReportService.Tests/SiteHealthScoreBuilderTests.cs create mode 100644 tests/integrations/test_gsc_summary.py create mode 100644 tests/reporting/test_content_analytics_bins.py diff --git a/services/AiService/src/AiService.Tools/Handlers/Report/ReportToolHandlers.cs b/services/AiService/src/AiService.Tools/Handlers/Report/ReportToolHandlers.cs index 7fbb6b5..37e2c83 100644 --- a/services/AiService/src/AiService.Tools/Handlers/Report/ReportToolHandlers.cs +++ b/services/AiService/src/AiService.Tools/Handlers/Report/ReportToolHandlers.cs @@ -363,6 +363,19 @@ private static JsonObject IssueCounts(IReadOnlyList issues) private static int? HealthScore(JsonObject payload) { + if (payload["site_health_score"] is JsonValue topLevel + && topLevel.TryGetValue(out int topScore)) + { + return topScore; + } + + if (payload["summary"] is JsonObject summary + && summary["site_health_score"] is JsonValue summaryScore + && summaryScore.TryGetValue(out int fromSummary)) + { + return fromSummary; + } + if (payload["categories"] is not JsonArray categories) { return null; diff --git a/services/ReportService/src/ReportService.Application/Build/Categories/MobileCategoryBuilder.cs b/services/ReportService/src/ReportService.Application/Build/Categories/MobileCategoryBuilder.cs index 578b4d2..11f2c30 100644 --- a/services/ReportService/src/ReportService.Application/Build/Categories/MobileCategoryBuilder.cs +++ b/services/ReportService/src/ReportService.Application/Build/Categories/MobileCategoryBuilder.cs @@ -1,3 +1,4 @@ +using System.Text.Json.Nodes; using System.Text.RegularExpressions; using ReportService.Application.Repositories; @@ -9,7 +10,11 @@ public static class MobileCategoryBuilder "width|device-width", RegexOptions.IgnoreCase | RegexOptions.CultureInvariant | RegexOptions.Compiled); - public static ReportCategory Build(IReadOnlyList rows) + private static readonly string[] LighthouseMobileAudits = ["tap-targets", "font-size"]; + + public static ReportCategory Build( + IReadOnlyList rows, + IReadOnlyDictionary? lighthouseByUrl = null) { var success = CategoryHelpers.SuccessRows(rows); if (success.Count == 0) @@ -49,6 +54,42 @@ public static ReportCategory Build(IReadOnlyList rows) } } + if (lighthouseByUrl is { Count: > 0 }) + { + foreach (var auditId in LighthouseMobileAudits) + { + var failing = new List(); + foreach (var (url, node) in lighthouseByUrl) + { + if (node is not JsonObject lh || !TryGetAuditScore(lh, auditId, out var score) || score >= 0.9) + { + continue; + } + + failing.Add(url); + } + + if (failing.Count == 0) + { + continue; + } + + var label = auditId switch + { + "tap-targets" => "tap targets too small or too close", + "font-size" => "text too small to read on mobile", + _ => auditId, + }; + issues.Add(CategoryHelpers.Issue( + $"{failing.Count} page(s) failed Lighthouse {auditId} ({label}).", + priority: "High", + recommendation: auditId == "tap-targets" + ? "Ensure tap targets are at least 48×48px with adequate spacing." + : "Use a base font size of at least 12px and avoid fixed small text.")); + deductions.Add((Math.Min(15, failing.Count * 3), true)); + } + } + var sorted = CategoryHelpers.SortIssues(issues); return new ReportCategory( "mobile", @@ -57,4 +98,39 @@ public static ReportCategory Build(IReadOnlyList rows) sorted, CategoryHelpers.RecommendationsFromIssues(sorted)); } + + private static bool TryGetAuditScore(JsonObject lh, string auditId, out double score) + { + score = 0; + if (!lh.TryGetPropertyValue("audits", out var auditsNode) || auditsNode is not JsonArray audits) + { + return false; + } + + foreach (var item in audits) + { + if (item is not JsonObject audit) + { + continue; + } + + if (!audit.TryGetPropertyValue("id", out var idNode) + || idNode is not JsonValue idVal + || !idVal.TryGetValue(out string? id) + || !string.Equals(id, auditId, StringComparison.Ordinal)) + { + continue; + } + + if (audit.TryGetPropertyValue("score", out var scoreNode) + && scoreNode is JsonValue scoreVal + && scoreVal.TryGetValue(out double raw)) + { + score = raw; + return true; + } + } + + return false; + } } diff --git a/services/ReportService/src/ReportService.Application/Build/Categories/PerformanceCategoryBuilder.cs b/services/ReportService/src/ReportService.Application/Build/Categories/PerformanceCategoryBuilder.cs index f646d09..cd609cc 100644 --- a/services/ReportService/src/ReportService.Application/Build/Categories/PerformanceCategoryBuilder.cs +++ b/services/ReportService/src/ReportService.Application/Build/Categories/PerformanceCategoryBuilder.cs @@ -57,13 +57,17 @@ public static ReportCategory Build(IReadOnlyList rows) if (totalImgs > 0) { var noLazy = success.Sum(r => r.ImgWithoutLazy ?? 0); - if (noLazy > totalImgs * 0.5) + if (noLazy > 0) { - issues.Add(CategoryHelpers.Issue( - "Many images without lazy loading.", - priority: "Medium", - recommendation: "Add loading='lazy' to off-screen images.")); - deductions.Add((10, true)); + var lazyPct = noLazy * 100.0 / totalImgs; + if (lazyPct > 20) + { + issues.Add(CategoryHelpers.Issue( + "Many images without lazy loading.", + priority: "Medium", + recommendation: "Add loading='lazy' to off-screen images.")); + deductions.Add((Math.Min(15, (int)(noLazy * 10.0 / totalImgs)), true)); + } } var noDims = success.Sum(r => r.ImgWithoutDimensions ?? 0); diff --git a/services/ReportService/src/ReportService.Application/Build/Categories/SearchPerformanceCategoryBuilder.cs b/services/ReportService/src/ReportService.Application/Build/Categories/SearchPerformanceCategoryBuilder.cs index d87234f..92d428d 100644 --- a/services/ReportService/src/ReportService.Application/Build/Categories/SearchPerformanceCategoryBuilder.cs +++ b/services/ReportService/src/ReportService.Application/Build/Categories/SearchPerformanceCategoryBuilder.cs @@ -69,7 +69,7 @@ public static class SearchPerformanceCategoryBuilder if (impressions >= MinImpressionsForCtr && position > 0) { - var expected = ExpectedCtr(position); + var expected = CtrCurve.IndustryCtrPercent(position); if (ctr < expected * 0.6) { issues.Add(new CategoryIssue( @@ -80,6 +80,26 @@ public static class SearchPerformanceCategoryBuilder } } + const double HighOpportunityMinImpressions = 50; + var highOpportunity = topQueries + .Where(q => + { + var pos = ToDouble(q.GetValueOrDefault("position")); + var impr = ToDouble(q.GetValueOrDefault("impressions")); + return pos > 3 && pos <= 10 && impr >= HighOpportunityMinImpressions; + }) + .ToList(); + if (highOpportunity.Count > 0) + { + var sample = string.Join(", ", highOpportunity.Take(3).Select(q => q.GetValueOrDefault("query")?.ToString()).Where(s => !string.IsNullOrEmpty(s))); + var more = highOpportunity.Count > 3 ? $" (+{highOpportunity.Count - 3} more)" : ""; + issues.Add(new CategoryIssue( + $"{highOpportunity.Count} quer(y/ies) rank on page 1 (positions 4–10): {sample}{more}.", + Priority: "Medium", + Recommendation: "Optimize titles, content, and internal links to push these queries into the top 3.")); + deductions.Add((Math.Min(8, highOpportunity.Count), true)); + } + var striking = topQueries .Where(q => { @@ -158,36 +178,6 @@ public static class SearchPerformanceCategoryBuilder recommendations); } - private static double ExpectedCtr(double position) - { - if (position <= 1.5) - { - return 28.0; - } - - if (position <= 2.5) - { - return 15.0; - } - - if (position <= 3.5) - { - return 11.0; - } - - if (position <= 5.0) - { - return 7.0; - } - - if (position <= 10.0) - { - return 3.0; - } - - return 1.0; - } - private static double ToDouble(object? value) => value switch { diff --git a/services/ReportService/src/ReportService.Application/Build/Categories/TechnicalSeoCategoryBuilder.cs b/services/ReportService/src/ReportService.Application/Build/Categories/TechnicalSeoCategoryBuilder.cs index 5760e8a..5618156 100644 --- a/services/ReportService/src/ReportService.Application/Build/Categories/TechnicalSeoCategoryBuilder.cs +++ b/services/ReportService/src/ReportService.Application/Build/Categories/TechnicalSeoCategoryBuilder.cs @@ -43,7 +43,8 @@ public static ReportCategory Build( deductions.Add((5, true)); } - if (GetBool(siteLevel, "ads_txt_present") == false) + if (GetBool(siteLevel, "enable_ads_txt_check") == true + && GetBool(siteLevel, "ads_txt_present") == false) { issues.Add(CategoryHelpers.Issue( "ads.txt is missing or unreachable.", @@ -51,7 +52,8 @@ public static ReportCategory Build( recommendation: "Add an ads.txt file at the site root if you run programmatic advertising.")); } - if (GetBool(siteLevel, "security_txt_present") == false) + if (GetBool(siteLevel, "enable_security_txt_check") == true + && GetBool(siteLevel, "security_txt_present") == false) { issues.Add(CategoryHelpers.Issue( "security.txt is missing or unreachable.", @@ -73,18 +75,26 @@ public static ReportCategory Build( if (rows.Any(r => r.CanonicalUrl is not null)) { + var missingCanonIssues = 0; foreach (var row in success) { var canonical = row.CanonicalUrl?.Trim(); - if (string.IsNullOrEmpty(canonical)) + if (!string.IsNullOrEmpty(canonical)) + { + continue; + } + + if (missingCanonIssues >= CategoryHelpers.MaxIssuesPerCheck) { - issues.Add(CategoryHelpers.Issue( - "Missing canonical URL.", - row.Url, - "Medium", - "Add a canonical link tag pointing to the preferred URL.")); break; } + + issues.Add(CategoryHelpers.Issue( + "Missing canonical URL.", + row.Url, + "Medium", + "Add a canonical link tag pointing to the preferred URL.")); + missingCanonIssues++; } var missingCanon = success.Count(r => string.IsNullOrWhiteSpace(r.CanonicalUrl)); @@ -93,6 +103,8 @@ public static ReportCategory Build( deductions.Add((Math.Min(15, missingCanon * 2), true)); } + var crossCanonIssues = 0; + var crossCanonCount = 0; foreach (var row in success) { var canonical = row.CanonicalUrl?.Trim(); @@ -103,16 +115,28 @@ public static ReportCategory Build( var pageUrl = row.Url.Trim().TrimEnd('/'); var canon = canonical.TrimEnd('/'); - if (!string.Equals(pageUrl, canon, StringComparison.OrdinalIgnoreCase)) + if (string.Equals(pageUrl, canon, StringComparison.OrdinalIgnoreCase)) { - issues.Add(CategoryHelpers.Issue( - $"Canonical points to different URL: {canon}", - row.Url, - "High", - "Set canonical to this page URL or the preferred duplicate.")); - deductions.Add((10, true)); - break; + continue; } + + crossCanonCount++; + if (crossCanonIssues >= CategoryHelpers.MaxIssuesPerCheck) + { + continue; + } + + issues.Add(CategoryHelpers.Issue( + $"Canonical points to different URL: {canon}", + row.Url, + "High", + "Set canonical to this page URL or the preferred duplicate.")); + crossCanonIssues++; + } + + if (crossCanonCount > 0) + { + deductions.Add((Math.Min(10, crossCanonCount * 2), true)); } } diff --git a/services/ReportService/src/ReportService.Application/Build/CategoryBuilder.cs b/services/ReportService/src/ReportService.Application/Build/CategoryBuilder.cs index f8062a0..556b8f9 100644 --- a/services/ReportService/src/ReportService.Application/Build/CategoryBuilder.cs +++ b/services/ReportService/src/ReportService.Application/Build/CategoryBuilder.cs @@ -34,7 +34,7 @@ public IReadOnlyList BuildCategories( PerformanceCategoryBuilder.Build(rows), HtmlAccessibilityCategoryBuilder.Build(rows, lighthouseByUrl, lighthouseSummary), LinkHealthCategoryBuilder.Build(rows, edges, broken, redirects), - MobileCategoryBuilder.Build(rows), + MobileCategoryBuilder.Build(rows, lighthouseByUrl), SecurityCategoryBuilder.Build(rows, startUrl, securityFindings), IntelligenceCategoryBuilder.Build(mlBundle), }; diff --git a/services/ReportService/src/ReportService.Application/Build/CategoryHelpers.cs b/services/ReportService/src/ReportService.Application/Build/CategoryHelpers.cs index 9105157..cc6d06d 100644 --- a/services/ReportService/src/ReportService.Application/Build/CategoryHelpers.cs +++ b/services/ReportService/src/ReportService.Application/Build/CategoryHelpers.cs @@ -8,7 +8,10 @@ namespace ReportService.Application.Build; public static partial class CategoryHelpers { public const int ResponseTimeSlowMs = 2000; - public const int RedirectChainLong = 2; + public const int RedirectChainLong = 3; + public const int MaxIssuesPerCheck = 20; + public const int MaxHreflangIssuesPerCheck = 15; + public const int ThinContentWords = 200; private static readonly Dictionary PriorityOrder = new(StringComparer.Ordinal) { @@ -134,7 +137,10 @@ public static List HreflangIssues(IReadOnlyList success url, "High", "Each hreflang alternate should use a unique language/region code.")); - break; + if (issues.Count(i => i.Message.StartsWith("Duplicate hreflang", StringComparison.Ordinal)) >= MaxHreflangIssuesPerCheck) + { + continue; + } } if (!string.IsNullOrEmpty(url) && hrefs.Count > 0 @@ -145,7 +151,10 @@ public static List HreflangIssues(IReadOnlyList success url, "Medium", "Include a hreflang link pointing to this page URL.")); - break; + if (issues.Count(i => i.Message.StartsWith("Hreflang cluster missing", StringComparison.Ordinal)) >= MaxHreflangIssuesPerCheck) + { + continue; + } } } diff --git a/services/ReportService/src/ReportService.Application/Build/ContentAnalyticsBuilder.cs b/services/ReportService/src/ReportService.Application/Build/ContentAnalyticsBuilder.cs index 6e8b667..c5218ee 100644 --- a/services/ReportService/src/ReportService.Application/Build/ContentAnalyticsBuilder.cs +++ b/services/ReportService/src/ReportService.Application/Build/ContentAnalyticsBuilder.cs @@ -74,16 +74,16 @@ public static class ContentAnalyticsBuilder if (success.Any(r => r.ContentHtmlRatio.HasValue)) { var ratios = success.Select(r => r.ContentHtmlRatio ?? 0).ToList(); - var crBins = new (double Lo, double Hi, string Label)[] + var crBins = new (double Lo, double Hi, string Label, bool InclusiveHi)[] { - (0, 10, "<10%"), - (10.01, 20, "10-20%"), - (20.01, 40, "20-40%"), - (40.01, 100, ">40%"), + (0, 10, "<10%", false), + (10, 20, "10-20%", false), + (20, 40, "20-40%", false), + (40, 100, ">40%", true), }; result["content_ratio_distribution"] = crBins.ToDictionary( b => b.Label, - b => ratios.Count(r => r >= b.Lo && r <= b.Hi)); + b => ratios.Count(r => r >= b.Lo && (b.InclusiveHi ? r <= b.Hi : r < b.Hi))); } if (success.Any(r => !string.IsNullOrWhiteSpace(r.TopKeywords))) diff --git a/services/ReportService/src/ReportService.Application/Build/ContentUrlListsBuilder.cs b/services/ReportService/src/ReportService.Application/Build/ContentUrlListsBuilder.cs index dc58bb5..26c01b2 100644 --- a/services/ReportService/src/ReportService.Application/Build/ContentUrlListsBuilder.cs +++ b/services/ReportService/src/ReportService.Application/Build/ContentUrlListsBuilder.cs @@ -90,25 +90,22 @@ public static class ContentUrlListsBuilder } } - if (rows.Any(r => r.ContentLength.HasValue)) + if (successRows.Any(r => r.ContentLength.HasValue || r.WordCount.HasValue)) { - foreach (var row in rows) + foreach (var row in successRows) { - if (string.IsNullOrWhiteSpace(row.Url)) + if (string.IsNullOrWhiteSpace(row.Url) || !ThinContentHelper.IsThin(row)) { continue; } - var c = row.ContentLength ?? 0; - if (c is > 0 and < SeoSummaryBuilder.ThinContentChars) + result["thin_content"].Add(new Dictionary { - result["thin_content"].Add(new Dictionary - { - ["url"] = row.Url.Trim(), - ["title"] = (row.Title ?? "").Trim(), - ["content_length"] = c, - }); - } + ["url"] = row.Url.Trim(), + ["title"] = (row.Title ?? "").Trim(), + ["content_length"] = row.ContentLength ?? 0, + ["word_count"] = row.WordCount, + }); } } diff --git a/services/ReportService/src/ReportService.Application/Build/CtrCurve.cs b/services/ReportService/src/ReportService.Application/Build/CtrCurve.cs new file mode 100644 index 0000000..93637ee --- /dev/null +++ b/services/ReportService/src/ReportService.Application/Build/CtrCurve.cs @@ -0,0 +1,30 @@ +namespace ReportService.Application.Build; + +/// AWR 2024 desktop organic CTR curve (ported from keyword_enrich.py). +public static class CtrCurve +{ + private static readonly Dictionary Curve = new() + { + [1] = 0.278, + [2] = 0.153, + [3] = 0.103, + [4] = 0.073, + [5] = 0.053, + [6] = 0.040, + [7] = 0.031, + [8] = 0.025, + [9] = 0.021, + [10] = 0.018, + }; + + private const double DefaultFraction = 0.008; + + public static double IndustryCtrFraction(double position) + { + var slot = position > 0 ? Math.Max(1, (int)Math.Ceiling(position)) : 1; + return Curve.GetValueOrDefault(slot, DefaultFraction); + } + + public static double IndustryCtrPercent(double position) => + IndustryCtrFraction(position) * 100.0; +} diff --git a/services/ReportService/src/ReportService.Application/Build/IssueImpactEnricher.cs b/services/ReportService/src/ReportService.Application/Build/IssueImpactEnricher.cs index 00c5c29..a1efc46 100644 --- a/services/ReportService/src/ReportService.Application/Build/IssueImpactEnricher.cs +++ b/services/ReportService/src/ReportService.Application/Build/IssueImpactEnricher.cs @@ -58,7 +58,7 @@ private static CategoryIssue EnrichIssue( continue; } - if (url.EndsWith(key, StringComparison.Ordinal)) + if (url == key || url.EndsWith("/" + key, StringComparison.Ordinal)) { ga4Sessions = Math.Max(ga4Sessions, ga.GetValueOrDefault("ga4_sessions")); } diff --git a/services/ReportService/src/ReportService.Application/Build/KeywordOpportunitiesBuilder.cs b/services/ReportService/src/ReportService.Application/Build/KeywordOpportunitiesBuilder.cs index a250d7c..b8bce76 100644 --- a/services/ReportService/src/ReportService.Application/Build/KeywordOpportunitiesBuilder.cs +++ b/services/ReportService/src/ReportService.Application/Build/KeywordOpportunitiesBuilder.cs @@ -21,7 +21,8 @@ public static class KeywordOpportunitiesBuilder public static Dictionary Build( IReadOnlyList rows, - IReadOnlyDictionary? config) + IReadOnlyDictionary? config, + IReadOnlyDictionary? googleData = null) { if (!ParseBool(config, "include_keyword_opportunities", defaultValue: true)) { @@ -53,7 +54,8 @@ public static class KeywordOpportunitiesBuilder } var corpusSize = success.Count; - var scored = ScoreKeywords(candidates, corpusSize); + var gscPositions = BuildGscPositionLookup(googleData); + var scored = ScoreKeywords(candidates, corpusSize, gscPositions: gscPositions); var clusters = TextHygieneHelper.FilterTopicClusters(ClusterKeywords(scored)).Take(50).ToList(); var quickWins = scored.Where(s => GetDouble(s, "difficulty") < 60).Take(10).ToList(); var highValue = scored.Where(s => GetDouble(s, "volume") >= 0.5).Take(10).ToList(); @@ -121,9 +123,11 @@ private static Dictionary ExtractCandidates(IReadOnlyList private static List> ScoreKeywords( Dictionary candidates, int corpusSize, - IReadOnlyDictionary? weights = null) + IReadOnlyDictionary? weights = null, + IReadOnlyDictionary? gscPositions = null) { weights ??= DefaultWeights; + gscPositions ??= new Dictionary(StringComparer.Ordinal); var relevanceScores = RelevanceTfIdf(candidates, corpusSize); var results = new List>(); @@ -134,12 +138,16 @@ private static Dictionary ExtractCandidates(IReadOnlyList continue; } - var rawVol = (data.Count / (double)Math.Max(corpusSize, 1)) * 100; + var rawVol = data.Count / (double)Math.Max(corpusSize, 1); var volume = Math.Min(1.0, rawVol); - const double difficulty = 50.0; + gscPositions.TryGetValue(kw, out var gscPosition); + var hasGscRank = gscPosition > 0; + var difficulty = EstimateDifficulty(kw, data.Count, corpusSize, hasGscRank ? gscPosition : null); var ease = 1.0 - (difficulty / 100.0); var relevance = relevanceScores.GetValueOrDefault(kw, 0.5); - const double ctrEst = 0.1; + var ctrEst = hasGscRank + ? CtrCurve.IndustryCtrFraction(gscPosition) + : 0.05; var score = weights.GetValueOrDefault("volume", 0.4) * volume + weights.GetValueOrDefault("relevance", 0.3) * relevance @@ -158,13 +166,13 @@ private static Dictionary ExtractCandidates(IReadOnlyList ["score"] = Math.Round(score, 4), ["volume"] = Math.Round(volume, 4), ["difficulty"] = difficulty, - ["difficulty_estimated"] = true, + ["difficulty_estimated"] = !hasGscRank, ["relevance"] = Math.Round(relevance, 4), ["ctr_est"] = Math.Round(ctrEst, 4), - ["current_rank"] = null, + ["current_rank"] = hasGscRank ? gscPosition : null, ["recommended_action"] = action, ["source"] = "site", - ["data_source"] = "crawl_heuristic", + ["data_source"] = hasGscRank ? "gsc+crawl" : "crawl_heuristic", ["sources_count"] = data.Sources.Count, }); } @@ -172,6 +180,86 @@ private static Dictionary ExtractCandidates(IReadOnlyList return results.OrderByDescending(r => GetDouble(r, "score")).ToList(); } + private static double EstimateDifficulty(string keyword, int count, int corpusSize, double? gscPosition) + { + var baseDifficulty = 50.0; + var words = keyword.Split(' ', StringSplitOptions.RemoveEmptyEntries); + if (words.Length >= 4) + { + baseDifficulty -= 20; + } + else if (words.Length == 1) + { + baseDifficulty += 15; + } + + var siteFreq = count / (double)Math.Max(corpusSize, 1); + baseDifficulty += 70 * (1.0 - siteFreq); + + if (gscPosition is > 0) + { + if (gscPosition < 5) + { + baseDifficulty -= 15; + } + else if (gscPosition < 20) + { + baseDifficulty -= 8; + } + } + + return Math.Clamp(baseDifficulty, 0, 100); + } + + private static Dictionary BuildGscPositionLookup(IReadOnlyDictionary? googleData) + { + var lookup = new Dictionary(StringComparer.Ordinal); + if (googleData is null) + { + return lookup; + } + + if (JsonObjectParser.AsDict(googleData.GetValueOrDefault("gsc")) is not { } gsc) + { + return lookup; + } + + foreach (var row in JsonObjectParser.AsDictRows(gsc.GetValueOrDefault("top_queries"))) + { + var query = row.GetValueOrDefault("query")?.ToString()?.Trim().ToLowerInvariant(); + if (string.IsNullOrEmpty(query)) + { + continue; + } + + var position = ToDouble(row.GetValueOrDefault("position")); + if (position <= 0) + { + continue; + } + + if (!lookup.TryGetValue(query, out var existing) || position < existing) + { + lookup[query] = position; + } + } + + return lookup; + } + + private static double ToDouble(object? value) => + value switch + { + null => 0, + double d => d, + float f => f, + int i => i, + long l => l, + decimal m => (double)m, + string s when double.TryParse(s, out var parsed) => parsed, + _ => 0, + }; + private static List> ClusterKeywords( IReadOnlyList> scored) { @@ -288,7 +376,7 @@ private static Dictionary RelevanceTfIdf( foreach (var (kw, data) in candidates) { var docFreq = Math.Max(data.Sources.Count, 1); - var idf = 1.0 + Math.Pow(totalDocs / (double)docFreq, 0.5); + var idf = 1.0 + Math.Log(totalDocs / (double)docFreq); var tf = Math.Min(1.0, data.Count / (double)Math.Max(totalDocs, 1)); scores[kw] = Math.Min(1.0, (tf * idf) / 10.0); } diff --git a/services/ReportService/src/ReportService.Application/Build/NativeReportBuilder.cs b/services/ReportService/src/ReportService.Application/Build/NativeReportBuilder.cs index 99befbb..070f791 100644 --- a/services/ReportService/src/ReportService.Application/Build/NativeReportBuilder.cs +++ b/services/ReportService/src/ReportService.Application/Build/NativeReportBuilder.cs @@ -97,6 +97,7 @@ public async Task BuildNativeSliceAsync( var outbound = ReportMetadataBuilder.BuildOutboundLinkDomains(rows, startUrl, maxOutbound); var summarySeo = BuildSummarySeoPayload(seo.Issues); var siteLevel = await siteLevelBuilder.FetchAsync(startUrl, cancellationToken); + siteLevel = MergeSiteLevelConfig(siteLevel, config); var runSecurityScan = ParseBool(config, "run_security_scan", defaultValue: true); var securityFindings = SecurityScanBuilder.BuildPassive(rows, startUrl, runSecurityScan); var categories = categoryBuilder.BuildCategories( @@ -172,7 +173,7 @@ public async Task BuildNativeSliceAsync( var successRows = CategoryHelpers.SuccessRows(rows); var contentUrls = ContentUrlListsBuilder.Build(rows, successRows); var contentAnalytics = ContentAnalyticsBuilder.BuildContentAnalytics(rows); - var keywordOpportunities = KeywordOpportunitiesBuilder.Build(rows, config); + var keywordOpportunities = KeywordOpportunitiesBuilder.Build(rows, config, googleData); var semanticKeywordClusters = await BuildSemanticKeywordClustersAsync( contentAnalytics, mlBundle, @@ -423,6 +424,20 @@ private static List ParseCompetitorDomains(IReadOnlyDictionary s.Length > 0) .ToList(); } + + private static Dictionary MergeSiteLevelConfig( + IReadOnlyDictionary? siteLevel, + IReadOnlyDictionary? config) + { + var merged = siteLevel is Dictionary dict + ? new Dictionary(dict) + : siteLevel?.ToDictionary(kv => kv.Key, kv => kv.Value) + ?? new Dictionary(); + + merged["enable_ads_txt_check"] = ParseBool(config, "enable_ads_txt_check", defaultValue: false); + merged["enable_security_txt_check"] = ParseBool(config, "enable_security_txt_check", defaultValue: false); + return merged; + } } public sealed record NativeReportSlice( diff --git a/services/ReportService/src/ReportService.Application/Build/NativeReportPayloadAssembler.cs b/services/ReportService/src/ReportService.Application/Build/NativeReportPayloadAssembler.cs index c663259..be7c5c9 100644 --- a/services/ReportService/src/ReportService.Application/Build/NativeReportPayloadAssembler.cs +++ b/services/ReportService/src/ReportService.Application/Build/NativeReportPayloadAssembler.cs @@ -170,12 +170,20 @@ public static class NativeReportPayloadAssembler siteLevel ??= new Dictionary(); mlBundle ??= new Dictionary(); + var (categoryScores, siteHealthScore) = SiteHealthScoreBuilder.ComputeWithCategoryScores(slice.Categories); + var summary = new Dictionary(slice.Summary) + { + ["site_health_score"] = siteHealthScore, + ["category_scores"] = categoryScores, + }; + var payload = new Dictionary { ["site_name"] = siteName ?? "", ["report_title"] = reportTitle ?? "", ["report_generated_at"] = DateTimeOffset.UtcNow.ToString("O"), - ["summary"] = slice.Summary, + ["summary"] = summary, + ["site_health_score"] = siteHealthScore, ["seo_health"] = slice.SeoHealth, ["issues"] = slice.Issues, ["recommendations"] = slice.Recommendations, diff --git a/services/ReportService/src/ReportService.Application/Build/SeoSummaryBuilder.cs b/services/ReportService/src/ReportService.Application/Build/SeoSummaryBuilder.cs index 1e126ae..26a0a8d 100644 --- a/services/ReportService/src/ReportService.Application/Build/SeoSummaryBuilder.cs +++ b/services/ReportService/src/ReportService.Application/Build/SeoSummaryBuilder.cs @@ -12,7 +12,6 @@ public static partial class SeoSummaryBuilder public const int TitleLenMax = 60; public const int MetaDescLenMin = 70; public const int MetaDescLenMax = 160; - public const int ThinContentChars = 300; public sealed record SeoSummaryResult( Dictionary Summary, @@ -28,9 +27,13 @@ public static SeoSummaryResult Compute(IReadOnlyList rows) var count4Xx = 0; var count5Xx = 0; var countError = 0; - var outlinkSum = 0; - var titleLenSum = 0; - double? crawlTimeS = rows.FirstOrDefault()?.CrawlTimeS; + var outlinkSum2Xx = 0; + var titleLenSum2Xx = 0; + var successCount = 0; + var crawlTimes = rows.Select(r => r.CrawlTimeS).Where(t => t.HasValue).Select(t => t!.Value).ToList(); + double? crawlTimeS = crawlTimes.Count >= 2 + ? crawlTimes.Max() - crawlTimes.Min() + : crawlTimes.FirstOrDefault(); var issues = new Dictionary>> { @@ -61,6 +64,114 @@ public static SeoSummaryResult Compute(IReadOnlyList rows) if (Status2Xx().IsMatch(st)) { count2Xx++; + successCount++; + outlinkSum2Xx += row.Outlinks ?? 0; + + var title = (row.Title ?? "").Trim(); + var titleLen = title.Length; + titleLenSum2Xx += titleLen; + + if (titleLen == 0) + { + seoHealth["missing_title"]++; + issues["seo"].Add(new Dictionary + { + ["type"] = "missing_title", + ["url"] = row.Url, + ["message"] = "Missing title", + }); + } + else if (titleLen < TitleLenMin) + { + seoHealth["title_short"]++; + issues["seo"].Add(new Dictionary + { + ["type"] = "title_short", + ["url"] = row.Url, + ["message"] = $"Title too short ({titleLen} chars)", + }); + } + else if (titleLen > TitleLenMax) + { + seoHealth["title_long"]++; + issues["seo"].Add(new Dictionary + { + ["type"] = "title_long", + ["url"] = row.Url, + ["message"] = $"Title too long ({titleLen} chars)", + }); + } + else + { + seoHealth["title_ok"]++; + } + + var mdLen = row.MetaDescriptionLen ?? 0; + if (mdLen == 0) + { + seoHealth["missing_meta_desc"]++; + } + else if (mdLen < MetaDescLenMin) + { + seoHealth["meta_desc_short"]++; + issues["seo"].Add(new Dictionary + { + ["type"] = "meta_desc_short", + ["url"] = row.Url, + ["message"] = $"Meta description too short ({mdLen} chars)", + }); + } + else if (mdLen > MetaDescLenMax) + { + seoHealth["meta_desc_long"]++; + issues["seo"].Add(new Dictionary + { + ["type"] = "meta_desc_long", + ["url"] = row.Url, + ["message"] = $"Meta description too long ({mdLen} chars)", + }); + } + else + { + seoHealth["meta_desc_ok"]++; + } + + var h1c = row.H1Count ?? -1; + if (h1c == 0) + { + seoHealth["h1_zero"]++; + issues["seo"].Add(new Dictionary + { + ["type"] = "h1_missing", + ["url"] = row.Url, + ["message"] = "Missing H1", + }); + } + else if (h1c == 1) + { + seoHealth["h1_one"]++; + } + else if (h1c > 1) + { + seoHealth["h1_multi"]++; + issues["seo"].Add(new Dictionary + { + ["type"] = "h1_multi", + ["url"] = row.Url, + ["message"] = $"Multiple H1s ({h1c})", + }); + } + + if (ThinContentHelper.IsThin(row)) + { + seoHealth["thin_content"]++; + issues["seo"].Add(new Dictionary + { + ["type"] = "thin_content", + ["url"] = row.Url, + ["message"] = ThinContentHelper.ThinContentMessage(row), + }); + } } else if (Status3Xx().IsMatch(st)) { @@ -89,114 +200,6 @@ public static SeoSummaryResult Compute(IReadOnlyList rows) issues["broken"].Add(new Dictionary { ["url"] = row.Url, ["status"] = st }); } - - outlinkSum += row.Outlinks ?? 0; - var title = (row.Title ?? "").Trim(); - var titleLen = title.Length; - titleLenSum += titleLen; - - if (titleLen == 0) - { - seoHealth["missing_title"]++; - issues["seo"].Add(new Dictionary - { - ["type"] = "missing_title", - ["url"] = row.Url, - ["message"] = "Missing title", - }); - } - else if (titleLen < TitleLenMin) - { - seoHealth["title_short"]++; - issues["seo"].Add(new Dictionary - { - ["type"] = "title_short", - ["url"] = row.Url, - ["message"] = $"Title too short ({titleLen} chars)", - }); - } - else if (titleLen > TitleLenMax) - { - seoHealth["title_long"]++; - issues["seo"].Add(new Dictionary - { - ["type"] = "title_long", - ["url"] = row.Url, - ["message"] = $"Title too long ({titleLen} chars)", - }); - } - else - { - seoHealth["title_ok"]++; - } - - var mdLen = row.MetaDescriptionLen ?? 0; - if (mdLen == 0) - { - seoHealth["missing_meta_desc"]++; - } - else if (mdLen < MetaDescLenMin) - { - seoHealth["meta_desc_short"]++; - issues["seo"].Add(new Dictionary - { - ["type"] = "meta_desc_short", - ["url"] = row.Url, - ["message"] = $"Meta description too short ({mdLen} chars)", - }); - } - else if (mdLen > MetaDescLenMax) - { - seoHealth["meta_desc_long"]++; - issues["seo"].Add(new Dictionary - { - ["type"] = "meta_desc_long", - ["url"] = row.Url, - ["message"] = $"Meta description too long ({mdLen} chars)", - }); - } - else - { - seoHealth["meta_desc_ok"]++; - } - - var h1c = row.H1Count ?? -1; - if (h1c == 0) - { - seoHealth["h1_zero"]++; - issues["seo"].Add(new Dictionary - { - ["type"] = "h1_missing", - ["url"] = row.Url, - ["message"] = "Missing H1", - }); - } - else if (h1c == 1) - { - seoHealth["h1_one"]++; - } - else if (h1c > 1) - { - seoHealth["h1_multi"]++; - issues["seo"].Add(new Dictionary - { - ["type"] = "h1_multi", - ["url"] = row.Url, - ["message"] = $"Multiple H1s ({h1c})", - }); - } - - var cl = row.ContentLength ?? 0; - if (cl > 0 && cl < ThinContentChars) - { - seoHealth["thin_content"]++; - issues["seo"].Add(new Dictionary - { - ["type"] = "thin_content", - ["url"] = row.Url, - ["message"] = $"Thin content ({cl} chars)", - }); - } } var summary = new Dictionary @@ -208,8 +211,8 @@ public static SeoSummaryResult Compute(IReadOnlyList rows) ["count_5xx"] = count5Xx, ["count_error"] = countError, ["success_rate"] = total > 0 ? Math.Round(100.0 * count2Xx / total, 1) : 0, - ["avg_outlinks"] = total > 0 ? Math.Round((double)outlinkSum / total, 1) : 0, - ["avg_title_len"] = total > 0 ? Math.Round((double)titleLenSum / total, 1) : 0, + ["avg_outlinks"] = successCount > 0 ? Math.Round((double)outlinkSum2Xx / successCount, 1) : 0, + ["avg_title_len"] = successCount > 0 ? Math.Round((double)titleLenSum2Xx / successCount, 1) : 0, ["crawl_time_s"] = crawlTimeS is not null ? Math.Round(crawlTimeS.Value, 1) : null, }; @@ -264,7 +267,7 @@ private static List BuildRecommendations( if (seoHealth["thin_content"] > 0) { - recs.Add($"Expand thin content on {seoHealth["thin_content"]} page(s) (under {ThinContentChars} chars)."); + recs.Add($"Expand thin content on {seoHealth["thin_content"]} page(s) (under {CategoryHelpers.ThinContentWords} words)."); } return recs; diff --git a/services/ReportService/src/ReportService.Application/Build/SiteHealthScoreBuilder.cs b/services/ReportService/src/ReportService.Application/Build/SiteHealthScoreBuilder.cs new file mode 100644 index 0000000..09ad34d --- /dev/null +++ b/services/ReportService/src/ReportService.Application/Build/SiteHealthScoreBuilder.cs @@ -0,0 +1,77 @@ +namespace ReportService.Application.Build; + +/// Weighted site health score from fixable audit categories (excludes search_performance, intelligence). +public static class SiteHealthScoreBuilder +{ + private static readonly Dictionary Weights = new(StringComparer.Ordinal) + { + ["technical_seo"] = 0.25, + ["link_health"] = 0.20, + ["performance"] = 0.15, + ["security"] = 0.15, + ["core_web_vitals"] = 0.10, + ["mobile"] = 0.10, + ["html_accessibility"] = 0.05, + }; + + private static readonly HashSet Excluded = new(StringComparer.Ordinal) + { + "search_performance", + "intelligence", + }; + + public static int? Compute(IReadOnlyList categories) + { + var (_, score) = ComputeWithCategoryScores(categories); + return score; + } + + public static (Dictionary CategoryScores, int? SiteHealthScore) ComputeWithCategoryScores( + IReadOnlyList categories) + { + var categoryScores = new Dictionary(StringComparer.Ordinal); + foreach (var cat in categories) + { + if (Excluded.Contains(cat.Id) || cat.Score is not int score) + { + continue; + } + + categoryScores[cat.Id] = score; + } + + double weightedSum = 0; + double weightTotal = 0; + foreach (var (id, weight) in Weights) + { + if (!categoryScores.TryGetValue(id, out var score)) + { + continue; + } + + weightedSum += score * weight; + weightTotal += weight; + } + + int? siteHealth = weightTotal > 0 + ? (int)Math.Round(weightedSum / weightTotal) + : null; + + return (categoryScores, siteHealth); + } + + public static (Dictionary CategoryScores, int? SiteHealthScore) ComputeWithCategoryScores( + IEnumerable> payloadCategories) + { + var categories = payloadCategories + .Select(c => new ReportCategory( + c.GetValueOrDefault("id")?.ToString() ?? "", + c.GetValueOrDefault("name")?.ToString() ?? "", + c.TryGetValue("score", out var s) && s is int or double or float ? Convert.ToInt32(s) : null, + [], + [])) + .ToList(); + + return ComputeWithCategoryScores(categories); + } +} diff --git a/services/ReportService/src/ReportService.Application/Build/ThinContentHelper.cs b/services/ReportService/src/ReportService.Application/Build/ThinContentHelper.cs new file mode 100644 index 0000000..d9e4ae9 --- /dev/null +++ b/services/ReportService/src/ReportService.Application/Build/ThinContentHelper.cs @@ -0,0 +1,28 @@ +using ReportService.Application.Repositories; + +namespace ReportService.Application.Build; + +/// Shared thin-content detection (200 words; char fallback when word_count absent). +public static class ThinContentHelper +{ + public static bool IsThin(CrawlRow row) + { + if (row.WordCount is > 0) + { + return row.WordCount < CategoryHelpers.ThinContentWords; + } + + var chars = row.ContentLength ?? 0; + return chars > 0 && chars / 5 < CategoryHelpers.ThinContentWords; + } + + public static string ThinContentMessage(CrawlRow row) + { + if (row.WordCount is > 0) + { + return $"Thin content ({row.WordCount} words)"; + } + + return $"Thin content (~{(row.ContentLength ?? 0) / 5} words, from {row.ContentLength} chars)"; + } +} diff --git a/services/ReportService/src/ReportService.Application/Repositories/ReportPayloadWriter.cs b/services/ReportService/src/ReportService.Application/Repositories/ReportPayloadWriter.cs index 18e7888..2d7c3a3 100644 --- a/services/ReportService/src/ReportService.Application/Repositories/ReportPayloadWriter.cs +++ b/services/ReportService/src/ReportService.Application/Repositories/ReportPayloadWriter.cs @@ -1,4 +1,5 @@ using System.Text.Json; +using ReportService.Application.Build; using ReportService.Application.Persistence; using ReportService.Domain.Entities; @@ -57,13 +58,17 @@ private async Task WriteHealthSnapshotAsync( return; } - var scores = categories - .Where(c => c.TryGetValue("score", out var s) && s is int or double or float) - .Select(c => Convert.ToDouble(c["score"])) - .ToList(); - int? healthScore = scores.Count > 0 ? (int)Math.Round(scores.Average()) : null; + var (categoryScores, healthScore) = reportData.TryGetValue("summary", out var summaryObj) + && summaryObj is Dictionary summary + && summary.TryGetValue("site_health_score", out var hsObj) + && hsObj is int hsFromSummary + ? ( + summary.TryGetValue("category_scores", out var csObj) && csObj is Dictionary csDict + ? csDict + : SiteHealthScoreBuilder.ComputeWithCategoryScores(categories).CategoryScores, + (int?)hsFromSummary) + : SiteHealthScoreBuilder.ComputeWithCategoryScores(categories); - var categoryScores = new Dictionary(StringComparer.Ordinal); var issueCounts = new Dictionary(StringComparer.Ordinal) { ["Critical"] = 0, ["High"] = 0, ["Medium"] = 0, ["Low"] = 0, @@ -71,14 +76,6 @@ private async Task WriteHealthSnapshotAsync( foreach (var cat in categories) { - var key = cat.GetValueOrDefault("id")?.ToString() - ?? cat.GetValueOrDefault("name")?.ToString() - ?? "unknown"; - if (cat.TryGetValue("score", out var scoreObj) && scoreObj is int or double or float) - { - categoryScores[key] = Convert.ToDouble(scoreObj); - } - if (cat.TryGetValue("issues", out var issuesObj) && issuesObj is IEnumerable issueList) { foreach (var issueObj in issueList) diff --git a/services/ReportService/tests/ReportService.Tests/CategoryBuilderGoldenTests.cs b/services/ReportService/tests/ReportService.Tests/CategoryBuilderGoldenTests.cs index bd66b3b..dcbd320 100644 --- a/services/ReportService/tests/ReportService.Tests/CategoryBuilderGoldenTests.cs +++ b/services/ReportService/tests/ReportService.Tests/CategoryBuilderGoldenTests.cs @@ -125,7 +125,9 @@ public void BuildCategories_missing_site_files_are_low_priority_in_technical_seo "sitemap_present": true, "sitemap_valid": true, "ads_txt_present": false, - "security_txt_present": false + "security_txt_present": false, + "enable_ads_txt_check": true, + "enable_security_txt_check": true } """)!; diff --git a/services/ReportService/tests/ReportService.Tests/CategoryHelpersHreflangTests.cs b/services/ReportService/tests/ReportService.Tests/CategoryHelpersHreflangTests.cs new file mode 100644 index 0000000..54a0545 --- /dev/null +++ b/services/ReportService/tests/ReportService.Tests/CategoryHelpersHreflangTests.cs @@ -0,0 +1,27 @@ +using ReportService.Application.Build; +using ReportService.Application.Repositories; + +namespace ReportService.Tests; + +public sealed class CategoryHelpersHreflangTests +{ + [Fact] + public void HreflangIssues_reports_multiple_self_reference_gaps() + { + var rows = Enumerable.Range(0, 3) + .Select(i => new CrawlRow + { + Url = $"https://example.com/{i}", + Status = "200", + PageAnalysisJson = $$""" + {"hreflang_alternates":[{"hreflang":"en","href":"https://example.com/other"}]} + """, + }) + .ToList(); + + var issues = CategoryHelpers.HreflangIssues(rows); + + Assert.Equal(3, issues.Count(i => + i.Message.StartsWith("Hreflang cluster missing", StringComparison.Ordinal))); + } +} diff --git a/services/ReportService/tests/ReportService.Tests/CtrCurveTests.cs b/services/ReportService/tests/ReportService.Tests/CtrCurveTests.cs new file mode 100644 index 0000000..5ae63fd --- /dev/null +++ b/services/ReportService/tests/ReportService.Tests/CtrCurveTests.cs @@ -0,0 +1,22 @@ +using ReportService.Application.Build; + +namespace ReportService.Tests; + +public sealed class CtrCurveTests +{ + [Fact] + public void IndustryCtrPercent_is_continuous_near_position_four() + { + var atThreeFive = CtrCurve.IndustryCtrPercent(3.5); + var atFour = CtrCurve.IndustryCtrPercent(4.0); + + Assert.True(Math.Abs(atThreeFive - atFour) < 5.0); + } + + [Fact] + public void IndustryCtrFraction_matches_position_slot() + { + Assert.Equal(0.278, CtrCurve.IndustryCtrFraction(1.0), 3); + Assert.Equal(0.008, CtrCurve.IndustryCtrFraction(15.0), 3); + } +} diff --git a/services/ReportService/tests/ReportService.Tests/KeywordOpportunitiesBuilderTests.cs b/services/ReportService/tests/ReportService.Tests/KeywordOpportunitiesBuilderTests.cs index db1cf64..bfd6a6d 100644 --- a/services/ReportService/tests/ReportService.Tests/KeywordOpportunitiesBuilderTests.cs +++ b/services/ReportService/tests/ReportService.Tests/KeywordOpportunitiesBuilderTests.cs @@ -62,6 +62,62 @@ public void Build_produces_quick_wins_and_clusters_for_success_rows() Assert.Contains(quickWins, item => item["keyword"]?.ToString()?.Contains("seo", StringComparison.Ordinal) == true); } + [Fact] + public void Build_with_gsc_varies_difficulty_and_ctr() + { + var rows = Enumerable.Range(0, 100) + .Select(i => new CrawlRow + { + Url = $"https://example.com/page-{i}", + Status = "200", + Title = "seo guide for beginners", + H1 = "seo guide", + }) + .ToList(); + + var google = new Dictionary + { + ["gsc"] = new Dictionary + { + ["top_queries"] = new List + { + new Dictionary { ["query"] = "seo guide", ["position"] = 8.0 }, + }, + }, + }; + + var result = KeywordOpportunitiesBuilder.Build(rows, new Dictionary(), google); + var quickWins = Assert.IsType>>(result["quick_wins"]); + + Assert.NotEmpty(quickWins); + var seoGuide = quickWins.FirstOrDefault(k => k["keyword"]?.ToString() == "seo guide"); + Assert.NotNull(seoGuide); + Assert.NotEqual(50.0, Convert.ToDouble(seoGuide["difficulty"])); + Assert.NotEqual(0.1, Convert.ToDouble(seoGuide["ctr_est"])); + Assert.Equal(8.0, Convert.ToDouble(seoGuide["current_rank"])); + } + + [Fact] + public void Build_high_value_requires_volume_at_least_half() + { + var rows = Enumerable.Range(0, 10) + .Select(i => new CrawlRow + { + Url = $"https://example.com/page-{i}", + Status = "200", + Title = "sharedkeyword page title here for length", + H1 = "sharedkeyword", + }) + .ToList(); + + var result = KeywordOpportunitiesBuilder.Build(rows, new Dictionary()); + var highValue = Assert.IsType>>(result["high_value"]); + + Assert.Contains(highValue, item => + item["keyword"]?.ToString() == "sharedkeyword" + && Convert.ToDouble(item["volume"]) >= 0.5); + } + [Fact] public void Build_filters_junk_heading_tokens() { diff --git a/services/ReportService/tests/ReportService.Tests/NativeReportPayloadAssemblerTests.cs b/services/ReportService/tests/ReportService.Tests/NativeReportPayloadAssemblerTests.cs index 2e1968c..b4b4f60 100644 --- a/services/ReportService/tests/ReportService.Tests/NativeReportPayloadAssemblerTests.cs +++ b/services/ReportService/tests/ReportService.Tests/NativeReportPayloadAssemblerTests.cs @@ -51,6 +51,8 @@ public void AssembleCore_includes_categories_and_native_build_meta() Assert.Equal("Example", payload["site_name"]); Assert.Equal(5, payload["summary"] is Dictionary s ? s["total_urls"] : null); + Assert.Equal(90, (payload["summary"] as Dictionary)!["site_health_score"]); + Assert.Equal(90, payload["site_health_score"]); var cats = Assert.IsType>>(payload["categories"]); Assert.Equal(2, cats.Count); diff --git a/services/ReportService/tests/ReportService.Tests/SearchPerformanceCategoryBuilderTests.cs b/services/ReportService/tests/ReportService.Tests/SearchPerformanceCategoryBuilderTests.cs index 2f8f829..e1e6dbd 100644 --- a/services/ReportService/tests/ReportService.Tests/SearchPerformanceCategoryBuilderTests.cs +++ b/services/ReportService/tests/ReportService.Tests/SearchPerformanceCategoryBuilderTests.cs @@ -19,6 +19,39 @@ public void Build_returns_null_without_gsc_impressions() Assert.Null(SearchPerformanceCategoryBuilder.Build(google)); } + [Fact] + public void Build_scores_page_one_high_opportunity_queries() + { + var google = new Dictionary + { + ["gsc"] = new Dictionary + { + ["summary"] = new Dictionary + { + ["impressions"] = 1000, + ["position"] = 5.0, + ["ctr"] = 4.0, + }, + ["top_queries"] = new List + { + new Dictionary + { + ["query"] = "widgets", + ["position"] = 8.0, + ["impressions"] = 120, + ["clicks"] = 2, + }, + }, + ["daily"] = new List(), + }, + }; + + var category = SearchPerformanceCategoryBuilder.Build(google); + + Assert.NotNull(category); + Assert.Contains(category!.Issues, i => i.Message.Contains("positions 4–10", StringComparison.Ordinal)); + } + [Fact] public void Build_scores_page_two_position_issue() { @@ -80,4 +113,34 @@ [new CategoryIssue("Missing title", "https://example.com/about", "High", "Fix ti Assert.Equal(100, issue.GscImpressions); Assert.True(issue.ImpactScore > 100); } + + [Fact] + public void Enrich_does_not_match_partial_ga4_path_suffix() + { + var google = new Dictionary + { + ["ga4"] = new Dictionary + { + ["top_pages"] = new List + { + new Dictionary + { + ["path"] = "/blog", + ["sessions"] = 500, + }, + }, + }, + }; + var categories = new List + { + new("technical_seo", "Technical SEO", 90, + [new CategoryIssue("Issue", "https://example.com/news-blog", "High", "Fix")], + []), + }; + + var enriched = IssueImpactEnricher.Enrich(categories, google); + var issue = enriched[0].Issues[0]; + + Assert.Equal(0, issue.Ga4Sessions); + } } diff --git a/services/ReportService/tests/ReportService.Tests/SeoSummaryBuilderTests.cs b/services/ReportService/tests/ReportService.Tests/SeoSummaryBuilderTests.cs index 9354092..4e562ce 100644 --- a/services/ReportService/tests/ReportService.Tests/SeoSummaryBuilderTests.cs +++ b/services/ReportService/tests/ReportService.Tests/SeoSummaryBuilderTests.cs @@ -1,4 +1,3 @@ -using System.Text.Json; using ReportService.Application.Build; using ReportService.Application.Repositories; @@ -6,97 +5,70 @@ namespace ReportService.Tests; public sealed class SeoSummaryBuilderTests { - private static List LoadFixtureRows() + [Fact] + public void Compute_excludes_4xx_from_seo_health_counts() { - var root = FindRepoRoot(); - var json = File.ReadAllText(Path.Combine(root, "tests/fixtures/report/minimal_crawl.json")); - using var doc = JsonDocument.Parse(json); - var rows = new List(); - foreach (var el in doc.RootElement.EnumerateArray()) + var rows = new List { - rows.Add(new CrawlRow + new() { - Url = el.GetProperty("url").GetString()!.Trim().TrimEnd('/'), - Status = el.TryGetProperty("status", out var st) ? st.GetString() : null, - Title = el.TryGetProperty("title", out var title) ? title.GetString() : null, - MetaDescription = el.TryGetProperty("meta_description", out var md) ? md.GetString() : null, - MetaDescriptionLen = el.TryGetProperty("meta_description", out var mdl) - ? (mdl.GetString() ?? "").Length - : null, - H1 = el.TryGetProperty("h1", out var h1) ? h1.GetString() : null, - H1Count = el.TryGetProperty("h1", out var h1c) - ? string.IsNullOrWhiteSpace(h1c.GetString()) ? 0 : 1 - : null, - WordCount = el.TryGetProperty("word_count", out var wc) && wc.TryGetInt32(out var wci) ? wci : null, - ContentLength = el.TryGetProperty("word_count", out var cl) && cl.TryGetInt32(out var cli) ? cli * 5 : null, - Outlinks = 0, - PageAnalysisJson = el.TryGetProperty("page_analysis", out var pa) ? pa.GetString() : null, - }); - } - - return rows; - } + Url = "https://example.com/ok", + Status = "200", + Title = "Good title with enough length here", + MetaDescriptionLen = 100, + H1Count = 1, + WordCount = 500, + }, + new() + { + Url = "https://example.com/missing", + Status = "404", + Title = "", + H1Count = 0, + ContentLength = 50, + }, + }; - [Fact] - public void Compute_counts_urls_and_redirects() - { - var rows = LoadFixtureRows(); var result = SeoSummaryBuilder.Compute(rows); - Assert.Equal(5, result.Summary["total_urls"]); - Assert.Equal(4, result.Summary["count_2xx"]); - Assert.Equal(1, result.Summary["count_3xx"]); - Assert.Single(result.Issues["redirects"]); - } - - [Fact] - public void BuildHreflangSummary_counts_hreflang_pages() - { - var rows = LoadFixtureRows(); - var summary = ReportMetadataBuilder.BuildHreflangSummary(rows); - - Assert.Equal(4, summary["pages_200"]); - Assert.Equal(1, (int)summary["pages_with_hreflang_links"]!); + Assert.Equal(0, result.SeoHealth["missing_title"]); + Assert.Equal(0, result.SeoHealth["h1_zero"]); + Assert.Equal(0, result.SeoHealth["thin_content"]); + Assert.Empty(result.Issues["seo"]); } [Fact] - public void BuildUrlFingerprints_is_stable() + public void Compute_flags_thin_content_by_word_count() { - var rows = LoadFixtureRows(); - var fps = ReportMetadataBuilder.BuildUrlFingerprints(rows); - - Assert.Equal(5, fps.Count); - Assert.All(fps, fp => + var rows = new List { - Assert.False(string.IsNullOrWhiteSpace(fp["content_fingerprint"]?.ToString())); - Assert.False(string.IsNullOrWhiteSpace(fp["structure_fingerprint"]?.ToString())); - }); - } + new() + { + Url = "https://example.com/thin", + Status = "200", + Title = "Title with enough characters for SEO checks", + MetaDescriptionLen = 100, + H1Count = 1, + WordCount = 150, + }, + }; - [Fact] - public void ValidateUrlCounts_flags_mismatch() - { - using var doc = JsonDocument.Parse( - """{"links":[{"url":"a"}],"summary":{"total_urls":2},"report_meta":{"crawl_scope":{"pages_crawled":3}}}"""); - var warnings = ReportNativeValidator.ValidateUrlCounts(doc.RootElement, 5); + var result = SeoSummaryBuilder.Compute(rows); - Assert.Single(warnings); - Assert.Contains("mismatch", warnings[0], StringComparison.OrdinalIgnoreCase); + Assert.Equal(1, result.SeoHealth["thin_content"]); } - private static string FindRepoRoot() + [Fact] + public void Compute_crawl_time_is_duration_not_first_timestamp() { - var dir = AppContext.BaseDirectory; - while (!string.IsNullOrEmpty(dir)) + var rows = new List { - if (File.Exists(Path.Combine(dir, "tests/fixtures/report/minimal_crawl.json"))) - { - return dir; - } + new() { Url = "https://example.com/a", Status = "200", CrawlTimeS = 0 }, + new() { Url = "https://example.com/b", Status = "200", CrawlTimeS = 42.5 }, + }; - dir = Directory.GetParent(dir)?.FullName ?? ""; - } + var result = SeoSummaryBuilder.Compute(rows); - throw new InvalidOperationException("Could not locate repo root for fixtures."); + Assert.Equal(42.5, result.Summary["crawl_time_s"]); } } diff --git a/services/ReportService/tests/ReportService.Tests/SiteHealthScoreBuilderTests.cs b/services/ReportService/tests/ReportService.Tests/SiteHealthScoreBuilderTests.cs new file mode 100644 index 0000000..105f2ee --- /dev/null +++ b/services/ReportService/tests/ReportService.Tests/SiteHealthScoreBuilderTests.cs @@ -0,0 +1,42 @@ +using ReportService.Application.Build; + +namespace ReportService.Tests; + +public sealed class SiteHealthScoreBuilderTests +{ + [Fact] + public void Compute_excludes_search_performance_and_intelligence() + { + var categories = new List + { + new("technical_seo", "Technical SEO", 80, [], []), + new("link_health", "Link Health", 80, [], []), + new("performance", "Performance", 80, [], []), + new("security", "Security", 80, [], []), + new("core_web_vitals", "Core Web Vitals", 80, [], []), + new("mobile", "Mobile", 80, [], []), + new("html_accessibility", "Accessibility", 80, [], []), + new("search_performance", "Search performance", 20, [], []), + new("intelligence", "Intelligence", 50, [], []), + }; + + var score = SiteHealthScoreBuilder.Compute(categories); + + Assert.Equal(80, score); + } + + [Fact] + public void Compute_normalizes_when_categories_missing() + { + var categories = new List + { + new("technical_seo", "Technical SEO", 100, [], []), + new("link_health", "Link Health", 60, [], []), + }; + + var score = SiteHealthScoreBuilder.Compute(categories); + + // (100*0.25 + 60*0.20) / (0.25+0.20) = 82.22 -> 82 + Assert.Equal(82, score); + } +} diff --git a/services/ReportService/tests/ReportService.Tests/TechnicalSeoCategoryBuilderTests.cs b/services/ReportService/tests/ReportService.Tests/TechnicalSeoCategoryBuilderTests.cs index edea392..f1664b2 100644 --- a/services/ReportService/tests/ReportService.Tests/TechnicalSeoCategoryBuilderTests.cs +++ b/services/ReportService/tests/ReportService.Tests/TechnicalSeoCategoryBuilderTests.cs @@ -64,6 +64,24 @@ public void Build_site_level_issues_without_success_rows() Assert.Contains(cat.Issues, i => i.Message.Contains("robots.txt", StringComparison.OrdinalIgnoreCase)); } + [Fact] + public void Build_reports_multiple_missing_canonical_urls() + { + var rows = Enumerable.Range(0, 3) + .Select(i => new CrawlRow + { + Url = $"https://example.com/{i}", + Status = "200", + CanonicalUrl = "", + }) + .ToList(); + + var cat = TechnicalSeoCategoryBuilder.Build(rows, new Dictionary()); + var missing = cat.Issues.Where(i => i.Message.Contains("Missing canonical", StringComparison.OrdinalIgnoreCase)).ToList(); + + Assert.Equal(3, missing.Count); + } + [Fact] public void Build_missing_canonical_and_duplicate_title_meta() { diff --git a/src/website_profiling/integrations/google/gsc.py b/src/website_profiling/integrations/google/gsc.py index 5d2d1e0..4d2d7f5 100644 --- a/src/website_profiling/integrations/google/gsc.py +++ b/src/website_profiling/integrations/google/gsc.py @@ -142,8 +142,11 @@ def _to_daily_record(row: dict) -> dict: round(total_clicks / total_impressions * 100, 2) if total_impressions else 0.0 ) avg_position = ( - round(sum(r["position"] for r in all_pages) / len(all_pages), 1) - if all_pages + round( + sum(r["position"] * r["impressions"] for r in all_pages) / total_impressions, + 1, + ) + if total_impressions else 0.0 ) diff --git a/src/website_profiling/reporting/content_analytics.py b/src/website_profiling/reporting/content_analytics.py index a24466f..5f980af 100644 --- a/src/website_profiling/reporting/content_analytics.py +++ b/src/website_profiling/reporting/content_analytics.py @@ -57,10 +57,13 @@ def _build_content_analytics(df: pd.DataFrame) -> dict: if "content_html_ratio" in success_df.columns: cr = pd.to_numeric(success_df["content_html_ratio"], errors="coerce").fillna(0) - cr_bins = [(0, 10), (10.01, 20), (20.01, 40), (40.01, 100)] + cr_bins = [(0, 10), (10, 20), (20, 40), (40, 100)] cr_labels = ["<10%", "10-20%", "20-40%", ">40%"] result["content_ratio_distribution"] = { - lbl: int(((cr >= lo) & (cr <= hi)).sum()) for (lo, hi), lbl in zip(cr_bins, cr_labels) + lbl: int( + ((cr >= lo) & (cr < hi if hi < 100 else cr <= hi)).sum() + ) + for (lo, hi), lbl in zip(cr_bins, cr_labels) } if "top_keywords" in success_df.columns: diff --git a/tests/integrations/test_gsc_summary.py b/tests/integrations/test_gsc_summary.py new file mode 100644 index 0000000..4334db5 --- /dev/null +++ b/tests/integrations/test_gsc_summary.py @@ -0,0 +1,23 @@ +"""GSC summary aggregation tests.""" + +from __future__ import annotations + + +def _weighted_avg_position(pages: list[dict]) -> float: + total_impr = sum(r["impressions"] for r in pages) + if not total_impr: + return 0.0 + return round( + sum(r["position"] * r["impressions"] for r in pages) / total_impr, + 1, + ) + + +def test_avg_position_is_impression_weighted() -> None: + pages = [ + {"position": 1, "impressions": 10}, + {"position": 15, "impressions": 100_000}, + ] + assert _weighted_avg_position(pages) == 15.0 + unweighted = round(sum(r["position"] for r in pages) / len(pages), 1) + assert unweighted == 8.0 diff --git a/tests/reporting/test_content_analytics_bins.py b/tests/reporting/test_content_analytics_bins.py new file mode 100644 index 0000000..8148016 --- /dev/null +++ b/tests/reporting/test_content_analytics_bins.py @@ -0,0 +1,26 @@ +"""Content ratio distribution bin contiguity.""" + +from __future__ import annotations + +import pandas as pd + +from website_profiling.reporting.content_analytics import _build_content_analytics + + +def test_content_ratio_distribution_has_no_bin_gaps() -> None: + df = pd.DataFrame( + { + "status": ["200"] * 4, + "word_count": [100, 200, 300, 400], + "content_html_ratio": [10.005, 10.0, 19.99, 40.0], + } + ) + + out = _build_content_analytics(df) + dist = out["content_ratio_distribution"] + + assert sum(dist.values()) == 4 + assert dist["<10%"] == 0 + assert dist["10-20%"] == 2 + assert dist["20-40%"] == 1 + assert dist[">40%"] == 1 diff --git a/web/src/_linkutils_extract.json b/web/src/_linkutils_extract.json index 6c732f2..af967f5 100644 --- a/web/src/_linkutils_extract.json +++ b/web/src/_linkutils_extract.json @@ -24,7 +24,7 @@ "missing_meta_desc": "Add a meta description (70–160 chars).", "meta_desc_short": "Aim for 70–160 characters.", "meta_desc_long": "Shorten to 70–160 characters.", - "thin_content": "Expand content to at least 300 characters." + "thin_content": "Expand content to at least 200 words." }, "seoIssueRecommendations": { "missing_title": "Add a unique title (30–60 chars).", @@ -34,6 +34,6 @@ "meta_desc_long": "Shorten to 70–160 characters.", "h1_missing": "Add exactly one H1 per page.", "h1_multi": "Use a single H1 per page.", - "thin_content": "Expand content to at least 300 characters." + "thin_content": "Expand content to at least 200 words." } } \ No newline at end of file diff --git a/web/src/lib/dashboard/engine/datasets.ts b/web/src/lib/dashboard/engine/datasets.ts index 98afe2a..3623b4a 100644 --- a/web/src/lib/dashboard/engine/datasets.ts +++ b/web/src/lib/dashboard/engine/datasets.ts @@ -53,7 +53,11 @@ export const DATASETS: DatasetDef[] = [ ...flatPrefix('ga4', d.google?.ga4?.summary as Record | undefined), ...flatPrefix('lh', d.lighthouse_summary?.median_metrics as Record | undefined), ...flatPrefix('social', d.social_coverage as Record | undefined), - health_score: d.portfolio_benchmark?.property_health_score ?? null, + health_score: + d.summary?.site_health_score + ?? d.site_health_score + ?? d.portfolio_benchmark?.property_health_score + ?? null, }, ], fields: [ diff --git a/web/src/strings.json b/web/src/strings.json index 0332b6e..220d969 100644 --- a/web/src/strings.json +++ b/web/src/strings.json @@ -1599,7 +1599,7 @@ "missing_meta_desc": "Add a meta description (70–160 chars).", "meta_desc_short": "Aim for 70–160 characters.", "meta_desc_long": "Shorten to 70–160 characters.", - "thin_content": "Expand content to at least 300 characters." + "thin_content": "Expand content to at least 200 words." }, "seoIssueRecommendations": { "missing_title": "Add a unique title (30–60 chars).", @@ -1609,7 +1609,7 @@ "meta_desc_long": "Shorten to 70–160 characters.", "h1_missing": "Add exactly one H1 per page.", "h1_multi": "Use a single H1 per page.", - "thin_content": "Expand content to at least 300 characters." + "thin_content": "Expand content to at least 200 words." } }, "views": { diff --git a/web/src/types/report.ts b/web/src/types/report.ts index 6c44559..d33ed28 100644 --- a/web/src/types/report.ts +++ b/web/src/types/report.ts @@ -637,6 +637,7 @@ export interface ReportPayload { link_rel_summary?: LinkRelSummary; inlink_anchor_matrix?: InlinkAnchorRow[]; portfolio_benchmark?: PortfolioBenchmark; + site_health_score?: number | null; rich_results_validation?: RichResultsValidationRow[]; rich_results_meta?: RichResultsMeta; competitor_keyword_gap?: CompetitorKeywordGapRow[]; @@ -722,6 +723,8 @@ export interface ReportSummary { success_rate?: number; crawl_time_s?: number; avg_outlinks?: number; + site_health_score?: number | null; + category_scores?: Record; } export interface ReportCategory { From e6ab78b6142c8e43642cf02d859b88356d7a0583 Mon Sep 17 00:00:00 2001 From: PrashantUnity Date: Sat, 27 Jun 2026 17:13:52 +0530 Subject: [PATCH 3/8] Micro Slop 2 --- .../overview/OverviewSummaryTab.tsx | 9 +--- web/src/lib/dashboard/engine/datasets.test.ts | 5 ++- web/src/lib/homePortfolio.ts | 12 +---- web/src/lib/reportCompare.ts | 17 +++---- web/src/lib/siteHealthScore.test.ts | 38 ++++++++++++++++ web/src/lib/siteHealthScore.ts | 44 +++++++++++++++++++ web/src/strings.json | 9 ++-- web/src/types/report.ts | 1 + web/src/views/Content.tsx | 6 +-- web/src/views/ContentAnalytics.tsx | 10 ++--- web/src/views/Links.tsx | 5 ++- 11 files changed, 112 insertions(+), 44 deletions(-) create mode 100644 web/src/lib/siteHealthScore.test.ts create mode 100644 web/src/lib/siteHealthScore.ts diff --git a/web/src/components/overview/OverviewSummaryTab.tsx b/web/src/components/overview/OverviewSummaryTab.tsx index 8a84dd3..42b980e 100644 --- a/web/src/components/overview/OverviewSummaryTab.tsx +++ b/web/src/components/overview/OverviewSummaryTab.tsx @@ -9,6 +9,7 @@ import { FileDown, } from 'lucide-react'; import type { ReportPayload } from '@/types'; +import { siteHealthScoreFromPayload } from '@/lib/siteHealthScore'; import type { DataSourceId } from '@/lib/dataProvenance'; import { strings, format } from '@/lib/strings'; import { metricHelpHint } from '@/lib/metricHelp'; @@ -49,13 +50,7 @@ export function OverviewSummaryTab({ ); const { currentHealth, topIssues } = useMemo(() => { - const scores = (data.categories || []) - .map((c) => Number(c?.score)) - .filter((n) => Number.isFinite(n)); - const health = - scores.length > 0 - ? Math.round(scores.reduce((a, b) => a + b, 0) / scores.length) - : null; + const health = siteHealthScoreFromPayload(data); const exec = (data.executive_summary?.top_issues || []).slice(0, 5); const fallback = (data.categories || []) .flatMap((cat) => diff --git a/web/src/lib/dashboard/engine/datasets.test.ts b/web/src/lib/dashboard/engine/datasets.test.ts index 50453b8..2d0300d 100644 --- a/web/src/lib/dashboard/engine/datasets.test.ts +++ b/web/src/lib/dashboard/engine/datasets.test.ts @@ -7,10 +7,11 @@ import type { ReportPayload } from '@/types/report'; /** A payload that touches every section a dataset reads from. (Cast: real link * rows carry more fields than the partial ReportLink interface declares.) */ const PAYLOAD = { - summary: { total_urls: 100, count_2xx: 90, count_4xx: 8, count_5xx: 2, success_rate: 90, avg_outlinks: 5 }, + summary: { total_urls: 100, count_2xx: 90, count_4xx: 8, count_5xx: 2, success_rate: 90, avg_outlinks: 5, site_health_score: 72 }, seo_health: { missing_title: 3, missing_meta_desc: 4, thin_content: 6 }, social_coverage: { og_coverage_pct: 65 }, portfolio_benchmark: { property_health_score: 78 }, + site_health_score: 72, status_counts: { '200': 90, '404': 8, '500': 2 }, categories: [ { id: 'seo', name: 'Technical SEO', score: 80, issues: [{ message: 'm1', priority: 'High', impact_score: 5 }] }, @@ -93,7 +94,7 @@ describe('dataset registry integrity', () => { expect(datasetById.get('issues')!.accessor(PAYLOAD)).toHaveLength(3); expect(datasetById.get('status_counts')!.accessor(PAYLOAD)).toHaveLength(3); expect(datasetById.get('mime_types')!.accessor(PAYLOAD)).toHaveLength(2); - expect(datasetById.get('summary')!.accessor(PAYLOAD)[0].health_score).toBe(78); + expect(datasetById.get('summary')!.accessor(PAYLOAD)[0].health_score).toBe(72); // links accessor derives host/path expect(datasetById.get('links')!.accessor(PAYLOAD)[0].host).toBe('x.com'); }); diff --git a/web/src/lib/homePortfolio.ts b/web/src/lib/homePortfolio.ts index d9d051e..60ce953 100644 --- a/web/src/lib/homePortfolio.ts +++ b/web/src/lib/homePortfolio.ts @@ -2,6 +2,7 @@ import { crawledUrlCount } from './crawlCounts'; import { DATA_SOURCE_IDS, type DataSourceId } from './dataProvenance'; import { canonicalDomainFromPayload, extractHostname, slugifyDomain } from './domainSlug'; import { titleCoveragePct } from './portfolioCrawlHistory'; +import { siteHealthScoreFromPayload } from './siteHealthScore'; import type { CrawlRunSummary, PortfolioCategorySnapshot, @@ -112,15 +113,6 @@ function lighthouseScoresFromPayload(payload: ReportPayload): { perf: number | n return { perf, seo }; } -function scoreFromCategories(categories: Array<{ score?: number }> = []): number | null { - const numeric = (categories || []) - .map((c) => Number(c?.score)) - .filter((n) => Number.isFinite(n)); - if (!numeric.length) return null; - const avg = numeric.reduce((a, b) => a + b, 0) / numeric.length; - return Math.round(avg); -} - function toLocalDateTime(value: string | null | undefined): string { if (!value) return ''; const d = new Date(value); @@ -220,7 +212,7 @@ export async function computeDomainGroups( }; const urlCount = crawledUrlCount(payload); const successPct = urlCount > 0 ? Math.round((statusCounts.s2xx / urlCount) * 100) : 0; - const healthScore = scoreFromCategories(payload?.categories) ?? 0; + const healthScore = siteHealthScoreFromPayload(payload ?? {}) ?? 0; const runCreatedAt = runId != null ? runCreatedAtByRunId.get(runId) : ''; const lastCrawl = toLocalDateTime( runCreatedAt || payload?.crawl_run_created_at || payload?.report_generated_at || r.generated_at, diff --git a/web/src/lib/reportCompare.ts b/web/src/lib/reportCompare.ts index f397abc..077c7fb 100644 --- a/web/src/lib/reportCompare.ts +++ b/web/src/lib/reportCompare.ts @@ -11,6 +11,9 @@ import { type CompareExtrasLabels, type ReportCompareExtras, } from './reportCompareExtras'; +import { siteHealthScoreFromPayload } from './siteHealthScore'; + +export type { CompareExtrasLabels }; export type { IssueDeltaRow, @@ -79,16 +82,6 @@ export interface ReportCompareSummary { extras: ReportCompareExtras; } -export type { CompareExtrasLabels }; - -function scoreFromCategories(categories: ReportCategory[] = []): number | null { - const numeric = categories - .map((c) => Number(c?.score)) - .filter((n) => Number.isFinite(n)); - if (!numeric.length) return null; - return Math.round(numeric.reduce((a, b) => a + b, 0) / numeric.length); -} - function countCategoryIssues(categories: ReportCategory[] = []): number { return categories.reduce((n, c) => n + (c.issues?.length ?? 0), 0); } @@ -275,8 +268,8 @@ export function buildReportCompareSummary( deltaRow( 'health_score', labels.healthScore, - scoreFromCategories(current.categories ?? []), - scoreFromCategories(baseline.categories ?? []), + siteHealthScoreFromPayload(current), + siteHealthScoreFromPayload(baseline), true, 'score', ), diff --git a/web/src/lib/siteHealthScore.test.ts b/web/src/lib/siteHealthScore.test.ts new file mode 100644 index 0000000..e650679 --- /dev/null +++ b/web/src/lib/siteHealthScore.test.ts @@ -0,0 +1,38 @@ +import { describe, it, expect } from 'vitest'; +import { siteHealthScoreFromCategories, siteHealthScoreFromPayload } from './siteHealthScore'; +import type { ReportCategory } from '@/types/report'; + +const weightedCategories: ReportCategory[] = [ + { id: 'technical_seo', name: 'Technical SEO', score: 80, issues: [] }, + { id: 'link_health', name: 'Link Health', score: 60, issues: [] }, + { id: 'performance', name: 'Performance', score: 70, issues: [] }, + { id: 'security', name: 'Security', score: 90, issues: [] }, + { id: 'core_web_vitals', name: 'CWV', score: 50, issues: [] }, + { id: 'mobile', name: 'Mobile', score: 40, issues: [] }, + { id: 'html_accessibility', name: 'A11y', score: 100, issues: [] }, + { id: 'search_performance', name: 'Search', score: 10, issues: [] }, + { id: 'intelligence', name: 'Intel', score: 0, issues: [] }, +]; + +describe('siteHealthScoreFromCategories', () => { + it('weights fixable categories and excludes search_performance and intelligence', () => { + // 80*0.25 + 60*0.2 + 70*0.15 + 90*0.15 + 50*0.1 + 40*0.1 + 100*0.05 = 69.5 → 70 + expect(siteHealthScoreFromCategories(weightedCategories)).toBe(70); + }); +}); + +describe('siteHealthScoreFromPayload', () => { + it('prefers summary.site_health_score over portfolio benchmark', () => { + expect( + siteHealthScoreFromPayload({ + summary: { site_health_score: 72 }, + site_health_score: 65, + categories: weightedCategories, + }), + ).toBe(72); + }); + + it('falls back to weighted category score when payload field missing', () => { + expect(siteHealthScoreFromPayload({ categories: weightedCategories })).toBe(70); + }); +}); diff --git a/web/src/lib/siteHealthScore.ts b/web/src/lib/siteHealthScore.ts new file mode 100644 index 0000000..453bd8f --- /dev/null +++ b/web/src/lib/siteHealthScore.ts @@ -0,0 +1,44 @@ +import type { ReportCategory, ReportPayload } from '@/types/report'; + +/** Mirrors ReportService SiteHealthScoreBuilder — fixable categories only. */ +const WEIGHTS: Record = { + technical_seo: 0.25, + link_health: 0.2, + performance: 0.15, + security: 0.15, + core_web_vitals: 0.1, + mobile: 0.1, + html_accessibility: 0.05, +}; + +const EXCLUDED = new Set(['search_performance', 'intelligence']); + +export function siteHealthScoreFromCategories(categories: ReportCategory[] = []): number | null { + let weightedSum = 0; + let weightTotal = 0; + + for (const [id, weight] of Object.entries(WEIGHTS)) { + const cat = categories.find((c) => c.id === id); + const score = Number(cat?.score); + if (!Number.isFinite(score)) continue; + weightedSum += score * weight; + weightTotal += weight; + } + + return weightTotal > 0 ? Math.round(weightedSum / weightTotal) : null; +} + +/** Prefer native payload field; fall back to client weighted score for older reports. */ +export function siteHealthScoreFromPayload( + payload: Pick, +): number | null { + const fromPayload = payload.summary?.site_health_score ?? payload.site_health_score; + if (typeof fromPayload === 'number' && Number.isFinite(fromPayload)) { + return Math.round(fromPayload); + } + + const categories = (payload.categories ?? []).filter( + (c) => c.id && !EXCLUDED.has(String(c.id)), + ); + return siteHealthScoreFromCategories(categories); +} diff --git a/web/src/strings.json b/web/src/strings.json index 220d969..2c75ebf 100644 --- a/web/src/strings.json +++ b/web/src/strings.json @@ -344,7 +344,7 @@ "body": "How pages are distributed by crawl depth — hops from the start URL via internal links." }, "thinSmallBody": { - "body": "Pages flagged as thin by visible character count thresholds (separate from the under-300-word list)." + "body": "Pages flagged as thin in SEO health (under 200 words)." }, "wordCountDist": { "body": "Pages grouped by visible word-count bands from extracted crawl text." @@ -377,7 +377,7 @@ "body": "Share of pages whose title, meta description, and H1 fall in recommended length or count ranges." }, "thinSignals": { - "body": "Comparison of thin-content signals: under 300 words vs small visible body character count." + "body": "Comparison of thin-content signals: analytics list (under 300 words) vs SEO health flag (under 200 words)." }, "h1Dist": { "body": "Pages grouped by H1 heading count — SEO best practice is exactly one H1 per page." @@ -2646,7 +2646,7 @@ { "key": "thin_content", "label": "Thin Content", - "guidance": "Pages with very little text (under ~300 words) are often low-value to search engines. Expand or consolidate thin pages." + "guidance": "Pages with very little text (under ~200 words) are often low-value to search engines. Expand or consolidate thin pages." } ], "dupClusters": "Near-duplicate clusters", @@ -2666,6 +2666,7 @@ "tableLength": "Length", "tableH1Count": "H1 Count", "tableChars": "Chars", + "tableWords": "Words", "urlOne": "URL", "urlMany": "URLs" }, @@ -4337,7 +4338,7 @@ "stackedNeeds": "Needs attention", "thinSignals": "Thin content signals", "thinUnder300": "Under 300 words", - "thinSmallBody": "Small HTML body", + "thinSmallBody": "SEO flagged (<200 words)", "thinMatchSearch": "{shown} of {total} thin page{plural} match search", "thinHaveLittle": "{count} page{plural} have very little content", "noThinSearch": "No thin pages match your search.", diff --git a/web/src/types/report.ts b/web/src/types/report.ts index d33ed28..21e89d7 100644 --- a/web/src/types/report.ts +++ b/web/src/types/report.ts @@ -65,6 +65,7 @@ export interface ContentUrlEntry { meta_desc_len?: number; h1_count?: number; content_length?: number; + word_count?: number; } export type ContentUrlsMap = Record; diff --git a/web/src/views/Content.tsx b/web/src/views/Content.tsx index dab4339..28eb989 100644 --- a/web/src/views/Content.tsx +++ b/web/src/views/Content.tsx @@ -312,7 +312,7 @@ export default function Content({ searchQuery = '' }: ViewProps) { {vc.tableH1Count} )} {filter === 'thin_content' && ( - {vc.tableChars} + {vc.tableWords} )} @@ -364,7 +364,7 @@ export default function Content({ searchQuery = '' }: ViewProps) { )} {filter === 'thin_content' && ( - {vc.tableChars}: {item.content_length ?? sj.emDash} + {vc.tableWords}: {item.word_count ?? item.content_length ?? sj.emDash} )}

@@ -387,7 +387,7 @@ export default function Content({ searchQuery = '' }: ViewProps) { )} {filter === 'thin_content' && ( - {item.content_length ?? sj.emDash} + {item.word_count ?? item.content_length ?? sj.emDash} )} diff --git a/web/src/views/ContentAnalytics.tsx b/web/src/views/ContentAnalytics.tsx index 24dc015..faa4602 100644 --- a/web/src/views/ContentAnalytics.tsx +++ b/web/src/views/ContentAnalytics.tsx @@ -400,9 +400,9 @@ export default function ContentAnalytics({ searchQuery = '' }: ViewProps) { const h1OkPct = h1Total > 0 ? (100 * (seoHealth.h1_one || 0)) / h1Total : 0; const hasSeoOptimalBar = titleTotal > 0 && metaTotal > 0 && h1Total > 0; - const thinByWords = thinPages.length; - const thinByChars = Number(seoHealth.thin_content) || 0; - const hasThinCompare = thinByWords > 0 || thinByChars > 0; + const thinPagesListed = thinPages.length; + const thinSeoFlagged = Number(seoHealth.thin_content) || 0; + const hasThinCompare = thinPagesListed > 0 || thinSeoFlagged > 0; const titleBucketCounts = [ seoHealth.missing_title || 0, @@ -755,13 +755,13 @@ export default function ContentAnalytics({ searchQuery = '' }: ViewProps) {
diff --git a/web/src/views/Links.tsx b/web/src/views/Links.tsx index 3349568..95ae1c5 100644 --- a/web/src/views/Links.tsx +++ b/web/src/views/Links.tsx @@ -338,7 +338,10 @@ export default function Links({ searchQuery = '' }: ViewProps) { if (entry) { let detail: string | null = null; if (key === 'meta_desc_short' || key === 'meta_desc_long') detail = `${entry.meta_desc_len ?? 0} chars`; - if (key === 'thin_content') detail = `${entry.content_length ?? 0} chars`; + if (key === 'thin_content') { + const words = entry.word_count; + detail = words != null ? `${words} words` : `${entry.content_length ?? 0} chars`; + } if (key === 'multiple_h1') detail = `${entry.h1_count ?? 0} H1s`; contentFlags.push({ type: key, From ecc4d44fb8a429c805d279610c753fc73b5e3b6f Mon Sep 17 00:00:00 2001 From: PrashantUnity Date: Sat, 27 Jun 2026 17:58:50 +0530 Subject: [PATCH 4/8] Ai slop --- requirements.txt | 1 + .../Handlers/Report/ReportToolHandlers.cs | 42 +---- .../Portfolio/PortfolioGrouping.cs | 16 +- .../Repositories/ReportRepository.cs | 17 +- .../Data.Tests/SiteHealthScoreBuilderTests.cs | 46 ++++++ .../Mapping/AuditReportMapper.cs | 15 +- .../CoreWebVitalsCategoryBuilder.cs | 26 ++- .../HtmlAccessibilityCategoryBuilder.cs | 12 +- .../Categories/SecurityCategoryBuilder.cs | 48 ------ .../Build/CategoryHelpers.cs | 65 ++++++++ .../Build/CtrCurve.cs | 18 ++- .../Build/NativeReportBuilder.cs | 10 +- .../Build/NativeReportPayloadAssembler.cs | 5 + .../Build/SiteHealthScoreBuilder.cs | 79 ++-------- .../Integrations/CruxOriginMetricsFetcher.cs | 149 ++++++++++++++++++ .../CategoryBuilderParityTests.cs | 4 +- .../CoreWebVitalsCategoryBuilderTests.cs | 34 ++++ .../CruxOriginMetricsFetcherTests.cs | 32 ++++ .../ReportService.Tests/CtrCurveTests.cs | 8 + .../IndexationCoverageIssuesTests.cs | 30 ++++ .../SecurityCategoryBuilderTests.cs | 29 +++- .../Report/SiteHealthScoreBuilder.cs | 146 +++++++++++++++++ src/website_profiling/db/report_store.py | 9 +- .../integrations/google/keyword_enrich.py | 12 +- src/website_profiling/parsing/tech.py | 5 + src/website_profiling/reporting/builder.py | 13 +- .../builder_sections/content_urls.py | 2 +- .../reporting/categories/_helpers.py | 41 +++-- .../reporting/categories/accessibility.py | 30 ++-- .../reporting/categories/performance.py | 28 +++- .../reporting/categories/security.py | 40 ----- .../reporting/categories/technical_seo.py | 22 ++- .../reporting/seo_summary.py | 64 ++++---- .../reporting/thin_content_helper.py | 41 +++++ src/website_profiling/scoring.py | 51 ++++++ tests/reporting/test_categories_coverage.py | 42 +++-- .../reporting/test_content_analytics_bins.py | 4 +- tests/reporting/test_contrast_issues.py | 9 +- .../test_pipeline_report_pool_unit.py | 17 ++ tests/test_keyword_enrich.py | 9 +- tests/test_scoring.py | 30 +++- .../overview/OverviewSummaryTab.tsx | 2 +- web/src/lib/dashboard/engine/datasets.test.ts | 21 ++- web/src/lib/dashboard/engine/datasets.ts | 7 +- web/src/strings.json | 17 +- web/src/views/ContentAnalytics.tsx | 3 +- web/src/views/TechStack.tsx | 16 +- 47 files changed, 987 insertions(+), 380 deletions(-) create mode 100644 services/Data/tests/Data.Tests/SiteHealthScoreBuilderTests.cs create mode 100644 services/ReportService/src/ReportService.Application/Integrations/CruxOriginMetricsFetcher.cs create mode 100644 services/ReportService/tests/ReportService.Tests/CoreWebVitalsCategoryBuilderTests.cs create mode 100644 services/ReportService/tests/ReportService.Tests/CruxOriginMetricsFetcherTests.cs create mode 100644 services/ReportService/tests/ReportService.Tests/IndexationCoverageIssuesTests.cs create mode 100644 services/Shared/WebsiteProfiling.Contracts/Report/SiteHealthScoreBuilder.cs create mode 100644 src/website_profiling/reporting/thin_content_helper.py diff --git a/requirements.txt b/requirements.txt index aa4ab71..b217d0d 100644 --- a/requirements.txt +++ b/requirements.txt @@ -4,6 +4,7 @@ lxml==6.1.1 pandas==3.0.3 tqdm==4.67.3 networkx==3.6.1 +setuptools>=75.0.0 python-Wappalyzer==0.3.1 # HTML → Markdown extraction (page_markdown package) diff --git a/services/AiService/src/AiService.Tools/Handlers/Report/ReportToolHandlers.cs b/services/AiService/src/AiService.Tools/Handlers/Report/ReportToolHandlers.cs index 37e2c83..027b5ec 100644 --- a/services/AiService/src/AiService.Tools/Handlers/Report/ReportToolHandlers.cs +++ b/services/AiService/src/AiService.Tools/Handlers/Report/ReportToolHandlers.cs @@ -1,7 +1,9 @@ +using System.Text.Json; using System.Text.Json.Nodes; using AiService.Tools.Context; using AiService.Tools.Persistence; +using WebsiteProfiling.Contracts.Report; namespace AiService.Tools.Handlers.Report; /// @@ -363,44 +365,8 @@ private static JsonObject IssueCounts(IReadOnlyList issues) private static int? HealthScore(JsonObject payload) { - if (payload["site_health_score"] is JsonValue topLevel - && topLevel.TryGetValue(out int topScore)) - { - return topScore; - } - - if (payload["summary"] is JsonObject summary - && summary["site_health_score"] is JsonValue summaryScore - && summaryScore.TryGetValue(out int fromSummary)) - { - return fromSummary; - } - - if (payload["categories"] is not JsonArray categories) - { - return null; - } - - var scores = new List(); - foreach (var catNode in categories) - { - if (catNode is not JsonObject cat) - { - continue; - } - - if (cat["score"] is JsonValue scoreValue && scoreValue.TryGetValue(out double score)) - { - scores.Add(score); - } - } - - if (scores.Count == 0) - { - return null; - } - - return (int)Math.Round(scores.Average(), MidpointRounding.AwayFromZero); + var element = JsonSerializer.SerializeToElement(payload); + return SiteHealthScoreBuilder.ResolveFromPayload(element); } private static string CategoryDisplayName(string name) diff --git a/services/Data/src/Data.Application/Portfolio/PortfolioGrouping.cs b/services/Data/src/Data.Application/Portfolio/PortfolioGrouping.cs index 17a648b..c797224 100644 --- a/services/Data/src/Data.Application/Portfolio/PortfolioGrouping.cs +++ b/services/Data/src/Data.Application/Portfolio/PortfolioGrouping.cs @@ -2,6 +2,7 @@ using System.Text.Json.Nodes; using Data.Application.Dto.Portfolio; using Data.Application.Mapping; +using WebsiteProfiling.Contracts.Report; namespace Data.Application.Portfolio; @@ -174,7 +175,7 @@ private static (string BrandKey, PortfolioGroupDto Group, double GeneratedAtMs) var successPct = urlCount > 0 ? (int)Math.Round(statusCounts.S2xx / (double)urlCount * 100, MidpointRounding.ToEven) : 0; - var healthScore = ScoreFromCategories(PortfolioHelpers.GetArrayOrEmpty(payload, "categories")) ?? 0; + var healthScore = SiteHealthScoreBuilder.ResolveFromPayload(payload) ?? 0; var runCreatedAt = runIdInt is not null && maps.RunCreatedAtByRunId.TryGetValue(runIdInt.Value, out var rc) ? rc : ""; @@ -251,19 +252,6 @@ private static (string BrandKey, PortfolioGroupDto Group, double GeneratedAtMs) private static PortfolioIssueCountsDto EmptyIssueCounts() => new(); - private static int? ScoreFromCategories(JsonElement categories) - { - if (categories.ValueKind != JsonValueKind.Array) return null; - var nums = new List(); - foreach (var cat in categories.EnumerateArray()) - { - if (cat.TryGetProperty("score", out var sc) && sc.ValueKind == JsonValueKind.Number) - nums.Add(sc.GetDouble()); - } - if (nums.Count == 0) return null; - return (int)Math.Round(nums.Sum() / nums.Count, MidpointRounding.ToEven); - } - private static (PortfolioIssueCountsDto Counts, int Total) IssueCountsFromPayload(JsonElement payload) { var counts = EmptyIssueCounts(); diff --git a/services/Data/src/Data.Application/Repositories/ReportRepository.cs b/services/Data/src/Data.Application/Repositories/ReportRepository.cs index 6c6218b..abb8df3 100644 --- a/services/Data/src/Data.Application/Repositories/ReportRepository.cs +++ b/services/Data/src/Data.Application/Repositories/ReportRepository.cs @@ -8,6 +8,7 @@ using Data.Domain.Entities; using Microsoft.EntityFrameworkCore; using Microsoft.Extensions.Logging; +using WebsiteProfiling.Contracts.Report; namespace Data.Application.Repositories; @@ -213,7 +214,9 @@ private static AuditHistoryItem MapHistoryItem( CanonicalDomain = row.CanonicalDomain, SiteName = row.SiteName, GeneratedAt = PyIso.Format(row.GeneratedAt), - HealthScore = AvgScore(categories), + HealthScore = data.ValueKind == JsonValueKind.Object + ? SiteHealthScoreBuilder.ResolveFromPayload(data) + : null, CategoryScores = categoryScores, IssueCounts = issueCounts, PerfScore = LhScore(data, "performance_score", "performance"), @@ -222,18 +225,6 @@ private static AuditHistoryItem MapHistoryItem( }; } - // round(sum/len) with banker's rounding — mirrors Python's builtin round(). - private static int? AvgScore(JsonElement categories) - { - if (categories.ValueKind != JsonValueKind.Array) return null; - var nums = new List(); - foreach (var cat in categories.EnumerateArray()) - if (cat.TryGetProperty("score", out var s) && s.ValueKind == JsonValueKind.Number) - nums.Add(s.GetDouble()); - if (nums.Count == 0) return null; - return (int)Math.Round(nums.Sum() / nums.Count, MidpointRounding.ToEven); - } - // Mirrors _lh_scores: try median_metrics.{mmKey} first (non-zero), then category_scores.{csKey}. private static int? LhScore(JsonElement data, string mmKey, string csKey) { diff --git a/services/Data/tests/Data.Tests/SiteHealthScoreBuilderTests.cs b/services/Data/tests/Data.Tests/SiteHealthScoreBuilderTests.cs new file mode 100644 index 0000000..27b3c14 --- /dev/null +++ b/services/Data/tests/Data.Tests/SiteHealthScoreBuilderTests.cs @@ -0,0 +1,46 @@ +using System.Text.Json; +using WebsiteProfiling.Contracts.Report; + +namespace Data.Tests; + +public sealed class SiteHealthScoreBuilderTests +{ + [Fact] + public void ResolveFromPayload_prefers_summary_site_health_score() + { + const string json = """ + { + "summary": { "site_health_score": 72 }, + "site_health_score": 65, + "categories": [{ "id": "technical_seo", "score": 10 }] + } + """; + + using var doc = JsonDocument.Parse(json); + Assert.Equal(72, SiteHealthScoreBuilder.ResolveFromPayload(doc.RootElement)); + } + + [Fact] + public void ComputeFromJsonCategories_uses_weighted_fixable_categories() + { + const string json = """ + { + "categories": [ + { "id": "technical_seo", "score": 80 }, + { "id": "link_health", "score": 60 }, + { "id": "performance", "score": 70 }, + { "id": "security", "score": 90 }, + { "id": "core_web_vitals", "score": 50 }, + { "id": "mobile", "score": 40 }, + { "id": "html_accessibility", "score": 100 }, + { "id": "search_performance", "score": 10 }, + { "id": "intelligence", "score": 0 } + ] + } + """; + + using var doc = JsonDocument.Parse(json); + var categories = doc.RootElement.GetProperty("categories"); + Assert.Equal(70, SiteHealthScoreBuilder.ComputeFromJsonCategories(categories)); + } +} diff --git a/services/FileService/src/FileService.Application/Mapping/AuditReportMapper.cs b/services/FileService/src/FileService.Application/Mapping/AuditReportMapper.cs index d24e742..e581cf0 100644 --- a/services/FileService/src/FileService.Application/Mapping/AuditReportMapper.cs +++ b/services/FileService/src/FileService.Application/Mapping/AuditReportMapper.cs @@ -296,19 +296,8 @@ private static Dictionary CountByPriority(IEnumerable { return (int)Math.Round(overall.GetDouble()); } - if (!payload.TryGetProperty("categories", out var categories) || categories.ValueKind != JsonValueKind.Array) - { - return null; - } - var scores = new List(); - foreach (var cat in categories.EnumerateArray()) - { - if (cat.TryGetProperty("score", out var scoreEl) && scoreEl.ValueKind == JsonValueKind.Number) - { - scores.Add(scoreEl.GetDouble()); - } - } - return scores.Count == 0 ? null : (int)Math.Round(scores.Average()); + + return SiteHealthScoreBuilder.ResolveFromPayload(payload); } private static string ScoreBand(int? score) => score switch diff --git a/services/ReportService/src/ReportService.Application/Build/Categories/CoreWebVitalsCategoryBuilder.cs b/services/ReportService/src/ReportService.Application/Build/Categories/CoreWebVitalsCategoryBuilder.cs index c1ddfef..05e824d 100644 --- a/services/ReportService/src/ReportService.Application/Build/Categories/CoreWebVitalsCategoryBuilder.cs +++ b/services/ReportService/src/ReportService.Application/Build/Categories/CoreWebVitalsCategoryBuilder.cs @@ -59,11 +59,35 @@ public static ReportCategory Build( recommendations.Add("Core Web Vitals measured by Lighthouse; see median_metrics in lighthouse_summary.json."); } + var cruxDeduction = 0; + if (cruxSummary is not null + && GetBool(cruxSummary, "ok") == true + && cruxSummary.TryGetValue("pass", out var passDedObj) + && passDedObj is JsonElement passDed + && passDed.ValueKind == JsonValueKind.Object) + { + foreach (var metric in new[] { "lcp", "inp", "cls" }) + { + if (passDed.TryGetProperty(metric, out var m) && m.ValueKind == JsonValueKind.False) + { + cruxDeduction += 15; + } + } + + cruxDeduction = Math.Min(45, cruxDeduction); + } + + int? score = perfScore; + if (cruxDeduction > 0) + { + score = Math.Max(0, (perfScore ?? 0) - cruxDeduction); + } + var sorted = CategoryHelpers.SortIssues(issues); return new ReportCategory( "core_web_vitals", "Core Web Vitals", - perfScore, + score, sorted, recommendations); } diff --git a/services/ReportService/src/ReportService.Application/Build/Categories/HtmlAccessibilityCategoryBuilder.cs b/services/ReportService/src/ReportService.Application/Build/Categories/HtmlAccessibilityCategoryBuilder.cs index 30f78bb..e1e83f1 100644 --- a/services/ReportService/src/ReportService.Application/Build/Categories/HtmlAccessibilityCategoryBuilder.cs +++ b/services/ReportService/src/ReportService.Application/Build/Categories/HtmlAccessibilityCategoryBuilder.cs @@ -91,7 +91,7 @@ public static ReportCategory Build( if (veryThin > 0) { issues.Add(CategoryHelpers.Issue( - $"{veryThin} page(s) with very thin content (under 100 words).", + $"{veryThin} page(s) with very thin content (under 100 words; SEO thin flag uses 200 words).", priority: "High", recommendation: "Expand thin pages with meaningful content (aim for 300+ words).")); deductions.Add((Math.Min(15, veryThin * 3), true)); @@ -132,11 +132,6 @@ public static ReportCategory Build( } var score = CategoryHelpers.ScoreDeductions(100, deductions); - if (success.Count > 0 && score == 0) - { - score = 5; - } - score = Math.Min(100, Math.Max(0, score)); var sorted = CategoryHelpers.SortIssues(issues); return new ReportCategory( @@ -227,7 +222,10 @@ private static List ContrastIssuesFromSources( url, "Medium", rec ?? "Fix text/background contrast to meet WCAG AA (axe-core).")); - break; + if (issues.Count(i => i.Message.StartsWith("axe:", StringComparison.Ordinal)) >= CategoryHelpers.MaxIssuesPerCheck) + { + break; + } } } catch (JsonException) diff --git a/services/ReportService/src/ReportService.Application/Build/Categories/SecurityCategoryBuilder.cs b/services/ReportService/src/ReportService.Application/Build/Categories/SecurityCategoryBuilder.cs index 088f5ec..fae90ea 100644 --- a/services/ReportService/src/ReportService.Application/Build/Categories/SecurityCategoryBuilder.cs +++ b/services/ReportService/src/ReportService.Application/Build/Categories/SecurityCategoryBuilder.cs @@ -35,54 +35,6 @@ public static ReportCategory Build( deductions.Add((20, true)); } - var success = CategoryHelpers.SuccessRows(rows); - if (success.Count > 0) - { - var missingHsts = success.Count(r => string.IsNullOrWhiteSpace(r.StrictTransportSecurity)); - if (missingHsts >= success.Count * 0.5) - { - issues.Add(CategoryHelpers.Issue( - "Strict-Transport-Security header not set.", - "", - "High", - "Add Strict-Transport-Security to enforce HTTPS.")); - deductions.Add((15, true)); - } - - var missingXcto = success.Count(r => string.IsNullOrWhiteSpace(r.XContentTypeOptions)); - if (missingXcto >= success.Count * 0.5) - { - issues.Add(CategoryHelpers.Issue( - "X-Content-Type-Options header not set.", - "", - "Medium", - "Add X-Content-Type-Options: nosniff.")); - deductions.Add((5, true)); - } - - var missingXfo = success.Count(r => string.IsNullOrWhiteSpace(r.XFrameOptions)); - if (missingXfo >= success.Count * 0.5) - { - issues.Add(CategoryHelpers.Issue( - "X-Frame-Options header not set.", - "", - "Medium", - "Add X-Frame-Options: DENY or SAMEORIGIN.")); - deductions.Add((5, true)); - } - - var mixed = success.Sum(r => r.MixedContentCount ?? 0); - if (mixed > 0 && startUrl.StartsWith("https://", StringComparison.OrdinalIgnoreCase)) - { - issues.Add(CategoryHelpers.Issue( - $"Mixed content: {mixed} HTTP resource(s) on HTTPS pages.", - "", - "High", - "Load all resources over HTTPS to avoid mixed content.")); - deductions.Add((15, true)); - } - } - if (securityFindings is not null) { foreach (var finding in securityFindings) diff --git a/services/ReportService/src/ReportService.Application/Build/CategoryHelpers.cs b/services/ReportService/src/ReportService.Application/Build/CategoryHelpers.cs index cc6d06d..e57f280 100644 --- a/services/ReportService/src/ReportService.Application/Build/CategoryHelpers.cs +++ b/services/ReportService/src/ReportService.Application/Build/CategoryHelpers.cs @@ -255,9 +255,74 @@ public static List IndexationCoverageIssues( } } + var sitemapUrls = ExtractSitemapUrls(indexation); + if (sitemapUrls.Count > 0) + { + var sitemapNorm = new HashSet( + sitemapUrls.Select(UrlNormalizeHelper.NormalizeUrl).Where(u => u.Length > 0), + StringComparer.Ordinal); + var noindexCount = 0; + foreach (var row in SuccessRows(rows)) + { + if (noindexCount >= 15) + { + break; + } + + if (row.Noindex != true) + { + continue; + } + + var url = row.Url.Trim(); + if (string.IsNullOrEmpty(url)) + { + continue; + } + + if (!sitemapNorm.Contains(UrlNormalizeHelper.NormalizeUrl(url))) + { + continue; + } + + issues.Add(Issue( + "Page has noindex but is listed in XML sitemap.", + url, + "Critical", + "Remove the URL from the sitemap or remove noindex if the page should be indexed.")); + noindexCount++; + } + } + return issues; } + private static List ExtractSitemapUrls(IReadOnlyDictionary indexation) + { + if (!indexation.TryGetValue("sitemap_urls", out var urlsObj) || urlsObj is null) + { + return []; + } + + if (urlsObj is JsonElement el && el.ValueKind == JsonValueKind.Array) + { + return el.EnumerateArray() + .Select(u => u.GetString()?.Trim() ?? "") + .Where(u => u.Length > 0) + .ToList(); + } + + if (urlsObj is IEnumerable list) + { + return list + .Select(u => u?.ToString()?.Trim() ?? "") + .Where(u => u.Length > 0) + .ToList(); + } + + return []; + } + public static void MergeIssuesIntoCategory( IList categories, string categoryId, diff --git a/services/ReportService/src/ReportService.Application/Build/CtrCurve.cs b/services/ReportService/src/ReportService.Application/Build/CtrCurve.cs index 93637ee..7290c32 100644 --- a/services/ReportService/src/ReportService.Application/Build/CtrCurve.cs +++ b/services/ReportService/src/ReportService.Application/Build/CtrCurve.cs @@ -21,8 +21,22 @@ public static class CtrCurve public static double IndustryCtrFraction(double position) { - var slot = position > 0 ? Math.Max(1, (int)Math.Ceiling(position)) : 1; - return Curve.GetValueOrDefault(slot, DefaultFraction); + if (position <= 0) + { + return Curve.GetValueOrDefault(1, DefaultFraction); + } + + var lower = Math.Max(1, (int)Math.Floor(position)); + var upper = Math.Max(1, (int)Math.Ceiling(position)); + if (lower == upper || Math.Abs(position - lower) < 1e-9) + { + return Curve.GetValueOrDefault(lower, DefaultFraction); + } + + var lowerCtr = Curve.GetValueOrDefault(lower, DefaultFraction); + var upperCtr = Curve.GetValueOrDefault(upper, DefaultFraction); + var weight = position - lower; + return lowerCtr + (upperCtr - lowerCtr) * weight; } public static double IndustryCtrPercent(double position) => diff --git a/services/ReportService/src/ReportService.Application/Build/NativeReportBuilder.cs b/services/ReportService/src/ReportService.Application/Build/NativeReportBuilder.cs index 070f791..a7867de 100644 --- a/services/ReportService/src/ReportService.Application/Build/NativeReportBuilder.cs +++ b/services/ReportService/src/ReportService.Application/Build/NativeReportBuilder.cs @@ -100,6 +100,12 @@ public async Task BuildNativeSliceAsync( siteLevel = MergeSiteLevelConfig(siteLevel, config); var runSecurityScan = ParseBool(config, "run_security_scan", defaultValue: true); var securityFindings = SecurityScanBuilder.BuildPassive(rows, startUrl, runSecurityScan); + Dictionary? cruxSummary = null; + if (ParseBool(config, "enable_crux", defaultValue: false) && !string.IsNullOrWhiteSpace(startUrl)) + { + cruxSummary = await CruxOriginMetricsFetcher.FetchAsync(httpClientFactory, startUrl, cancellationToken); + } + var categories = categoryBuilder.BuildCategories( rows, edges, @@ -107,7 +113,7 @@ public async Task BuildNativeSliceAsync( siteLevel, startUrl, lighthouseSummary, - cruxSummary: null, + cruxSummary: cruxSummary, lighthouseByUrl: lhByUrl, mlBundle: mlBundle, securityFindings: securityFindings); @@ -233,6 +239,7 @@ public async Task BuildNativeSliceAsync( competitorGap, securityFindings, lighthouseSummary, + CruxSummary: cruxSummary, ContactIntelligence: contactIntelligence, ImageInventory: imageInventory, ImageInventorySummary: imageInventorySummary, @@ -475,6 +482,7 @@ public sealed record NativeReportSlice( Dictionary? CompetitorLinkGap = null, List>? SecurityFindings = null, Dictionary? LighthouseSummary = null, + Dictionary? CruxSummary = null, Dictionary? ContactIntelligence = null, List>? ImageInventory = null, Dictionary? ImageInventorySummary = null, diff --git a/services/ReportService/src/ReportService.Application/Build/NativeReportPayloadAssembler.cs b/services/ReportService/src/ReportService.Application/Build/NativeReportPayloadAssembler.cs index be7c5c9..862c908 100644 --- a/services/ReportService/src/ReportService.Application/Build/NativeReportPayloadAssembler.cs +++ b/services/ReportService/src/ReportService.Application/Build/NativeReportPayloadAssembler.cs @@ -129,6 +129,11 @@ public static class NativeReportPayloadAssembler payload["lighthouse_human_summary"] = LighthouseJsonHelper.ExtractHumanSummary(slice.LighthouseSummary); } + if (slice.CruxSummary is not null) + { + payload["crux_summary"] = slice.CruxSummary; + } + MergeAnalysisIntoPayload(payload, mlBundle); if (propertyId is not null) diff --git a/services/ReportService/src/ReportService.Application/Build/SiteHealthScoreBuilder.cs b/services/ReportService/src/ReportService.Application/Build/SiteHealthScoreBuilder.cs index 09ad34d..cb0755c 100644 --- a/services/ReportService/src/ReportService.Application/Build/SiteHealthScoreBuilder.cs +++ b/services/ReportService/src/ReportService.Application/Build/SiteHealthScoreBuilder.cs @@ -1,77 +1,20 @@ +using WebsiteProfiling.Contracts.Report; + namespace ReportService.Application.Build; -/// Weighted site health score from fixable audit categories (excludes search_performance, intelligence). +/// ReportService adapters over shared . public static class SiteHealthScoreBuilder { - private static readonly Dictionary Weights = new(StringComparer.Ordinal) - { - ["technical_seo"] = 0.25, - ["link_health"] = 0.20, - ["performance"] = 0.15, - ["security"] = 0.15, - ["core_web_vitals"] = 0.10, - ["mobile"] = 0.10, - ["html_accessibility"] = 0.05, - }; - - private static readonly HashSet Excluded = new(StringComparer.Ordinal) - { - "search_performance", - "intelligence", - }; - - public static int? Compute(IReadOnlyList categories) - { - var (_, score) = ComputeWithCategoryScores(categories); - return score; - } + public static int? Compute(IReadOnlyList categories) => + WebsiteProfiling.Contracts.Report.SiteHealthScoreBuilder.Compute( + categories.Select(c => new SiteHealthCategory(c.Id, c.Score)).ToList()); public static (Dictionary CategoryScores, int? SiteHealthScore) ComputeWithCategoryScores( - IReadOnlyList categories) - { - var categoryScores = new Dictionary(StringComparer.Ordinal); - foreach (var cat in categories) - { - if (Excluded.Contains(cat.Id) || cat.Score is not int score) - { - continue; - } - - categoryScores[cat.Id] = score; - } - - double weightedSum = 0; - double weightTotal = 0; - foreach (var (id, weight) in Weights) - { - if (!categoryScores.TryGetValue(id, out var score)) - { - continue; - } - - weightedSum += score * weight; - weightTotal += weight; - } - - int? siteHealth = weightTotal > 0 - ? (int)Math.Round(weightedSum / weightTotal) - : null; - - return (categoryScores, siteHealth); - } + IReadOnlyList categories) => + WebsiteProfiling.Contracts.Report.SiteHealthScoreBuilder.ComputeWithCategoryScores( + categories.Select(c => new SiteHealthCategory(c.Id, c.Score)).ToList()); public static (Dictionary CategoryScores, int? SiteHealthScore) ComputeWithCategoryScores( - IEnumerable> payloadCategories) - { - var categories = payloadCategories - .Select(c => new ReportCategory( - c.GetValueOrDefault("id")?.ToString() ?? "", - c.GetValueOrDefault("name")?.ToString() ?? "", - c.TryGetValue("score", out var s) && s is int or double or float ? Convert.ToInt32(s) : null, - [], - [])) - .ToList(); - - return ComputeWithCategoryScores(categories); - } + IEnumerable> payloadCategories) => + WebsiteProfiling.Contracts.Report.SiteHealthScoreBuilder.ComputeWithCategoryScores(payloadCategories); } diff --git a/services/ReportService/src/ReportService.Application/Integrations/CruxOriginMetricsFetcher.cs b/services/ReportService/src/ReportService.Application/Integrations/CruxOriginMetricsFetcher.cs new file mode 100644 index 0000000..74d8703 --- /dev/null +++ b/services/ReportService/src/ReportService.Application/Integrations/CruxOriginMetricsFetcher.cs @@ -0,0 +1,149 @@ +using System.Net.Http.Json; +using System.Text.Json; + +namespace ReportService.Application.Integrations; + +/// Port of Python integrations/crux/fetch.py — origin-level CrUX field metrics. +public static class CruxOriginMetricsFetcher +{ + private const string CruxApi = "https://chromeuxreport.googleapis.com/v1/records:queryRecord"; + + public static async Task> FetchAsync( + IHttpClientFactory httpClientFactory, + string startUrl, + CancellationToken cancellationToken = default) + { + var origin = OriginFromUrl(startUrl); + if (string.IsNullOrEmpty(origin)) + { + return new Dictionary { ["ok"] = false, ["error"] = "Invalid origin" }; + } + + var apiKey = Environment.GetEnvironmentVariable("CRUX_API_KEY")?.Trim(); + var requestUrl = string.IsNullOrEmpty(apiKey) + ? CruxApi + : $"{CruxApi}?key={Uri.EscapeDataString(apiKey)}"; + + try + { + var client = httpClientFactory.CreateClient(); + using var response = await client.PostAsJsonAsync( + requestUrl, + new { origin }, + cancellationToken); + response.EnsureSuccessStatusCode(); + await using var stream = await response.Content.ReadAsStreamAsync(cancellationToken); + using var doc = await JsonDocument.ParseAsync(stream, cancellationToken: cancellationToken); + return ParseRecord(origin, doc.RootElement); + } + catch (Exception ex) + { + return new Dictionary + { + ["ok"] = false, + ["origin"] = origin, + ["error"] = ex.Message, + }; + } + } + + internal static Dictionary ParseRecord(string origin, JsonElement data) + { + var record = data.TryGetProperty("record", out var recordEl) ? recordEl : default; + var metricsEl = record.ValueKind == JsonValueKind.Object + && record.TryGetProperty("metrics", out var m) + ? m + : default; + + var metrics = new Dictionary(StringComparer.Ordinal); + if (metricsEl.ValueKind == JsonValueKind.Object) + { + foreach (var prop in metricsEl.EnumerateObject()) + { + if (prop.Value.ValueKind != JsonValueKind.Object) + { + continue; + } + + object? p75 = null; + if (prop.Value.TryGetProperty("percentiles", out var pct) + && pct.ValueKind == JsonValueKind.Object + && pct.TryGetProperty("p75", out var p75El)) + { + p75 = p75El.ValueKind switch + { + JsonValueKind.Number => p75El.TryGetInt64(out var n) ? n : p75El.GetDouble(), + JsonValueKind.String => p75El.GetString(), + _ => null, + }; + } + + metrics[prop.Name] = new Dictionary { ["p75"] = p75 }; + } + } + + double? LcpP75() => MetricP75(metrics, "largest_contentful_paint"); + double? InpP75() => MetricP75(metrics, "interaction_to_next_paint"); + double? ClsP75() => MetricP75(metrics, "cumulative_layout_shift"); + + return new Dictionary + { + ["origin"] = origin, + ["ok"] = true, + ["metrics"] = metrics, + ["pass"] = new Dictionary + { + ["lcp"] = PassThreshold(LcpP75(), 2500), + ["inp"] = PassThreshold(InpP75(), 200), + ["cls"] = PassThreshold(ClsP75(), 0.1), + }, + }; + } + + private static double? MetricP75(IReadOnlyDictionary metrics, string name) + { + if (!metrics.TryGetValue(name, out var metricObj) + || metricObj is not Dictionary metric + || !metric.TryGetValue("p75", out var p75Obj) + || p75Obj is null) + { + return null; + } + + return p75Obj switch + { + double d => d, + float f => f, + long l => l, + int i => i, + string s when double.TryParse(s, out var parsed) => parsed, + _ => null, + }; + } + + private static bool PassThreshold(double? value, double limit) + { + if (value is null) + { + return false; + } + + return value.Value <= limit; + } + + private static string OriginFromUrl(string url) + { + var trimmed = url.Trim(); + if (string.IsNullOrEmpty(trimmed)) + { + return ""; + } + + if (!Uri.TryCreate(trimmed, UriKind.Absolute, out var uri)) + { + return ""; + } + + return $"{uri.Scheme}://{uri.Authority}"; + } +} diff --git a/services/ReportService/tests/ReportService.Tests/CategoryBuilderParityTests.cs b/services/ReportService/tests/ReportService.Tests/CategoryBuilderParityTests.cs index 01aee10..5ae2db0 100644 --- a/services/ReportService/tests/ReportService.Tests/CategoryBuilderParityTests.cs +++ b/services/ReportService/tests/ReportService.Tests/CategoryBuilderParityTests.cs @@ -58,8 +58,8 @@ public void BuildCategories_scores_match_python_fixture() Assert.Equal(94, byId["html_accessibility"].Score); // Python: one broken (-2) + one redirect (-1) Assert.Equal(97, byId["link_health"].Score); - // Python: all success rows missing HSTS/XCTO/XFO when header columns absent (-25) - Assert.Equal(75, byId["security"].Score); + // Security header deductions come from passive findings only (not inline header checks). + Assert.Equal(100, byId["security"].Score); Assert.Equal(100, byId["intelligence"].Score); } diff --git a/services/ReportService/tests/ReportService.Tests/CoreWebVitalsCategoryBuilderTests.cs b/services/ReportService/tests/ReportService.Tests/CoreWebVitalsCategoryBuilderTests.cs new file mode 100644 index 0000000..93e0b79 --- /dev/null +++ b/services/ReportService/tests/ReportService.Tests/CoreWebVitalsCategoryBuilderTests.cs @@ -0,0 +1,34 @@ +using System.Text.Json; +using ReportService.Application.Build.Categories; + +namespace ReportService.Tests; + +public sealed class CoreWebVitalsCategoryBuilderTests +{ + [Fact] + public void Build_applies_crux_deductions_to_lighthouse_score() + { + var lh = JsonSerializer.Deserialize>( + """{"median_metrics": {"performance_score": 0.95}, "top_failures": []}""")!; + var crux = JsonSerializer.Deserialize>( + """{"ok": true, "pass": {"lcp": false, "inp": false, "cls": false}}""")!; + + var category = CoreWebVitalsCategoryBuilder.Build(lh, crux); + + Assert.Equal(50, category.Score); + Assert.Equal(3, category.Issues.Count(i => i.Message.Contains("CrUX", StringComparison.Ordinal))); + } + + [Fact] + public void Build_leaves_score_unchanged_when_crux_passes() + { + var lh = JsonSerializer.Deserialize>( + """{"median_metrics": {"performance_score": 0.95}, "top_failures": []}""")!; + var crux = JsonSerializer.Deserialize>( + """{"ok": true, "pass": {"lcp": true, "inp": true, "cls": true}}""")!; + + var category = CoreWebVitalsCategoryBuilder.Build(lh, crux); + + Assert.Equal(95, category.Score); + } +} diff --git a/services/ReportService/tests/ReportService.Tests/CruxOriginMetricsFetcherTests.cs b/services/ReportService/tests/ReportService.Tests/CruxOriginMetricsFetcherTests.cs new file mode 100644 index 0000000..28ed361 --- /dev/null +++ b/services/ReportService/tests/ReportService.Tests/CruxOriginMetricsFetcherTests.cs @@ -0,0 +1,32 @@ +using System.Text.Json; +using ReportService.Application.Integrations; + +namespace ReportService.Tests; + +public sealed class CruxOriginMetricsFetcherTests +{ + [Fact] + public void ParseRecord_marks_failing_metrics_and_passing_cls() + { + const string json = """ + { + "record": { + "metrics": { + "largest_contentful_paint": {"percentiles": {"p75": 3000}}, + "interaction_to_next_paint": {"percentiles": {"p75": 250}}, + "cumulative_layout_shift": {"percentiles": {"p75": "0.05"}} + } + } + } + """; + + using var doc = JsonDocument.Parse(json); + var parsed = CruxOriginMetricsFetcher.ParseRecord("https://example.com", doc.RootElement); + + Assert.True(parsed["ok"] is true); + var pass = Assert.IsType>(parsed["pass"]); + Assert.Equal(false, pass["lcp"]); + Assert.Equal(false, pass["inp"]); + Assert.Equal(true, pass["cls"]); + } +} diff --git a/services/ReportService/tests/ReportService.Tests/CtrCurveTests.cs b/services/ReportService/tests/ReportService.Tests/CtrCurveTests.cs index 5ae63fd..802eeda 100644 --- a/services/ReportService/tests/ReportService.Tests/CtrCurveTests.cs +++ b/services/ReportService/tests/ReportService.Tests/CtrCurveTests.cs @@ -13,6 +13,14 @@ public void IndustryCtrPercent_is_continuous_near_position_four() Assert.True(Math.Abs(atThreeFive - atFour) < 5.0); } + [Fact] + public void IndustryCtrFraction_interpolates_between_slots() + { + Assert.Equal(0.103, CtrCurve.IndustryCtrFraction(3.0), 3); + Assert.Equal(0.100, CtrCurve.IndustryCtrFraction(3.1), 3); + Assert.Equal(0.076, CtrCurve.IndustryCtrFraction(3.9), 3); + } + [Fact] public void IndustryCtrFraction_matches_position_slot() { diff --git a/services/ReportService/tests/ReportService.Tests/IndexationCoverageIssuesTests.cs b/services/ReportService/tests/ReportService.Tests/IndexationCoverageIssuesTests.cs new file mode 100644 index 0000000..8f202e6 --- /dev/null +++ b/services/ReportService/tests/ReportService.Tests/IndexationCoverageIssuesTests.cs @@ -0,0 +1,30 @@ +using System.Text.Json; +using ReportService.Application.Build; +using ReportService.Application.Repositories; + +namespace ReportService.Tests; + +public sealed class IndexationCoverageIssuesTests +{ + [Fact] + public void IndexationCoverageIssues_flags_noindex_urls_in_sitemap() + { + var rows = new List + { + new() { Url = "https://example.com/private", Status = "200", Noindex = true }, + new() { Url = "https://example.com/public", Status = "200", Noindex = false }, + }; + var indexation = JsonSerializer.Deserialize>( + """ + { + "sitemap_urls": ["https://example.com/private", "https://example.com/public"] + } + """)!; + + var issues = CategoryHelpers.IndexationCoverageIssues(rows, indexation); + + Assert.Single(issues); + Assert.Contains("noindex", issues[0].Message, StringComparison.OrdinalIgnoreCase); + Assert.Equal("Critical", issues[0].Priority); + } +} diff --git a/services/ReportService/tests/ReportService.Tests/SecurityCategoryBuilderTests.cs b/services/ReportService/tests/ReportService.Tests/SecurityCategoryBuilderTests.cs index 6eb1fa8..d8fcf78 100644 --- a/services/ReportService/tests/ReportService.Tests/SecurityCategoryBuilderTests.cs +++ b/services/ReportService/tests/ReportService.Tests/SecurityCategoryBuilderTests.cs @@ -23,7 +23,8 @@ public void Build_headers_mixed_content_findings() }, }; - var findings = new List> + var passiveFindings = SecurityScanBuilder.BuildPassive(rows, "https://example.com/"); + var findings = passiveFindings.Concat(new List> { new() { @@ -40,7 +41,7 @@ public void Build_headers_mixed_content_findings() ["url"] = "", ["recommendation"] = "", }, - }; + }).ToList(); var cat = SecurityCategoryBuilder.Build(rows, "https://example.com/", findings); var msgs = string.Join(" ", cat.Issues.Select(i => i.Message)).ToLowerInvariant(); @@ -58,4 +59,28 @@ public void Build_headers_mixed_content_findings() (i["message"]?.ToString() ?? "").Contains("SQL injection", StringComparison.OrdinalIgnoreCase)); Assert.Equal("sql_injection", issueDict["finding_type"]); } + + [Fact] + public void Build_passive_header_findings_not_double_counted() + { + var rows = new List + { + new() + { + Url = "https://example.com/", + Status = "200", + FinalUrl = "https://example.com/", + StrictTransportSecurity = "", + XContentTypeOptions = "", + XFrameOptions = "", + ContentSecurityPolicy = "", + }, + }; + + var findings = SecurityScanBuilder.BuildPassive(rows, "https://example.com/"); + var cat = SecurityCategoryBuilder.Build(rows, "https://example.com/", findings); + + Assert.Equal(75, cat.Score); + Assert.Equal(4, cat.Issues.Count); + } } diff --git a/services/Shared/WebsiteProfiling.Contracts/Report/SiteHealthScoreBuilder.cs b/services/Shared/WebsiteProfiling.Contracts/Report/SiteHealthScoreBuilder.cs new file mode 100644 index 0000000..e031f86 --- /dev/null +++ b/services/Shared/WebsiteProfiling.Contracts/Report/SiteHealthScoreBuilder.cs @@ -0,0 +1,146 @@ +using System.Text.Json; + +namespace WebsiteProfiling.Contracts.Report; + +public readonly record struct SiteHealthCategory(string Id, int? Score); + +/// Weighted site health score from fixable audit categories (excludes search_performance, intelligence). +public static class SiteHealthScoreBuilder +{ + private static readonly Dictionary Weights = new(StringComparer.Ordinal) + { + ["technical_seo"] = 0.25, + ["link_health"] = 0.20, + ["performance"] = 0.15, + ["security"] = 0.15, + ["core_web_vitals"] = 0.10, + ["mobile"] = 0.10, + ["html_accessibility"] = 0.05, + }; + + private static readonly HashSet Excluded = new(StringComparer.Ordinal) + { + "search_performance", + "intelligence", + }; + + public static int? Compute(IReadOnlyList categories) + { + var (_, score) = ComputeWithCategoryScores(categories); + return score; + } + + public static (Dictionary CategoryScores, int? SiteHealthScore) ComputeWithCategoryScores( + IReadOnlyList categories) + { + var categoryScores = new Dictionary(StringComparer.Ordinal); + foreach (var cat in categories) + { + if (Excluded.Contains(cat.Id) || cat.Score is not int score) + { + continue; + } + + categoryScores[cat.Id] = score; + } + + return (categoryScores, WeightedScore(categoryScores)); + } + + public static (Dictionary CategoryScores, int? SiteHealthScore) ComputeWithCategoryScores( + IEnumerable> payloadCategories) + { + var categories = payloadCategories + .Select(c => new SiteHealthCategory( + c.GetValueOrDefault("id")?.ToString() ?? "", + c.TryGetValue("score", out var s) && s is int or double or float ? Convert.ToInt32(s) : null)) + .ToList(); + + return ComputeWithCategoryScores(categories); + } + + /// + /// Prefer payload site_health_score fields; fall back to weighted category score for older reports. + /// + public static int? ResolveFromPayload(JsonElement payload) + { + if (payload.ValueKind != JsonValueKind.Object) + { + return null; + } + + if (payload.TryGetProperty("summary", out var summary) + && summary.ValueKind == JsonValueKind.Object + && summary.TryGetProperty("site_health_score", out var summaryScore) + && summaryScore.ValueKind == JsonValueKind.Number + && summaryScore.TryGetInt32(out var fromSummary)) + { + return fromSummary; + } + + if (payload.TryGetProperty("site_health_score", out var topScore) + && topScore.ValueKind == JsonValueKind.Number + && topScore.TryGetInt32(out var fromTop)) + { + return fromTop; + } + + if (payload.TryGetProperty("categories", out var categories) + && categories.ValueKind == JsonValueKind.Array) + { + return ComputeFromJsonCategories(categories); + } + + return null; + } + + public static int? ComputeFromJsonCategories(JsonElement categories) + { + if (categories.ValueKind != JsonValueKind.Array) + { + return null; + } + + var list = new List(); + foreach (var cat in categories.EnumerateArray()) + { + if (cat.ValueKind != JsonValueKind.Object) + { + continue; + } + + var id = cat.TryGetProperty("id", out var idEl) && idEl.ValueKind == JsonValueKind.String + ? idEl.GetString() ?? "" + : ""; + int? score = null; + if (cat.TryGetProperty("score", out var scoreEl) && scoreEl.ValueKind == JsonValueKind.Number) + { + score = (int)Math.Round(scoreEl.GetDouble()); + } + + list.Add(new SiteHealthCategory(id, score)); + } + + return Compute(list); + } + + private static int? WeightedScore(Dictionary categoryScores) + { + double weightedSum = 0; + double weightTotal = 0; + foreach (var (id, weight) in Weights) + { + if (!categoryScores.TryGetValue(id, out var score)) + { + continue; + } + + weightedSum += score * weight; + weightTotal += weight; + } + + return weightTotal > 0 + ? (int)Math.Round(weightedSum / weightTotal) + : null; + } +} diff --git a/src/website_profiling/db/report_store.py b/src/website_profiling/db/report_store.py index f601fea..8c7bddc 100644 --- a/src/website_profiling/db/report_store.py +++ b/src/website_profiling/db/report_store.py @@ -6,7 +6,7 @@ from psycopg import Connection -from ..scoring import round_half_up +from ..scoring import round_half_up, site_health_score_from_payload from ._common import _json_val, _now_iso, _parse_row_json, _row_field from .crawl_store import get_crawl_run_info @@ -44,13 +44,8 @@ def _write_audit_health_snapshot( report_data: dict[str, Any], ) -> None: """Persist health score row for portfolio sparklines and alerts.""" + health_score = site_health_score_from_payload(report_data) categories = report_data.get("categories") or [] - scores = [ - float(c.get("score")) - for c in categories - if isinstance(c, dict) and isinstance(c.get("score"), (int, float)) - ] - health_score = round_half_up(sum(scores) / len(scores)) if scores else None category_scores: dict[str, float] = {} issue_counts = {"Critical": 0, "High": 0, "Medium": 0, "Low": 0} for cat in categories: diff --git a/src/website_profiling/integrations/google/keyword_enrich.py b/src/website_profiling/integrations/google/keyword_enrich.py index 70e85b9..e51cae9 100644 --- a/src/website_profiling/integrations/google/keyword_enrich.py +++ b/src/website_profiling/integrations/google/keyword_enrich.py @@ -160,8 +160,16 @@ def opportunity_clicks(impressions: int, current_pos: float, target_pos: int = 3 def industry_ctr(pos: float) -> float: - pos_slot = max(1, math.ceil(pos)) if pos > 0 else 1 - return CTR_CURVE.get(pos_slot, CTR_CURVE_DEFAULT) + if pos <= 0: + return CTR_CURVE.get(1, CTR_CURVE_DEFAULT) + lower = max(1, math.floor(pos)) + upper = max(1, math.ceil(pos)) + if lower == upper or abs(pos - lower) < 1e-9: + return CTR_CURVE.get(lower, CTR_CURVE_DEFAULT) + lower_ctr = CTR_CURVE.get(lower, CTR_CURVE_DEFAULT) + upper_ctr = CTR_CURVE.get(upper, CTR_CURVE_DEFAULT) + weight = pos - lower + return lower_ctr + (upper_ctr - lower_ctr) * weight # ── Cannibalisation ─────────────────────────────────────────────────────────── diff --git a/src/website_profiling/parsing/tech.py b/src/website_profiling/parsing/tech.py index 587e22f..0c42390 100644 --- a/src/website_profiling/parsing/tech.py +++ b/src/website_profiling/parsing/tech.py @@ -9,6 +9,8 @@ ("WordPress", "html", "/wp-includes/"), ("Drupal", "meta_generator", "Drupal"), ("Joomla", "meta_generator", "Joomla"), + ("Hugo", "meta_generator", "Hugo"), + ("Jekyll", "meta_generator", "Jekyll"), ("Shopify", "html", "cdn.shopify.com"), ("Squarespace", "html", "squarespace.com"), ("Wix", "html", "wix.com"), @@ -30,8 +32,10 @@ ("Google Analytics", "html", "google-analytics.com/analytics.js"), ("Google Analytics", "html", "googletagmanager.com/gtag"), ("Google Tag Manager", "html", "googletagmanager.com/gtm.js"), + ("Google Tag Manager", "html", "googletagmanager.com"), ("Facebook Pixel", "html", "connect.facebook.net"), ("Hotjar", "html", "hotjar.com"), + ("Microsoft Clarity", "html", "clarity.ms"), ("Google Fonts", "html", "fonts.googleapis.com"), ("Font Awesome", "html", "fontawesome"), ("Cloudflare", "header", "cf-ray"), @@ -40,6 +44,7 @@ ("LiteSpeed", "header_server", "litespeed"), ("Vercel", "header_server", "vercel"), ("Netlify", "header_server", "netlify"), + ("GitHub Pages", "header_server", "github"), ("Amazon CloudFront", "header", "x-amz-cf-id"), ("AWS", "header_server", "amazons3"), ] diff --git a/src/website_profiling/reporting/builder.py b/src/website_profiling/reporting/builder.py index 745f3e5..880eb6b 100644 --- a/src/website_profiling/reporting/builder.py +++ b/src/website_profiling/reporting/builder.py @@ -18,7 +18,7 @@ from ..ai_service_client import cluster_keywords_llm, run_llm_enrichment from ..llm_config import load_llm_config_from_db, llm_is_enabled from ..security_scanner import run_security_scan -from ..scoring import round_half_up +from ..scoring import round_half_up, site_health_score_from_payload from .categories import build_categories from .content_analytics import ( _build_content_analytics, @@ -48,10 +48,10 @@ _parse_page_analysis_cell, _validate_report_url_counts, ) +from .categories._helpers import THIN_CONTENT_CHARS from .seo_summary import ( META_DESC_LEN_MAX, META_DESC_LEN_MIN, - THIN_CONTENT_CHARS, TITLE_LEN_MAX, TITLE_LEN_MIN, _compute_summary_seo_issues, @@ -712,14 +712,7 @@ def run_simple_report( from ..tools.audit_tools.context import AuditToolContext portfolio = get_portfolio_summary(conn, AuditToolContext(property_id=property_id), {}) - scores = [] - for c in report_data.get("categories") or []: - try: - if c.get("score") is not None: - scores.append(int(float(c.get("score")))) - except (TypeError, ValueError): - continue - prop_health = round_half_up(sum(scores) / len(scores)) if scores else None + prop_health = site_health_score_from_payload(report_data) prop_count = int(portfolio.get("count") or 0) median = portfolio.get("median_health_score") bench: dict[str, Any] = { diff --git a/src/website_profiling/reporting/builder_sections/content_urls.py b/src/website_profiling/reporting/builder_sections/content_urls.py index 7d7f0a0..3827700 100644 --- a/src/website_profiling/reporting/builder_sections/content_urls.py +++ b/src/website_profiling/reporting/builder_sections/content_urls.py @@ -9,10 +9,10 @@ import pandas as pd +from ..categories._helpers import THIN_CONTENT_CHARS from ..seo_summary import ( META_DESC_LEN_MAX, META_DESC_LEN_MIN, - THIN_CONTENT_CHARS, TITLE_LEN_MAX, TITLE_LEN_MIN, ) diff --git a/src/website_profiling/reporting/categories/_helpers.py b/src/website_profiling/reporting/categories/_helpers.py index 8b5d9c8..15fd2de 100644 --- a/src/website_profiling/reporting/categories/_helpers.py +++ b/src/website_profiling/reporting/categories/_helpers.py @@ -16,7 +16,9 @@ TITLE_LEN_MAX = 60 META_DESC_LEN_MIN = 70 META_DESC_LEN_MAX = 160 -REDIRECT_CHAIN_LONG = 2 +REDIRECT_CHAIN_LONG = 3 +MAX_ISSUES_PER_CHECK = 20 +MAX_HREFLANG_ISSUES_PER_CHECK = 15 def _issue(message: str, url: Optional[str] = None, priority: str = "Medium", recommendation: str = "") -> dict: @@ -55,6 +57,8 @@ def _hreflang_issues(success_df: pd.DataFrame) -> list[dict]: issues: list[dict] = [] if "page_analysis" not in success_df.columns: return issues + duplicate_count = 0 + self_ref_count = 0 for _, row in success_df.iterrows(): pa = _page_analysis_dict(row) alts = pa.get("hreflang_alternates") or [] @@ -64,21 +68,23 @@ def _hreflang_issues(success_df: pd.DataFrame) -> list[dict]: langs = [str(a.get("hreflang") or a.get("lang") or "").strip().lower() for a in alts if isinstance(a, dict)] hrefs = [str(a.get("href") or "").strip() for a in alts if isinstance(a, dict)] if langs and len(set(langs)) < len(langs): - issues.append(_issue( - "Duplicate hreflang language codes on page.", - url=url, - priority="High", - recommendation="Each hreflang alternate should use a unique language/region code.", - )) - break + if duplicate_count < MAX_HREFLANG_ISSUES_PER_CHECK: + issues.append(_issue( + "Duplicate hreflang language codes on page.", + url=url, + priority="High", + recommendation="Each hreflang alternate should use a unique language/region code.", + )) + duplicate_count += 1 if url and hrefs and url.rstrip("/") not in [h.rstrip("/") for h in hrefs]: - issues.append(_issue( - "Hreflang cluster missing self-referencing alternate.", - url=url, - priority="Medium", - recommendation="Include a hreflang link pointing to this page URL.", - )) - break + if self_ref_count < MAX_HREFLANG_ISSUES_PER_CHECK: + issues.append(_issue( + "Hreflang cluster missing self-referencing alternate.", + url=url, + priority="Medium", + recommendation="Include a hreflang link pointing to this page URL.", + )) + self_ref_count += 1 return issues @@ -166,7 +172,10 @@ def _indexation_coverage_issues( sitemap_norm = {normalize_url(u) for u in sitemap_urls} success = df[df["status"].astype(str).str.match(r"2\d{2}", na=False)] if "status" in df.columns else df + noindex_in_sitemap = 0 for _, row in success.iterrows(): + if noindex_in_sitemap >= 15: + break url = str(row.get("url") or "").strip() if not url: continue @@ -178,7 +187,7 @@ def _indexation_coverage_issues( priority="Critical", recommendation="Remove the URL from the sitemap or remove noindex if the page should be indexed.", )) - break + noindex_in_sitemap += 1 return issues diff --git a/src/website_profiling/reporting/categories/accessibility.py b/src/website_profiling/reporting/categories/accessibility.py index 0ecff34..35cdb69 100644 --- a/src/website_profiling/reporting/categories/accessibility.py +++ b/src/website_profiling/reporting/categories/accessibility.py @@ -130,17 +130,21 @@ def contrast_issues_from_sources( if not contrast_hits: continue seen_urls.add(url.rstrip("/")) - first = contrast_hits[0] - msg = str(first.get("description") or first.get("help") or "Color contrast violation") - issues.append(_issue( - f"axe: {msg}", - url=url, - priority="Medium", - recommendation=str( - first.get("help") - or "Fix text/background contrast to meet WCAG AA (axe-core)." - ), - )) + for violation in contrast_hits: + msg = str(violation.get("description") or violation.get("help") or "Color contrast violation") + issues.append(_issue( + f"axe: {msg}", + url=url, + priority="Medium", + recommendation=str( + violation.get("help") + or "Fix text/background contrast to meet WCAG AA (axe-core)." + ), + )) + if len(issues) >= 40: + break + if len(issues) >= 40: + break issues.extend( lighthouse_accessibility_issues_from_sources( @@ -223,7 +227,7 @@ def category_html_accessibility( very_thin = int(((wc > 0) & (wc < 100)).sum()) if very_thin > 0: issues.append(_issue( - f"{very_thin} page(s) with very thin content (under 100 words).", + f"{very_thin} page(s) with very thin content (under 100 words; SEO thin flag uses 200 words).", priority="High", recommendation="Expand thin pages with meaningful content (aim for 300+ words).", )) @@ -255,8 +259,6 @@ def category_html_accessibility( )) score = _score_deductions(100, deductions) - if len(success_df) > 0 and score == 0: - score = 5 score = min(100, max(0, score)) return { "id": "html_accessibility", diff --git a/src/website_profiling/reporting/categories/performance.py b/src/website_profiling/reporting/categories/performance.py index 2df7e58..e73a945 100644 --- a/src/website_profiling/reporting/categories/performance.py +++ b/src/website_profiling/reporting/categories/performance.py @@ -84,10 +84,20 @@ def category_core_web_vitals_from_lighthouse( priority="High", recommendation=rec, )) + crux_deduction = 0 + if crux_summary and crux_summary.get("ok"): + pw = crux_summary.get("pass") or {} + for metric in ("lcp", "inp", "cls"): + if pw.get(metric) is False: + crux_deduction += 15 + crux_deduction = min(45, crux_deduction) + score = perf_score + if crux_deduction > 0: + score = max(0, (perf_score if perf_score is not None else 0) - crux_deduction) return { "id": "core_web_vitals", "name": CATEGORY_CORE_WEB_VITALS, - "score": perf_score, + "score": score, "issues": _sort_issues(issues), "recommendations": recommendations or ["Core Web Vitals measured by Lighthouse; see median_metrics in lighthouse_summary.json."], } @@ -126,13 +136,15 @@ def category_performance(df: pd.DataFrame) -> dict: total_imgs = success_df["images_total"].fillna(0).astype(int).sum() if total_imgs > 0 and "img_without_lazy" in success_df.columns: no_lazy = success_df["img_without_lazy"].fillna(0).astype(int).sum() - if no_lazy > total_imgs * 0.5: - issues.append(_issue( - "Many images without lazy loading.", - priority="Medium", - recommendation="Add loading='lazy' to off-screen images.", - )) - deductions.append((10, True)) + if no_lazy > 0: + lazy_pct = no_lazy * 100.0 / total_imgs + if lazy_pct > 20: + issues.append(_issue( + "Many images without lazy loading.", + priority="Medium", + recommendation="Add loading='lazy' to off-screen images.", + )) + deductions.append((min(15, int(no_lazy * 10.0 / total_imgs)), True)) if total_imgs > 0 and "img_without_dimensions" in success_df.columns: no_dims = success_df["img_without_dimensions"].fillna(0).astype(int).sum() if no_dims > 0: diff --git a/src/website_profiling/reporting/categories/security.py b/src/website_profiling/reporting/categories/security.py index f975cc3..d8741bd 100644 --- a/src/website_profiling/reporting/categories/security.py +++ b/src/website_profiling/reporting/categories/security.py @@ -53,46 +53,6 @@ def category_security( )) deductions.append((20, True)) - success_df = df[df["status"].astype(str).str.match(r"2\d{2}", na=False)] if "status" in df.columns else pd.DataFrame() - if len(success_df) > 0: - # Security headers: sample from first row or aggregate (optional columns) - missing_hsts = (success_df["strict_transport_security"].fillna("").astype(str).str.strip() == "").sum() if "strict_transport_security" in success_df.columns else len(success_df) - missing_xcto = (success_df["x_content_type_options"].fillna("").astype(str).str.strip() == "").sum() if "x_content_type_options" in success_df.columns else len(success_df) - missing_xfo = (success_df["x_frame_options"].fillna("").astype(str).str.strip() == "").sum() if "x_frame_options" in success_df.columns else len(success_df) - if missing_hsts >= len(success_df) * 0.5: - issues.append(_issue( - "Strict-Transport-Security header not set.", - priority="High", - recommendation="Add Strict-Transport-Security to enforce HTTPS.", - )) - deductions.append((15, True)) - if missing_xcto >= len(success_df) * 0.5: - issues.append(_issue( - "X-Content-Type-Options header not set.", - priority="Medium", - recommendation="Add X-Content-Type-Options: nosniff.", - )) - deductions.append((5, True)) - if missing_xfo >= len(success_df) * 0.5: - issues.append(_issue( - "X-Frame-Options header not set.", - priority="Medium", - recommendation="Add X-Frame-Options: DENY or SAMEORIGIN.", - )) - deductions.append((5, True)) - - if "mixed_content_count" in success_df.columns: - mixed = success_df["mixed_content_count"].fillna(0).astype(int).sum() - scheme = (parsed.scheme or "").lower() - if mixed > 0 and scheme == "https": - issues.append(_issue( - f"Mixed content: {int(mixed)} HTTP resource(s) on HTTPS pages.", - priority="High", - recommendation="Load all resources over HTTPS to avoid mixed content.", - )) - deductions.append((15, True)) - - # Merge vulnerability scan findings (same format as issues: message, url, priority, recommendation) if security_findings: for f in security_findings: severity = f.get("severity", "Medium") diff --git a/src/website_profiling/reporting/categories/technical_seo.py b/src/website_profiling/reporting/categories/technical_seo.py index 725122e..87d0d20 100644 --- a/src/website_profiling/reporting/categories/technical_seo.py +++ b/src/website_profiling/reporting/categories/technical_seo.py @@ -6,6 +6,7 @@ import pandas as pd from ._helpers import ( + MAX_ISSUES_PER_CHECK, PRIORITY_ORDER, _broken_link_sources, _hreflang_issues, @@ -68,6 +69,7 @@ def category_technical_seo( # Canonical: missing or self-mismatch if "canonical_url" in df.columns and len(success_df) > 0: + missing_canon_issues = 0 for _, row in success_df.iterrows(): url = row.get("url") canon = row.get("canonical_url") @@ -76,12 +78,15 @@ def category_technical_seo( url = str(url).strip() canon = "" if pd.isna(canon) else str(canon).strip() if not canon: + if missing_canon_issues >= MAX_ISSUES_PER_CHECK: + continue issues.append(_issue("Missing canonical URL.", url=url, priority="Medium", recommendation="Add a canonical link tag pointing to the preferred URL.")) - break + missing_canon_issues += 1 missing_canon = success_df["canonical_url"].fillna("").astype(str).str.strip().eq("").sum() if missing_canon > 0: deductions.append((min(15, missing_canon * 2), True)) - # Self-canonical mismatch: canonical points to different URL + cross_canon_issues = 0 + cross_canon_count = 0 for _, row in success_df.iterrows(): url = row.get("url") canon = row.get("canonical_url") @@ -89,10 +94,15 @@ def category_technical_seo( continue url = str(url).rstrip("/") canon = str(canon).strip().rstrip("/") - if url != canon: - issues.append(_issue(f"Canonical points to different URL: {canon}", url=url, priority="High", recommendation="Set canonical to this page URL or the preferred duplicate.")) - deductions.append((10, True)) - break + if url == canon: + continue + cross_canon_count += 1 + if cross_canon_issues >= MAX_ISSUES_PER_CHECK: + continue + issues.append(_issue(f"Canonical points to different URL: {canon}", url=url, priority="High", recommendation="Set canonical to this page URL or the preferred duplicate.")) + cross_canon_issues += 1 + if cross_canon_count > 0: + deductions.append((min(10, cross_canon_count * 2), True)) # Noindex on important pages (CSV may store True/False as strings) if "noindex" in df.columns and len(success_df) > 0: diff --git a/src/website_profiling/reporting/seo_summary.py b/src/website_profiling/reporting/seo_summary.py index e930063..bc73e1f 100644 --- a/src/website_profiling/reporting/seo_summary.py +++ b/src/website_profiling/reporting/seo_summary.py @@ -3,12 +3,13 @@ import pandas as pd +from .thin_content_helper import THIN_CONTENT_WORDS, count_thin_rows, is_thin_row, thin_content_message + # SEO thresholds for recommendations TITLE_LEN_MIN = 30 TITLE_LEN_MAX = 60 META_DESC_LEN_MIN = 70 META_DESC_LEN_MAX = 160 -THIN_CONTENT_CHARS = 300 def _status_text(value: object) -> str: @@ -40,15 +41,17 @@ def _compute_summary_seo_issues(df: pd.DataFrame) -> dict: count_error = int((status_str.isin(["error", "blocked_by_robots"])).sum()) success_rate = round(100 * count_2xx / total, 1) if total else 0 + success_df = df[status_str.str.match(r"2\d{2}", na=False)] if "status" in df.columns else pd.DataFrame() + outlinks = ( - pd.to_numeric(df["outlinks"], errors="coerce").fillna(0).astype(int) - if "outlinks" in df.columns - else pd.Series([0] * len(df)) + pd.to_numeric(success_df["outlinks"], errors="coerce").fillna(0).astype(int) + if "outlinks" in success_df.columns and len(success_df) > 0 + else pd.Series([0] * len(success_df), dtype=int) ) title_len = ( - df["title"].fillna("").astype(str).apply(len) - if "title" in df.columns - else pd.Series([0] * len(df)) + success_df["title"].fillna("").astype(str).apply(len) + if "title" in success_df.columns and len(success_df) > 0 + else pd.Series([], dtype=int) ) crawl_time_s = float(df["crawl_time_s"].iloc[0]) if "crawl_time_s" in df.columns and len(df) else None @@ -60,33 +63,32 @@ def _compute_summary_seo_issues(df: pd.DataFrame) -> dict: "count_5xx": count_5xx, "count_error": count_error, "success_rate": success_rate, - "avg_outlinks": round(float(outlinks.mean()), 1) if total else 0, - "avg_title_len": round(float(title_len.mean()), 1) if total else 0, + "avg_outlinks": round(float(outlinks.mean()), 1) if len(outlinks) else 0, + "avg_title_len": round(float(title_len.mean()), 1) if len(title_len) else 0, "crawl_time_s": round(crawl_time_s, 1) if crawl_time_s is not None else None, } - # SEO health (when columns exist) + # SEO health counts only on successful (2xx) pages seo_health = {} - if "title" in df.columns: - titles = df["title"].fillna("").astype(str) + if "title" in df.columns and len(success_df) > 0: + titles = success_df["title"].fillna("").astype(str) seo_health["missing_title"] = int((titles.str.len() == 0).sum()) seo_health["title_short"] = int(((title_len > 0) & (title_len < TITLE_LEN_MIN)).sum()) seo_health["title_long"] = int((title_len > TITLE_LEN_MAX).sum()) seo_health["title_ok"] = int(((title_len >= TITLE_LEN_MIN) & (title_len <= TITLE_LEN_MAX)).sum()) - if "meta_description_len" in df.columns: - md_len = pd.to_numeric(df["meta_description_len"], errors="coerce").fillna(0).astype(int) + if "meta_description_len" in df.columns and len(success_df) > 0: + md_len = pd.to_numeric(success_df["meta_description_len"], errors="coerce").fillna(0).astype(int) seo_health["missing_meta_desc"] = int((md_len == 0).sum()) seo_health["meta_desc_short"] = int(((md_len > 0) & (md_len < META_DESC_LEN_MIN)).sum()) seo_health["meta_desc_long"] = int((md_len > META_DESC_LEN_MAX).sum()) seo_health["meta_desc_ok"] = int(((md_len >= META_DESC_LEN_MIN) & (md_len <= META_DESC_LEN_MAX)).sum()) - if "h1_count" in df.columns: - h1c = pd.to_numeric(df["h1_count"], errors="coerce").fillna(-1).astype(int) + if "h1_count" in df.columns and len(success_df) > 0: + h1c = pd.to_numeric(success_df["h1_count"], errors="coerce").fillna(-1).astype(int) seo_health["h1_zero"] = int((h1c == 0).sum()) seo_health["h1_one"] = int((h1c == 1).sum()) seo_health["h1_multi"] = int((h1c > 1).sum()) - if "content_length" in df.columns: - cl = pd.to_numeric(df["content_length"], errors="coerce").fillna(0).astype(int) - seo_health["thin_content"] = int(((cl > 0) & (cl < THIN_CONTENT_CHARS)).sum()) + if ("word_count" in df.columns or "content_length" in df.columns) and len(success_df) > 0: + seo_health["thin_content"] = count_thin_rows(success_df, success_only=False) # Issues: broken, redirects, SEO issues = {"broken": [], "redirects": [], "seo": []} @@ -103,7 +105,7 @@ def _compute_summary_seo_issues(df: pd.DataFrame) -> dict: issues["redirects"].append({"url": u, "status": st, "final_url": str(final) if pd.notna(final) else ""}) if "title" in df.columns: - for _, row in df.iterrows(): + for _, row in success_df.iterrows(): u = row.get("url") if pd.isna(u): continue @@ -117,7 +119,7 @@ def _compute_summary_seo_issues(df: pd.DataFrame) -> dict: elif tl > TITLE_LEN_MAX: issues["seo"].append({"type": "title_long", "url": u, "message": f"Title too long ({tl} chars)"}) if "meta_description_len" in df.columns: - for _, row in df.iterrows(): + for _, row in success_df.iterrows(): md_len = pd.to_numeric(row.get("meta_description_len"), errors="coerce") if pd.isna(md_len) or md_len == 0: continue @@ -131,7 +133,7 @@ def _compute_summary_seo_issues(df: pd.DataFrame) -> dict: elif ml > META_DESC_LEN_MAX: issues["seo"].append({"type": "meta_desc_long", "url": u, "message": f"Meta description too long ({ml} chars)"}) if "h1_count" in df.columns: - for _, row in df.iterrows(): + for _, row in success_df.iterrows(): h1c = pd.to_numeric(row.get("h1_count"), errors="coerce") if pd.isna(h1c) or h1c == 1: continue @@ -143,16 +145,18 @@ def _compute_summary_seo_issues(df: pd.DataFrame) -> dict: issues["seo"].append({"type": "h1_missing", "url": u, "message": "Missing H1"}) else: issues["seo"].append({"type": "h1_multi", "url": u, "message": f"Multiple H1s ({int(h1c)})"}) - if "content_length" in df.columns: - for _, row in df.iterrows(): - cl = pd.to_numeric(row.get("content_length"), errors="coerce") - cl = 0 if pd.isna(cl) else int(cl) - if cl >= THIN_CONTENT_CHARS or cl == 0: + if "word_count" in df.columns or "content_length" in df.columns: + for _, row in success_df.iterrows(): + if not is_thin_row(row): continue u = row.get("url") if pd.isna(u): continue - issues["seo"].append({"type": "thin_content", "url": str(u).strip(), "message": f"Thin content ({int(cl)} chars)"}) + issues["seo"].append({ + "type": "thin_content", + "url": str(u).strip(), + "message": thin_content_message(row), + }) # Recommendations (actionable bullets) recommendations = [] @@ -175,7 +179,9 @@ def _compute_summary_seo_issues(df: pd.DataFrame) -> dict: if seo_health.get("h1_multi", 0) > 0: recommendations.append(f"Use a single H1 per page on {seo_health['h1_multi']} page(s).") if seo_health.get("thin_content", 0) > 0: - recommendations.append(f"Expand thin content on {seo_health['thin_content']} page(s) (under {THIN_CONTENT_CHARS} chars).") + recommendations.append( + f"Expand thin content on {seo_health['thin_content']} page(s) (under {THIN_CONTENT_WORDS} words)." + ) return { "summary": summary, diff --git a/src/website_profiling/reporting/thin_content_helper.py b/src/website_profiling/reporting/thin_content_helper.py new file mode 100644 index 0000000..0251d5e --- /dev/null +++ b/src/website_profiling/reporting/thin_content_helper.py @@ -0,0 +1,41 @@ +"""Shared thin-content detection (200 words; char fallback when word_count absent).""" +from __future__ import annotations + +from typing import Any + +import pandas as pd + +THIN_CONTENT_WORDS = 200 + + +def is_thin_row(row: pd.Series) -> bool: + wc = pd.to_numeric(row.get("word_count"), errors="coerce") + if pd.notna(wc) and int(wc) > 0: + return int(wc) < THIN_CONTENT_WORDS + cl = pd.to_numeric(row.get("content_length"), errors="coerce") + if pd.isna(cl): + return False + chars = int(cl) + return chars > 0 and chars // 5 < THIN_CONTENT_WORDS + + +def thin_content_message(row: pd.Series) -> str: + wc = pd.to_numeric(row.get("word_count"), errors="coerce") + if pd.notna(wc) and int(wc) > 0: + return f"Thin content ({int(wc)} words)" + cl = pd.to_numeric(row.get("content_length"), errors="coerce") + chars = 0 if pd.isna(cl) else int(cl) + return f"Thin content (~{chars // 5} words, from {chars} chars)" + + +def count_thin_rows(df: pd.DataFrame, *, success_only: bool = True) -> int: + if df.empty: + return 0 + target = df + if success_only and "status" in df.columns: + target = df[df["status"].astype(str).str.match(r"2\d{2}", na=False)] + if target.empty: + return 0 + if "word_count" in target.columns or "content_length" in target.columns: + return int(sum(is_thin_row(row) for _, row in target.iterrows())) + return 0 diff --git a/src/website_profiling/scoring.py b/src/website_profiling/scoring.py index dbcb276..6dcedb5 100644 --- a/src/website_profiling/scoring.py +++ b/src/website_profiling/scoring.py @@ -2,8 +2,59 @@ from __future__ import annotations import math +from typing import Any + +WEIGHTS: dict[str, float] = { + "technical_seo": 0.25, + "link_health": 0.20, + "performance": 0.15, + "security": 0.15, + "core_web_vitals": 0.10, + "mobile": 0.10, + "html_accessibility": 0.05, +} + +EXCLUDED = frozenset({"search_performance", "intelligence"}) def round_half_up(value: float) -> int: """Round to nearest integer, halves away from zero (not banker's rounding).""" return math.floor(value + 0.5) + + +def site_health_score_from_categories(categories: list[Any] | None) -> int | None: + weighted_sum = 0.0 + weight_total = 0.0 + by_id: dict[str, float] = {} + for cat in categories or []: + if not isinstance(cat, dict): + continue + cat_id = str(cat.get("id") or "") + score = cat.get("score") + if not cat_id or cat_id in EXCLUDED or not isinstance(score, (int, float)): + continue + by_id[cat_id] = float(score) + for cat_id, weight in WEIGHTS.items(): + score = by_id.get(cat_id) + if score is None: + continue + weighted_sum += score * weight + weight_total += weight + if weight_total <= 0: + return None + return round_half_up(weighted_sum / weight_total) + + +def site_health_score_from_payload(report_data: dict[str, Any]) -> int | None: + summary = report_data.get("summary") + if isinstance(summary, dict): + score = summary.get("site_health_score") + if isinstance(score, (int, float)): + return round_half_up(float(score)) + top = report_data.get("site_health_score") + if isinstance(top, (int, float)): + return round_half_up(float(top)) + categories = report_data.get("categories") + if isinstance(categories, list): + return site_health_score_from_categories(categories) + return None diff --git a/tests/reporting/test_categories_coverage.py b/tests/reporting/test_categories_coverage.py index 072cbbc..8bafb32 100644 --- a/tests/reporting/test_categories_coverage.py +++ b/tests/reporting/test_categories_coverage.py @@ -400,6 +400,14 @@ def test_category_core_web_vitals_from_lighthouse_crux_inp_cls_failures() -> Non crux = {"ok": True, "pass": {"lcp": True, "inp": False, "cls": False}} cat = category_core_web_vitals_from_lighthouse(lh, crux) assert len([i for i in cat["issues"] if "CrUX" in i["message"]]) == 2 + assert cat["score"] == 60 + + +def test_category_core_web_vitals_from_lighthouse_crux_all_fail_deducts_score() -> None: + lh = {"median_metrics": {"performance_score": 0.95}, "top_failures": []} + crux = {"ok": True, "pass": {"lcp": False, "inp": False, "cls": False}} + cat = category_core_web_vitals_from_lighthouse(lh, crux) + assert cat["score"] == 50 def test_build_categories_without_lighthouse() -> None: @@ -438,6 +446,20 @@ def test_category_performance_slow_response_and_p95() -> None: def test_category_performance_lazy_load_img_cache_scripts() -> None: rows = [ + { + "url": "https://example.com/", + "status": "200", + "response_time_ms": 100, + "images_total": 20, + "img_without_lazy": 7, + } + ] + cat = category_performance(pd.DataFrame(rows)) + msgs = " ".join(i["message"].lower() for i in cat["issues"]) + assert "lazy loading" in msgs + assert cat["score"] == 97 + + rows_full = [ { "url": f"https://example.com/{i}", "status": "200", @@ -450,12 +472,11 @@ def test_category_performance_lazy_load_img_cache_scripts() -> None: } for i in range(2) ] - cat = category_performance(pd.DataFrame(rows)) - msgs = " ".join(i["message"].lower() for i in cat["issues"]) - assert "lazy loading" in msgs - assert "without width/height" in msgs - assert "cache-control" in msgs - assert "script tags" in msgs + cat_full = category_performance(pd.DataFrame(rows_full)) + msgs_full = " ".join(i["message"].lower() for i in cat_full["issues"]) + assert "without width/height" in msgs_full + assert "cache-control" in msgs_full + assert "script tags" in msgs_full # --------------------------------------------------------------------------- @@ -523,7 +544,7 @@ def test_category_html_accessibility_score_zero_floor() -> None: return_value=0, ): cat = category_html_accessibility(df) - assert cat["score"] == 5 + assert cat["score"] == 0 # --------------------------------------------------------------------------- @@ -589,6 +610,8 @@ def test_category_mobile_viewport_missing_and_invalid() -> None: def test_category_security_headers_mixed_content_findings() -> None: + from website_profiling.security_scanner import run_security_scan + df = pd.DataFrame([ { "url": "https://example.com/", @@ -600,7 +623,8 @@ def test_category_security_headers_mixed_content_findings() -> None: "mixed_content_count": 2, }, ]) - findings = [ + findings = run_security_scan(df, "https://example.com/", max_urls_to_probe=0) + findings.extend([ { "severity": "Critical", "finding_type": "sql_injection", @@ -609,7 +633,7 @@ def test_category_security_headers_mixed_content_findings() -> None: "recommendation": "Sanitize inputs", }, {"severity": "Unknown", "message": "Minor issue", "url": "", "recommendation": ""}, - ] + ]) cat = category_security(df, {}, "https://example.com/", findings) msgs = " ".join(i["message"].lower() for i in cat["issues"]) assert "strict-transport-security" in msgs diff --git a/tests/reporting/test_content_analytics_bins.py b/tests/reporting/test_content_analytics_bins.py index 8148016..27c9645 100644 --- a/tests/reporting/test_content_analytics_bins.py +++ b/tests/reporting/test_content_analytics_bins.py @@ -21,6 +21,6 @@ def test_content_ratio_distribution_has_no_bin_gaps() -> None: assert sum(dist.values()) == 4 assert dist["<10%"] == 0 - assert dist["10-20%"] == 2 - assert dist["20-40%"] == 1 + assert dist["10-20%"] == 3 + assert dist["20-40%"] == 0 assert dist[">40%"] == 1 diff --git a/tests/reporting/test_contrast_issues.py b/tests/reporting/test_contrast_issues.py index 92b15bf..f49441d 100644 --- a/tests/reporting/test_contrast_issues.py +++ b/tests/reporting/test_contrast_issues.py @@ -18,7 +18,12 @@ def test_contrast_from_axe_violations(): "id": "color-contrast", "description": "Elements must have sufficient color contrast", "help": "Fix contrast", - } + }, + { + "id": "color-contrast", + "description": "Second contrast failure", + "help": "Fix contrast", + }, ] } df = pd.DataFrame([ @@ -32,7 +37,7 @@ def test_contrast_from_axe_violations(): } ]) issues = contrast_issues_from_sources(df, {}) - assert len(issues) == 1 + assert len(issues) == 2 assert "axe" in issues[0]["message"].lower() assert issues[0]["url"] == "https://ex.com/page" diff --git a/tests/reporting/test_pipeline_report_pool_unit.py b/tests/reporting/test_pipeline_report_pool_unit.py index 55479e1..cc642ba 100644 --- a/tests/reporting/test_pipeline_report_pool_unit.py +++ b/tests/reporting/test_pipeline_report_pool_unit.py @@ -2,6 +2,7 @@ from __future__ import annotations import argparse +import json import types import pandas as pd @@ -221,6 +222,22 @@ def test_parse_tech_stack_and_wappalyzer_fallbacks(monkeypatch): stack = parse_tech_stack(soup, {"Server": "nginx", "cf-ray": "x"}, "https://a.com") assert "Next.js" in stack or "Drupal" in stack + hugo_html = ( + '' + '' + '' + ) + hugo_soup = BeautifulSoup(hugo_html, "lxml") + hugo_stack = json.loads(parse_tech_stack( + hugo_soup, + {"Server": "GitHub.com"}, + "https://codefrydev.in/", + )) + assert "Hugo" in hugo_stack + assert "Google Tag Manager" in hugo_stack + assert "Microsoft Clarity" in hugo_stack + assert "GitHub Pages" in hugo_stack + class FakeW: def analyze_with_versions_and_categories(self, _web): return {"WordPress": {"versions": [], "categories": ["CMS"]}} diff --git a/tests/test_keyword_enrich.py b/tests/test_keyword_enrich.py index f5bd465..35cfac4 100644 --- a/tests/test_keyword_enrich.py +++ b/tests/test_keyword_enrich.py @@ -30,6 +30,11 @@ def test_opportunity_clicks_boundary_position_three() -> None: assert opportunity_clicks(1000, 3.0, target_pos=3) == 0 -def test_industry_ctr_uses_ceil() -> None: - assert industry_ctr(2.1) == CTR_CURVE[3] +def test_industry_ctr_interpolates_between_slots() -> None: + assert industry_ctr(3.0) == pytest.approx(CTR_CURVE[3]) + assert industry_ctr(3.1) == pytest.approx(0.100, rel=1e-3) + assert industry_ctr(3.9) == pytest.approx(0.076, rel=1e-3) + + +def test_industry_ctr_exact_slot() -> None: assert industry_ctr(3.0) == CTR_CURVE[3] diff --git a/tests/test_scoring.py b/tests/test_scoring.py index ef259da..2399978 100644 --- a/tests/test_scoring.py +++ b/tests/test_scoring.py @@ -1,9 +1,37 @@ """Tests for scoring helpers.""" from __future__ import annotations -from website_profiling.scoring import round_half_up +from website_profiling.scoring import ( + round_half_up, + site_health_score_from_categories, + site_health_score_from_payload, +) def test_round_half_up_away_from_bankers_rounding() -> None: assert round_half_up(49.5) == 50 assert round_half_up(50.5) == 51 + + +def test_site_health_score_from_categories_weighted_fixture() -> None: + categories = [ + {"id": "technical_seo", "score": 80}, + {"id": "link_health", "score": 60}, + {"id": "performance", "score": 70}, + {"id": "security", "score": 90}, + {"id": "core_web_vitals", "score": 50}, + {"id": "mobile", "score": 40}, + {"id": "html_accessibility", "score": 100}, + {"id": "search_performance", "score": 10}, + {"id": "intelligence", "score": 0}, + ] + assert site_health_score_from_categories(categories) == 70 + + +def test_site_health_score_from_payload_prefers_summary() -> None: + payload = { + "summary": {"site_health_score": 72}, + "site_health_score": 65, + "categories": [{"id": "technical_seo", "score": 10}], + } + assert site_health_score_from_payload(payload) == 72 diff --git a/web/src/components/overview/OverviewSummaryTab.tsx b/web/src/components/overview/OverviewSummaryTab.tsx index 42b980e..eb6980d 100644 --- a/web/src/components/overview/OverviewSummaryTab.tsx +++ b/web/src/components/overview/OverviewSummaryTab.tsx @@ -62,7 +62,7 @@ export function OverviewSummaryTab({ .filter((iss) => iss.priority === 'Critical' || iss.priority === 'High') .slice(0, 3); return { currentHealth: health, topIssues: exec.length > 0 ? exec : fallback }; - }, [data.categories, data.executive_summary]); + }, [data]); const googleData = data.google; const googleSnap = googleSnapshotStatus(googleData); diff --git a/web/src/lib/dashboard/engine/datasets.test.ts b/web/src/lib/dashboard/engine/datasets.test.ts index 2d0300d..8a4bc70 100644 --- a/web/src/lib/dashboard/engine/datasets.test.ts +++ b/web/src/lib/dashboard/engine/datasets.test.ts @@ -2,7 +2,7 @@ import { describe, it, expect } from 'vitest'; import { DATASETS, datasetById, datasetsByGroup } from '@/lib/dashboard/engine/datasets'; import { SECTION_KEYS } from '@/lib/reportSections'; import { measureLabel } from '@/lib/dashboard/engine/runQuery'; -import type { ReportPayload } from '@/types/report'; +import type { ReportCategory, ReportPayload } from '@/types/report'; /** A payload that touches every section a dataset reads from. (Cast: real link * rows carry more fields than the partial ReportLink interface declares.) */ @@ -99,6 +99,25 @@ describe('dataset registry integrity', () => { expect(datasetById.get('links')!.accessor(PAYLOAD)[0].host).toBe('x.com'); }); + it('summary health_score falls back to weighted categories when payload field missing', () => { + const weightedCategories: ReportCategory[] = [ + { id: 'technical_seo', name: 'Technical SEO', score: 80, issues: [] }, + { id: 'link_health', name: 'Link Health', score: 60, issues: [] }, + { id: 'performance', name: 'Performance', score: 70, issues: [] }, + { id: 'security', name: 'Security', score: 90, issues: [] }, + { id: 'core_web_vitals', name: 'CWV', score: 50, issues: [] }, + { id: 'mobile', name: 'Mobile', score: 40, issues: [] }, + { id: 'html_accessibility', name: 'A11y', score: 100, issues: [] }, + { id: 'search_performance', name: 'Search', score: 10, issues: [] }, + { id: 'intelligence', name: 'Intel', score: 0, issues: [] }, + ]; + const legacyPayload = { + portfolio_benchmark: { property_health_score: 78 }, + categories: weightedCategories, + } as unknown as ReportPayload; + expect(datasetById.get('summary')!.accessor(legacyPayload)[0].health_score).toBe(70); + }); + it('measureLabel falls back to agg(field)', () => { expect(measureLabel({ field: 'x', agg: 'sum' })).toBe('sum(x)'); expect(measureLabel({ field: 'x', agg: 'sum', label: 'Total' })).toBe('Total'); diff --git a/web/src/lib/dashboard/engine/datasets.ts b/web/src/lib/dashboard/engine/datasets.ts index 3623b4a..8a8e9e4 100644 --- a/web/src/lib/dashboard/engine/datasets.ts +++ b/web/src/lib/dashboard/engine/datasets.ts @@ -5,6 +5,7 @@ */ import type { ReportPayload, ReportCategory, LighthousePageSummary } from '@/types/report'; import type { DatasetDef, VizType } from '@/lib/dashboard/engine/types'; +import { siteHealthScoreFromPayload } from '@/lib/siteHealthScore'; import { fromParallel, fromMap, @@ -53,11 +54,7 @@ export const DATASETS: DatasetDef[] = [ ...flatPrefix('ga4', d.google?.ga4?.summary as Record | undefined), ...flatPrefix('lh', d.lighthouse_summary?.median_metrics as Record | undefined), ...flatPrefix('social', d.social_coverage as Record | undefined), - health_score: - d.summary?.site_health_score - ?? d.site_health_score - ?? d.portfolio_benchmark?.property_health_score - ?? null, + health_score: siteHealthScoreFromPayload(d), }, ], fields: [ diff --git a/web/src/strings.json b/web/src/strings.json index 2c75ebf..1d8b6c6 100644 --- a/web/src/strings.json +++ b/web/src/strings.json @@ -39,7 +39,7 @@ }, "healthScore": { "title": "Site health", - "body": "Score from 0-100 averaged across this audit's category scores. Higher is better. Mainly reflects technical SEO health; also includes real Search Console performance (rankings, CTR, trends) when Google is connected." + "body": "Weighted score from 0–100 across fixable audit categories (Technical SEO, Link health, Performance, Security, Core Web Vitals, Mobile, Accessibility). Excludes Search performance and Intelligence. Higher is better." }, "impactScore": { "title": "Impact score", @@ -2506,6 +2506,7 @@ "categoryOther": "Other", "noSearchMatch": "No technologies match your search.", "noData": "No technology data yet. Run a crawl from Run audit first.", + "noTechnologiesDetected": "Pages were crawled but no known technologies matched. Re-run the audit to refresh fingerprints after engine updates.", "breakdownTitle": "Technology Breakdown", "colTechnology": "Technology", "colCategory": "Category", @@ -4175,10 +4176,10 @@ "lhCategoryHint": "From the attached Lighthouse summary (typically the audited landing or representative URL). Scores are 0–100; bars use the same good / needs work / poor coloring as elsewhere.", "healthByCategory": "Health by Category", "portfolioBenchmarkTitle": "Portfolio benchmark", - "portfolioBenchmarkHint": "Compare this property’s average category score to the median across all properties on this instance.", - "portfolioBenchmarkSubtitle": "See whether this site’s average audit health is ahead or behind your portfolio median.", + "portfolioBenchmarkHint": "Compare this property's site health score to the median across all properties on this instance.", + "portfolioBenchmarkSubtitle": "See whether this site's weighted audit health is ahead or behind your portfolio median.", "portfolioBenchmarkHelpTitle": "About portfolio benchmark", - "portfolioBenchmarkHelpBody": "We average category scores (Technical SEO, Performance, etc.) for this property and compare to the median across all properties on this instance.", + "portfolioBenchmarkHelpBody": "Site health uses a weighted blend of fixable category scores (excluding Search performance and Intelligence). We compare this property's score to the median across all properties on this instance.", "portfolioPropertyScore": "This property", "portfolioMedianScore": "Portfolio median", "portfolioDelta": "Vs median", @@ -4189,7 +4190,7 @@ "portfolioEvenMedian": "Matches portfolio median", "portfolioViewPortfolio": "View portfolio", "portfolioCompareRuns": "Compare audit runs", - "portfolioGaugeLabel": "Average category score", + "portfolioGaugeLabel": "Site health score", "portfolioNoBenchmarkLabel": "Your site health (no benchmark yet)", "portfolioSinglePropertyCta": "Add another property", "portfolioScrollCategories": "Review categories below", @@ -4336,9 +4337,9 @@ "pagesGoodRanges": "Pages in “good” ranges", "stackedInRange": "In range / optimal", "stackedNeeds": "Needs attention", - "thinSignals": "Thin content signals", - "thinUnder300": "Under 300 words", - "thinSmallBody": "SEO flagged (<200 words)", + "thinSignals": "Thin content comparison", + "thinUnder300": "Analytics list (<300 words)", + "thinSmallBody": "SEO health flag (<200 words)", "thinMatchSearch": "{shown} of {total} thin page{plural} match search", "thinHaveLittle": "{count} page{plural} have very little content", "noThinSearch": "No thin pages match your search.", diff --git a/web/src/views/ContentAnalytics.tsx b/web/src/views/ContentAnalytics.tsx index faa4602..6bc25ac 100644 --- a/web/src/views/ContentAnalytics.tsx +++ b/web/src/views/ContentAnalytics.tsx @@ -39,7 +39,6 @@ import { Sparkles, Link2, LayoutDashboard, - BarChart2 as BarChart2Icon, } from 'lucide-react'; import { useReport } from '../context/useReport'; import { strings, format } from '../lib/strings'; @@ -282,7 +281,7 @@ export default function ContentAnalytics({ searchQuery = '' }: ViewProps) { { id: 'analytics', label: vca.tabs.analytics, - icon: , + icon: , }, ], [vca.tabs]); diff --git a/web/src/views/TechStack.tsx b/web/src/views/TechStack.tsx index 7e3f284..2fc05cb 100644 --- a/web/src/views/TechStack.tsx +++ b/web/src/views/TechStack.tsx @@ -43,11 +43,11 @@ const barValueLabelsPlugin = { }; const TECH_CATEGORIES: Record = { - CMS: ['WordPress', 'Drupal', 'Joomla', 'Shopify', 'Squarespace', 'Wix'], + CMS: ['WordPress', 'Drupal', 'Joomla', 'Hugo', 'Jekyll', 'Shopify', 'Squarespace', 'Wix'], 'JS Frameworks': ['React', 'Next.js', 'Vue.js', 'Nuxt.js', 'Angular', 'Svelte', 'Gatsby', 'jQuery'], 'CSS Frameworks': ['Bootstrap', 'Tailwind CSS'], - Analytics: ['Google Analytics', 'Google Tag Manager', 'Facebook Pixel', 'Hotjar'], - Infrastructure: ['Cloudflare', 'Nginx', 'Apache', 'LiteSpeed', 'Vercel', 'Netlify', 'Amazon CloudFront', 'AWS'], + Analytics: ['Google Analytics', 'Google Tag Manager', 'Facebook Pixel', 'Hotjar', 'Microsoft Clarity'], + Infrastructure: ['Cloudflare', 'Nginx', 'Apache', 'LiteSpeed', 'Vercel', 'Netlify', 'GitHub Pages', 'Amazon CloudFront', 'AWS'], Fonts: ['Google Fonts', 'Font Awesome'], }; @@ -103,6 +103,12 @@ export default function TechStack({ searchQuery = '' }: ViewProps) { } const totalAnalyzed = ts.total_pages_analyzed || 0; + const emptyMessage = + (ts.technologies || []).length > 0 + ? vr.noSearchMatch + : totalAnalyzed > 0 + ? vr.noTechnologiesDetected + : vr.noData; const chartLabels = techs.map((t) => t.name); const chartValues = techs.map((t) => t.count); @@ -171,7 +177,7 @@ export default function TechStack({ searchQuery = '' }: ViewProps) { ) : (ts.technologies || []).length > 0 ? (
{vr.noSearchMatch}
) : ( -
{vr.noData}
+
{emptyMessage}
)}
@@ -211,7 +217,7 @@ export default function TechStack({ searchQuery = '' }: ViewProps) { ) : ( - {(ts.technologies || []).length > 0 ? vr.noSearchMatch : vr.noData} + {techs.length === 0 ? emptyMessage : vr.noSearchMatch} )} From a0a3e5723476c798bb27a16e37cebf6f6c65c2f7 Mon Sep 17 00:00:00 2001 From: PrashantUnity Date: Sat, 27 Jun 2026 18:08:13 +0530 Subject: [PATCH 5/8] after all will be ui Polishing --- .../Build/BrowserDiagnosticsAggregator.cs | 126 +++++++++-- .../Build/BrowserDiagnosticsHelper.cs | 205 +++++++++++++---- .../Build/LinksListBuilder.cs | 4 +- .../BrowserDiagnosticsAggregatorTests.cs | 5 + .../BrowserDiagnosticsHelperTests.cs | 26 +++ .../crawl/fetchers/browser_diagnostics.py | 46 ++++ web/src/lib/browserErrors.test.ts | 134 +++++++++++- web/src/lib/browserErrors.ts | 138 ++++++++++-- web/src/lib/buildBrowserErrorsPrompt.ts | 2 + web/src/strings.json | 17 +- web/src/types/report.ts | 3 + web/src/views/JavaScriptErrors.tsx | 206 +++++++++++------- 12 files changed, 745 insertions(+), 167 deletions(-) diff --git a/services/ReportService/src/ReportService.Application/Build/BrowserDiagnosticsAggregator.cs b/services/ReportService/src/ReportService.Application/Build/BrowserDiagnosticsAggregator.cs index 1a994a6..e8c10a5 100644 --- a/services/ReportService/src/ReportService.Application/Build/BrowserDiagnosticsAggregator.cs +++ b/services/ReportService/src/ReportService.Application/Build/BrowserDiagnosticsAggregator.cs @@ -24,9 +24,12 @@ private sealed class MessageBucket var pagesWithConsoleErrors = 0; var pagesWithPageErrors = 0; + var pagesWithFailedRequests = 0; var totalConsoleErrors = 0; var totalPageErrors = 0; + var totalFailedRequests = 0; var messageCounts = new Dictionary(StringComparer.Ordinal); + var exceptionCounts = new Dictionary(StringComparer.Ordinal); foreach (var row in rows) { @@ -50,46 +53,60 @@ private sealed class MessageBucket totalPageErrors += counts.PageErrorCount; } + if (counts.FailedRequestCount > 0) + { + pagesWithFailedRequests++; + totalFailedRequests += counts.FailedRequestCount; + } + AccumulateConsoleMessages(pa, url, messageCounts); + AccumulatePageErrors(pa, url, exceptionCounts); } if (pagesWithConsoleErrors == 0 && pagesWithPageErrors == 0 + && pagesWithFailedRequests == 0 && totalConsoleErrors == 0 - && totalPageErrors == 0) + && totalPageErrors == 0 + && totalFailedRequests == 0) { return null; } - var topConsoleMessages = messageCounts.Values - .OrderByDescending(b => b.Count) - .Take(5) - .Select(b => (Dictionary)new Dictionary - { - ["text"] = b.Text, - ["count"] = b.Count, - ["sample_urls"] = b.SampleUrls, - }) - .ToList(); + var topConsoleMessages = TopBuckets(messageCounts); + var topPageErrors = TopBuckets(exceptionCounts); return new Dictionary { ["pages_with_console_errors"] = pagesWithConsoleErrors, ["pages_with_page_errors"] = pagesWithPageErrors, + ["pages_with_failed_requests"] = pagesWithFailedRequests, ["total_console_errors"] = totalConsoleErrors, ["total_page_errors"] = totalPageErrors, + ["total_failed_requests"] = totalFailedRequests, ["top_console_messages"] = topConsoleMessages, + ["top_page_errors"] = topPageErrors, }; } + private static List> TopBuckets(Dictionary buckets) => + buckets.Values + .OrderByDescending(b => b.Count) + .Take(5) + .Select(b => (Dictionary)new Dictionary + { + ["text"] = b.Text, + ["count"] = b.Count, + ["sample_urls"] = b.SampleUrls, + }) + .ToList(); + private static void AccumulateConsoleMessages( IReadOnlyDictionary pageAnalysis, string url, Dictionary messageCounts) { - if (!pageAnalysis.TryGetValue("browser", out var browserRaw) - || browserRaw is not string browserJson - || string.IsNullOrWhiteSpace(browserJson)) + if (!TryGetBrowserJson(pageAnalysis, out var browserJson)) { return; } @@ -122,19 +139,48 @@ private static void AccumulateConsoleMessages( continue; } - if (!messageCounts.TryGetValue(text, out var bucket)) + AccumulateBucket(messageCounts, text, url); + } + } + catch (JsonException) + { + // ignore malformed browser payload + } + } + + private static void AccumulatePageErrors( + IReadOnlyDictionary pageAnalysis, + string url, + Dictionary exceptionCounts) + { + if (!TryGetBrowserJson(pageAnalysis, out var browserJson)) + { + return; + } + + try + { + using var doc = JsonDocument.Parse(browserJson); + if (!doc.RootElement.TryGetProperty("page_errors", out var pageErrors) + || pageErrors.ValueKind != JsonValueKind.Array) + { + return; + } + + foreach (var err in pageErrors.EnumerateArray()) + { + if (err.ValueKind != JsonValueKind.Object) { - bucket = new MessageBucket { Text = text }; - messageCounts[text] = bucket; + continue; } - bucket.Count++; - if (!string.IsNullOrEmpty(url) - && !bucket.SampleUrls.Contains(url, StringComparer.Ordinal) - && bucket.SampleUrls.Count < 3) + var text = err.TryGetProperty("message", out var messageEl) ? messageEl.GetString()?.Trim() : null; + if (string.IsNullOrEmpty(text)) { - bucket.SampleUrls.Add(url); + continue; } + + AccumulateBucket(exceptionCounts, text, url); } } catch (JsonException) @@ -142,4 +188,40 @@ private static void AccumulateConsoleMessages( // ignore malformed browser payload } } + + private static void AccumulateBucket( + Dictionary buckets, + string text, + string url) + { + if (!buckets.TryGetValue(text, out var bucket)) + { + bucket = new MessageBucket { Text = text }; + buckets[text] = bucket; + } + + bucket.Count++; + if (!string.IsNullOrEmpty(url) + && !bucket.SampleUrls.Contains(url, StringComparer.Ordinal) + && bucket.SampleUrls.Count < 3) + { + bucket.SampleUrls.Add(url); + } + } + + private static bool TryGetBrowserJson( + IReadOnlyDictionary pageAnalysis, + out string browserJson) + { + browserJson = ""; + if (!pageAnalysis.TryGetValue("browser", out var browserRaw) + || browserRaw is not string raw + || string.IsNullOrWhiteSpace(raw)) + { + return false; + } + + browserJson = raw; + return true; + } } diff --git a/services/ReportService/src/ReportService.Application/Build/BrowserDiagnosticsHelper.cs b/services/ReportService/src/ReportService.Application/Build/BrowserDiagnosticsHelper.cs index 0ef8e5c..e7c98a5 100644 --- a/services/ReportService/src/ReportService.Application/Build/BrowserDiagnosticsHelper.cs +++ b/services/ReportService/src/ReportService.Application/Build/BrowserDiagnosticsHelper.cs @@ -9,63 +9,199 @@ public static BrowserSummaryCounts SummaryFromPageAnalysis(IReadOnlyDictionary browserDict) { - if (browserEl.ValueKind != JsonValueKind.Object - || !browserEl.TryGetProperty("summary", out var summaryEl)) + return TryCountFromBrowserDictionary(browserDict, out counts); + } + + if (browserObj is string browserJson) + { + try + { + using var doc = JsonDocument.Parse(browserJson); + if (doc.RootElement.ValueKind == JsonValueKind.Object) + { + return TryCountFromBrowserElement(doc.RootElement, out counts); + } + } + catch (JsonException) { return false; } + } - counts = ReadSummaryElement(summaryEl); + return false; + } + + private static bool TryCountFromBrowserDictionary( + IReadOnlyDictionary browser, + out BrowserSummaryCounts counts) + { + counts = new BrowserSummaryCounts(0, 0, 0); + var hasArrayData = false; + var consoleErrors = 0; + var pageErrors = 0; + var failedRequests = 0; + + if (browser.TryGetValue("console", out var consoleObj)) + { + consoleErrors = CountConsoleErrors(consoleObj); + hasArrayData = true; + } + + if (browser.TryGetValue("page_errors", out var pageErrorsObj)) + { + pageErrors = CountArrayItems(pageErrorsObj); + hasArrayData = true; + } + + if (browser.TryGetValue("failed_requests", out var failedRequestsObj)) + { + failedRequests = CountArrayItems(failedRequestsObj); + hasArrayData = true; + } + + if (hasArrayData) + { + counts = new BrowserSummaryCounts(consoleErrors, pageErrors, failedRequests); return true; } - if (browserObj is IReadOnlyDictionary browser - && browser.TryGetValue("summary", out var summaryObj)) + if (browser.TryGetValue("summary", out var summaryObj)) { counts = ReadSummaryObject(summaryObj); return true; } - if (browserObj is string browserJson && TryGetBrowserSummary(browserJson, out counts)) + return false; + } + + private static bool TryCountFromBrowserElement(JsonElement browser, out BrowserSummaryCounts counts) + { + counts = new BrowserSummaryCounts(0, 0, 0); + var hasArrayData = false; + var consoleErrors = 0; + var pageErrors = 0; + var failedRequests = 0; + + if (browser.TryGetProperty("console", out var consoleEl) && consoleEl.ValueKind == JsonValueKind.Array) + { + consoleErrors = CountConsoleErrors(consoleEl); + hasArrayData = true; + } + + if (browser.TryGetProperty("page_errors", out var pageErrorsEl) && pageErrorsEl.ValueKind == JsonValueKind.Array) + { + pageErrors = pageErrorsEl.GetArrayLength(); + hasArrayData = true; + } + + if (browser.TryGetProperty("failed_requests", out var failedRequestsEl) + && failedRequestsEl.ValueKind == JsonValueKind.Array) + { + failedRequests = failedRequestsEl.GetArrayLength(); + hasArrayData = true; + } + + if (hasArrayData) + { + counts = new BrowserSummaryCounts(consoleErrors, pageErrors, failedRequests); + return true; + } + + if (browser.TryGetProperty("summary", out var summaryEl)) { + counts = ReadSummaryElement(summaryEl); return true; } return false; } + private static int CountConsoleErrors(object? consoleObj) => + consoleObj switch + { + JsonElement { ValueKind: JsonValueKind.Array } consoleEl => CountConsoleErrors(consoleEl), + IEnumerable items => items.Count(IsConsoleErrorObject), + _ => 0, + }; + + private static int CountConsoleErrors(JsonElement console) + { + var count = 0; + foreach (var msg in console.EnumerateArray()) + { + if (msg.ValueKind != JsonValueKind.Object) + { + continue; + } + + var level = msg.TryGetProperty("level", out var levelEl) ? levelEl.GetString() : null; + if (string.Equals(level, "error", StringComparison.OrdinalIgnoreCase)) + { + count++; + } + } + + return count; + } + + private static bool IsConsoleErrorObject(object? item) + { + if (item is JsonElement el && el.ValueKind == JsonValueKind.Object) + { + var level = el.TryGetProperty("level", out var levelEl) ? levelEl.GetString() : null; + return string.Equals(level, "error", StringComparison.OrdinalIgnoreCase); + } + + if (item is IReadOnlyDictionary dict) + { + var level = dict.GetValueOrDefault("level")?.ToString(); + return string.Equals(level, "error", StringComparison.OrdinalIgnoreCase); + } + + return false; + } + + private static int CountArrayItems(object? arrayObj) => + arrayObj switch + { + JsonElement { ValueKind: JsonValueKind.Array } el => el.GetArrayLength(), + IEnumerable items => items.Count(), + _ => 0, + }; + private static BrowserSummaryCounts ReadSummaryElement(JsonElement summary) { if (summary.ValueKind != JsonValueKind.Object) { - return new BrowserSummaryCounts(0, 0); + return new BrowserSummaryCounts(0, 0, 0); } return new BrowserSummaryCounts( summary.TryGetProperty("console_error_count", out var ce) && ce.TryGetInt32(out var cei) ? cei : 0, - summary.TryGetProperty("page_error_count", out var pe) && pe.TryGetInt32(out var pei) ? pei : 0); + summary.TryGetProperty("page_error_count", out var pe) && pe.TryGetInt32(out var pei) ? pei : 0, + summary.TryGetProperty("failed_request_count", out var fr) && fr.TryGetInt32(out var fri) ? fri : 0); } private static BrowserSummaryCounts ReadSummaryObject(object? summaryObj) => @@ -74,31 +210,11 @@ private static BrowserSummaryCounts ReadSummaryObject(object? summaryObj) => JsonElement el => ReadSummaryElement(el), IReadOnlyDictionary summary => new BrowserSummaryCounts( ToInt(summary.GetValueOrDefault("console_error_count")), - ToInt(summary.GetValueOrDefault("page_error_count"))), - _ => new BrowserSummaryCounts(0, 0), + ToInt(summary.GetValueOrDefault("page_error_count")), + ToInt(summary.GetValueOrDefault("failed_request_count"))), + _ => new BrowserSummaryCounts(0, 0, 0), }; - private static bool TryGetBrowserSummary(string browserJson, out BrowserSummaryCounts counts) - { - counts = new BrowserSummaryCounts(0, 0); - try - { - using var doc = JsonDocument.Parse(browserJson); - if (!doc.RootElement.TryGetProperty("summary", out var summary) - || summary.ValueKind != JsonValueKind.Object) - { - return false; - } - - counts = ReadSummaryElement(summary); - return true; - } - catch (JsonException) - { - return false; - } - } - private static int ToInt(object? value) => value switch { @@ -112,5 +228,8 @@ string s when int.TryParse(s, out var parsed) => parsed, _ => 0, }; - public sealed record BrowserSummaryCounts(int ConsoleErrorCount, int PageErrorCount); + public sealed record BrowserSummaryCounts( + int ConsoleErrorCount, + int PageErrorCount, + int FailedRequestCount = 0); } diff --git a/services/ReportService/src/ReportService.Application/Build/LinksListBuilder.cs b/services/ReportService/src/ReportService.Application/Build/LinksListBuilder.cs index c0490b3..d5e3bea 100644 --- a/services/ReportService/src/ReportService.Application/Build/LinksListBuilder.cs +++ b/services/ReportService/src/ReportService.Application/Build/LinksListBuilder.cs @@ -100,7 +100,9 @@ public static Dictionary BuildInDegree(IReadOnlyList<(string From, ["external_link_count"] = ToInt(pageAnalysis.GetValueOrDefault("external_link_count")), ["console_error_count"] = browser.ConsoleErrorCount, ["page_error_count"] = browser.PageErrorCount, - ["has_browser_errors"] = browser.ConsoleErrorCount > 0 || browser.PageErrorCount > 0, + ["has_browser_errors"] = browser.ConsoleErrorCount > 0 + || browser.PageErrorCount > 0 + || browser.FailedRequestCount > 0, ["lighthouse"] = SerializeLighthouse(LighthouseReportMerge.LighthouseForUrl(lighthouseByUrl, url)), }; diff --git a/services/ReportService/tests/ReportService.Tests/BrowserDiagnosticsAggregatorTests.cs b/services/ReportService/tests/ReportService.Tests/BrowserDiagnosticsAggregatorTests.cs index 544ab47..14f4389 100644 --- a/services/ReportService/tests/ReportService.Tests/BrowserDiagnosticsAggregatorTests.cs +++ b/services/ReportService/tests/ReportService.Tests/BrowserDiagnosticsAggregatorTests.cs @@ -38,6 +38,11 @@ public void Aggregate_counts_pages_messages_and_top_console() Assert.NotEmpty(top); Assert.Equal("Same error", top[0]["text"]); Assert.Equal(2, top[0]["count"]); + + var topExceptions = Assert.IsType>>(agg["top_page_errors"]); + Assert.NotEmpty(topExceptions); + Assert.Equal("Uncaught", topExceptions[0]["text"]); + Assert.Equal(2, topExceptions[0]["count"]); } [Fact] diff --git a/services/ReportService/tests/ReportService.Tests/BrowserDiagnosticsHelperTests.cs b/services/ReportService/tests/ReportService.Tests/BrowserDiagnosticsHelperTests.cs index c62bd87..541557f 100644 --- a/services/ReportService/tests/ReportService.Tests/BrowserDiagnosticsHelperTests.cs +++ b/services/ReportService/tests/ReportService.Tests/BrowserDiagnosticsHelperTests.cs @@ -62,4 +62,30 @@ public void SummaryFromPageAnalysis_reads_browser_json_string() Assert.Equal(2, counts.ConsoleErrorCount); Assert.Equal(1, counts.PageErrorCount); } + + [Fact] + public void SummaryFromPageAnalysis_prefers_console_array_over_stale_summary() + { + using var doc = JsonDocument.Parse( + """ + { + "browser": { + "console": [{"level": "error", "text": "Actual error"}], + "summary": { + "console_error_count": 9, + "page_error_count": 0 + } + } + } + """); + var pageAnalysis = new Dictionary + { + ["browser"] = JsonSerializer.Deserialize>( + doc.RootElement.GetProperty("browser").GetRawText()), + }; + + var counts = BrowserDiagnosticsHelper.SummaryFromPageAnalysis(pageAnalysis); + Assert.Equal(1, counts.ConsoleErrorCount); + Assert.Equal(0, counts.PageErrorCount); + } } diff --git a/src/website_profiling/crawl/fetchers/browser_diagnostics.py b/src/website_profiling/crawl/fetchers/browser_diagnostics.py index 3f55130..a6e5eea 100644 --- a/src/website_profiling/crawl/fetchers/browser_diagnostics.py +++ b/src/website_profiling/crawl/fetchers/browser_diagnostics.py @@ -63,6 +63,25 @@ def merge_browser_into_page_analysis( def browser_summary_from_page_analysis(pa: dict[str, Any]) -> dict[str, int]: browser = pa.get("browser") if isinstance(pa.get("browser"), dict) else {} + console = browser.get("console") + page_errors = browser.get("page_errors") + failed_requests = browser.get("failed_requests") + + if isinstance(console, list) or isinstance(page_errors, list) or isinstance(failed_requests, list): + console_list = console if isinstance(console, list) else [] + page_error_list = page_errors if isinstance(page_errors, list) else [] + failed_request_list = failed_requests if isinstance(failed_requests, list) else [] + return { + "console_error_count": sum( + 1 for c in console_list if isinstance(c, dict) and c.get("level") == "error" + ), + "console_warning_count": sum( + 1 for c in console_list if isinstance(c, dict) and c.get("level") == "warning" + ), + "page_error_count": len(page_error_list), + "failed_request_count": len(failed_request_list), + } + summary = browser.get("summary") if isinstance(browser.get("summary"), dict) else {} return { "console_error_count": int(summary.get("console_error_count") or 0), @@ -96,9 +115,12 @@ def aggregate_browser_diagnostics_df(df) -> dict[str, Any]: """Site-level browser diagnostic counts from crawl DataFrame page_analysis cells.""" pages_with_console_errors = 0 pages_with_page_errors = 0 + pages_with_failed_requests = 0 total_console_errors = 0 total_page_errors = 0 + total_failed_requests = 0 message_counts: dict[str, dict[str, Any]] = {} + exception_counts: dict[str, dict[str, Any]] = {} if df is None or getattr(df, "empty", True) or "page_analysis" not in df.columns: return {} @@ -111,12 +133,16 @@ def aggregate_browser_diagnostics_df(df) -> dict[str, Any]: url = str(row.get("url") or "").strip() ce = counts["console_error_count"] pe = counts["page_error_count"] + fr = counts["failed_request_count"] if ce > 0: pages_with_console_errors += 1 total_console_errors += ce if pe > 0: pages_with_page_errors += 1 total_page_errors += pe + if fr > 0: + pages_with_failed_requests += 1 + total_failed_requests += fr browser = pa.get("browser") if isinstance(pa.get("browser"), dict) else {} for msg in browser.get("console") or []: if not isinstance(msg, dict) or msg.get("level") != "error": @@ -128,12 +154,24 @@ def aggregate_browser_diagnostics_df(df) -> dict[str, Any]: bucket["count"] += 1 if url and url not in bucket["sample_urls"] and len(bucket["sample_urls"]) < 3: bucket["sample_urls"].append(url) + for err in browser.get("page_errors") or []: + if not isinstance(err, dict): + continue + text = str(err.get("message") or "").strip() + if not text: + continue + bucket = exception_counts.setdefault(text, {"text": text, "count": 0, "sample_urls": []}) + bucket["count"] += 1 + if url and url not in bucket["sample_urls"] and len(bucket["sample_urls"]) < 3: + bucket["sample_urls"].append(url) if ( pages_with_console_errors == 0 and pages_with_page_errors == 0 + and pages_with_failed_requests == 0 and total_console_errors == 0 and total_page_errors == 0 + and total_failed_requests == 0 ): return {} @@ -142,11 +180,19 @@ def aggregate_browser_diagnostics_df(df) -> dict[str, Any]: key=lambda x: int(x.get("count") or 0), reverse=True, )[:5] + top_page_errors = sorted( + exception_counts.values(), + key=lambda x: int(x.get("count") or 0), + reverse=True, + )[:5] return { "pages_with_console_errors": pages_with_console_errors, "pages_with_page_errors": pages_with_page_errors, + "pages_with_failed_requests": pages_with_failed_requests, "total_console_errors": total_console_errors, "total_page_errors": total_page_errors, + "total_failed_requests": total_failed_requests, "top_console_messages": top_console_messages, + "top_page_errors": top_page_errors, } diff --git a/web/src/lib/browserErrors.test.ts b/web/src/lib/browserErrors.test.ts index 420ecba..878ede3 100644 --- a/web/src/lib/browserErrors.test.ts +++ b/web/src/lib/browserErrors.test.ts @@ -2,10 +2,15 @@ import { describe, expect, it } from 'vitest'; import type { ReportLink, ReportPayload } from '@/types/report'; import { buildTopConsoleSummary, + buildTopConsoleSummaryFromRows, + buildTopExceptionSummaryFromRows, + computeBrowserErrorStats, flattenBrowserErrorsForTable, formatBrowserErrorSource, + formatPagesAffectedStat, getBrowserDiagnosticsScope, getLinksWithBrowserErrors, + linkBrowserCounts, linkHasBrowserErrors, linksInspectHref, } from '@/lib/browserErrors'; @@ -39,6 +44,32 @@ const linkWithException: ReportLink = { }, }; +const linkWithFailedRequest: ReportLink = { + url: 'https://example.com/d', + page_analysis: { + browser: { + console: [], + page_errors: [], + failed_requests: [{ url: 'https://cdn.example.com/app.js', failure: 'net::ERR_BLOCKED_BY_CLIENT' }], + summary: { console_error_count: 0, page_error_count: 0, failed_request_count: 1 }, + }, + }, +}; + +const ghostLink: ReportLink = { + url: 'https://example.com/ghost', + has_browser_errors: true, + console_error_count: 0, + page_error_count: 0, + page_analysis: { + browser: { + console: [], + page_errors: [], + summary: { console_error_count: 0, page_error_count: 0 }, + }, + }, +}; + const cleanLink: ReportLink = { url: 'https://example.com/c', has_browser_errors: false, @@ -51,13 +82,35 @@ const cleanLink: ReportLink = { }, }; +const mismatchLink: ReportLink = { + url: 'https://example.com/mismatch', + console_error_count: 9, + page_analysis: { + browser: { + console: [{ level: 'error', text: 'Actual error' }], + page_errors: [], + summary: { console_error_count: 9, page_error_count: 0 }, + }, + }, +}; + describe('browserErrors', () => { - it('detects links with browser errors', () => { + it('detects links with browser errors from stored arrays, not stale flags', () => { expect(linkHasBrowserErrors(linkWithConsoleError)).toBe(true); expect(linkHasBrowserErrors(linkWithException)).toBe(true); + expect(linkHasBrowserErrors(linkWithFailedRequest)).toBe(true); + expect(linkHasBrowserErrors(ghostLink)).toBe(false); expect(linkHasBrowserErrors(cleanLink)).toBe(false); }); + it('counts errors from browser arrays instead of summary fields', () => { + expect(linkBrowserCounts(mismatchLink)).toEqual({ + consoleErrors: 1, + pageErrors: 0, + failedRequests: 0, + }); + }); + it('reads diagnostics scope from report payload', () => { const data: ReportPayload = { report_meta: { @@ -83,17 +136,31 @@ describe('browserErrors', () => { expect(scope.usesBrowser).toBe(false); }); - it('filters and sorts links with errors', () => { - const links = getLinksWithBrowserErrors([cleanLink, linkWithConsoleError, linkWithException]); + it('filters and sorts links with errors by array counts', () => { + const links = getLinksWithBrowserErrors([ + cleanLink, + ghostLink, + linkWithConsoleError, + mismatchLink, + linkWithException, + linkWithFailedRequest, + ]); expect(links.map((l) => l.url)).toEqual([ 'https://example.com/a', 'https://example.com/b', + 'https://example.com/d', + 'https://example.com/mismatch', ]); }); - it('flattens console errors and exceptions into table rows', () => { - const rows = flattenBrowserErrorsForTable([linkWithConsoleError, linkWithException, cleanLink]); - expect(rows).toHaveLength(2); + it('flattens console errors, exceptions, and failed requests into table rows', () => { + const rows = flattenBrowserErrorsForTable([ + linkWithConsoleError, + linkWithException, + linkWithFailedRequest, + cleanLink, + ]); + expect(rows).toHaveLength(3); expect(rows[0]).toMatchObject({ url: 'https://example.com/a', type: 'console', @@ -107,6 +174,56 @@ describe('browserErrors', () => { message: 'Uncaught TypeError: x is not a function', stack: 'at foo (app.js:10)', }); + expect(rows[2]).toMatchObject({ + url: 'https://example.com/d', + type: 'failed_request', + message: 'net::ERR_BLOCKED_BY_CLIENT', + source_url: 'https://cdn.example.com/app.js', + }); + }); + + it('computes unified stats from link arrays', () => { + const stats = computeBrowserErrorStats([ + linkWithConsoleError, + linkWithException, + linkWithFailedRequest, + cleanLink, + ghostLink, + ]); + expect(stats.totalConsole).toBe(1); + expect(stats.totalExceptions).toBe(1); + expect(stats.totalFailedRequests).toBe(1); + expect(stats.pagesWithConsole).toBe(1); + expect(stats.pagesWithExceptions).toBe(1); + expect(stats.pagesWithFailedRequests).toBe(1); + expect(stats.affectedPages).toBe(3); + expect(stats.allRows).toHaveLength(3); + }); + + it('builds top summaries from flattened rows', () => { + const duplicateException: ReportLink = { + url: 'https://example.com/e2', + page_analysis: { + browser: { + page_errors: [{ message: 'Uncaught TypeError: x is not a function' }], + }, + }, + }; + const rows = flattenBrowserErrorsForTable([linkWithConsoleError, linkWithException, duplicateException]); + expect(buildTopConsoleSummaryFromRows(rows)).toEqual([ + { + text: 'Failed to load', + count: 1, + sample_urls: ['https://example.com/a'], + }, + ]); + expect(buildTopExceptionSummaryFromRows(rows)).toEqual([ + { + text: 'Uncaught TypeError: x is not a function', + count: 2, + sample_urls: ['https://example.com/b', 'https://example.com/e2'], + }, + ]); }); it('builds top console summary from scope aggregate', () => { @@ -120,6 +237,11 @@ describe('browserErrors', () => { ]); }); + it('formats pages affected with percentage', () => { + expect(formatPagesAffectedStat(3, 80)).toBe('3 / 80 (4%)'); + expect(formatPagesAffectedStat(0, 0)).toBe('0'); + }); + it('formats source location', () => { expect(formatBrowserErrorSource('https://x.com/a.js', 10)).toBe('https://x.com/a.js:10'); expect(formatBrowserErrorSource(undefined)).toBe('—'); diff --git a/web/src/lib/browserErrors.ts b/web/src/lib/browserErrors.ts index 52dd5b3..069d5e4 100644 --- a/web/src/lib/browserErrors.ts +++ b/web/src/lib/browserErrors.ts @@ -6,7 +6,7 @@ import type { TopConsoleMessage, } from '@/types/report'; -export type BrowserErrorRowType = 'console' | 'exception'; +export type BrowserErrorRowType = 'console' | 'exception' | 'failed_request'; export interface BrowserDiagnosticsScope { renderMode: string; @@ -30,6 +30,24 @@ export interface TopConsoleSummaryRow { sample_urls: string[]; } +export interface LinkBrowserCounts { + consoleErrors: number; + pageErrors: number; + failedRequests: number; +} + +export interface BrowserErrorStats { + allRows: FlatBrowserErrorRow[]; + totalConsole: number; + totalExceptions: number; + totalFailedRequests: number; + pagesWithConsole: number; + pagesWithExceptions: number; + pagesWithFailedRequests: number; + affectedPages: number; + totalPages: number; +} + function pageAnalysisBrowser(link: ReportLink): PageAnalysis['browser'] | undefined { const pa = link.page_analysis; if (!pa || typeof pa !== 'object') return undefined; @@ -37,20 +55,24 @@ function pageAnalysisBrowser(link: ReportLink): PageAnalysis['browser'] | undefi return browser && typeof browser === 'object' ? browser : undefined; } -export function linkHasBrowserErrors(link: ReportLink): boolean { - if (link.has_browser_errors) return true; - if ((link.console_error_count ?? 0) > 0) return true; - if ((link.page_error_count ?? 0) > 0) return true; +export function linkBrowserCounts(link: ReportLink): LinkBrowserCounts { const browser = pageAnalysisBrowser(link); - if (!browser) return false; - const summary = browser.summary; - if (summary) { - if ((summary.console_error_count ?? 0) > 0) return true; - if ((summary.page_error_count ?? 0) > 0) return true; + if (!browser) { + return { consoleErrors: 0, pageErrors: 0, failedRequests: 0 }; } - const consoleMsgs = browser.console ?? []; - if (consoleMsgs.some((m) => String(m.level ?? '').toLowerCase() === 'error')) return true; - return (browser.page_errors?.length ?? 0) > 0; + + const consoleErrors = (browser.console ?? []).filter( + (m) => String(m.level ?? '').toLowerCase() === 'error', + ).length; + const pageErrors = (browser.page_errors ?? []).length; + const failedRequests = (browser.failed_requests ?? []).length; + + return { consoleErrors, pageErrors, failedRequests }; +} + +export function linkHasBrowserErrors(link: ReportLink): boolean { + const counts = linkBrowserCounts(link); + return counts.consoleErrors + counts.pageErrors + counts.failedRequests > 0; } export function getBrowserDiagnosticsScope( @@ -68,8 +90,10 @@ export function getBrowserDiagnosticsScope( export function getLinksWithBrowserErrors(links: ReportLink[] | undefined): ReportLink[] { if (!links?.length) return []; return links.filter(linkHasBrowserErrors).sort((a, b) => { - const aTotal = (a.console_error_count ?? 0) + (a.page_error_count ?? 0); - const bTotal = (b.console_error_count ?? 0) + (b.page_error_count ?? 0); + const aCounts = linkBrowserCounts(a); + const bCounts = linkBrowserCounts(b); + const aTotal = aCounts.consoleErrors + aCounts.pageErrors + aCounts.failedRequests; + const bTotal = bCounts.consoleErrors + bCounts.pageErrors + bCounts.failedRequests; return bTotal - aTotal || String(a.url).localeCompare(String(b.url)); }); } @@ -105,11 +129,95 @@ export function flattenBrowserErrorsForTable(links: ReportLink[] | undefined): F stack: err.stack, }); } + + for (const [i, req] of (browser.failed_requests ?? []).entries()) { + const failure = String(req.failure ?? '').trim(); + const reqUrl = String(req.url ?? '').trim(); + rows.push({ + id: `${url}::failed::${i}`, + url, + type: 'failed_request', + message: failure || reqUrl || '—', + source_url: reqUrl || undefined, + }); + } } return rows; } +function buildTopSummaryFromRows( + rows: FlatBrowserErrorRow[], + type: BrowserErrorRowType, + limit = 5, +): TopConsoleSummaryRow[] { + const buckets = new Map(); + + for (const row of rows) { + if (row.type !== type) continue; + const text = row.message.trim(); + if (!text || text === '—') continue; + + const bucket = buckets.get(text) ?? { count: 0, sample_urls: [] }; + bucket.count += 1; + if (row.url && !bucket.sample_urls.includes(row.url) && bucket.sample_urls.length < 3) { + bucket.sample_urls.push(row.url); + } + buckets.set(text, bucket); + } + + return [...buckets.entries()] + .map(([text, bucket]) => ({ + text, + count: bucket.count, + sample_urls: bucket.sample_urls, + })) + .sort((a, b) => b.count - a.count || a.text.localeCompare(b.text)) + .slice(0, limit); +} + +export function buildTopConsoleSummaryFromRows(rows: FlatBrowserErrorRow[]): TopConsoleSummaryRow[] { + return buildTopSummaryFromRows(rows, 'console'); +} + +export function buildTopExceptionSummaryFromRows(rows: FlatBrowserErrorRow[]): TopConsoleSummaryRow[] { + return buildTopSummaryFromRows(rows, 'exception'); +} + +export function computeBrowserErrorStats(links: ReportLink[] | undefined): BrowserErrorStats { + const allRows = flattenBrowserErrorsForTable(links); + const pagesWithConsole = new Set(); + const pagesWithExceptions = new Set(); + const pagesWithFailedRequests = new Set(); + + for (const link of links ?? []) { + const url = String(link.url ?? '').trim(); + if (!url) continue; + const counts = linkBrowserCounts(link); + if (counts.consoleErrors > 0) pagesWithConsole.add(url); + if (counts.pageErrors > 0) pagesWithExceptions.add(url); + if (counts.failedRequests > 0) pagesWithFailedRequests.add(url); + } + + return { + allRows, + totalConsole: allRows.filter((row) => row.type === 'console').length, + totalExceptions: allRows.filter((row) => row.type === 'exception').length, + totalFailedRequests: allRows.filter((row) => row.type === 'failed_request').length, + pagesWithConsole: pagesWithConsole.size, + pagesWithExceptions: pagesWithExceptions.size, + pagesWithFailedRequests: pagesWithFailedRequests.size, + affectedPages: new Set(allRows.map((row) => row.url)).size, + totalPages: links?.length ?? 0, + }; +} + +export function formatPagesAffectedStat(pages: number, totalPages: number): string { + if (totalPages <= 0) return pages.toLocaleString(); + const pct = Math.round((pages / totalPages) * 100); + return `${pages.toLocaleString()} / ${totalPages.toLocaleString()} (${pct}%)`; +} + export function buildTopConsoleSummary( scope: BrowserDiagnosticsAggregate | null | undefined, ): TopConsoleSummaryRow[] { diff --git a/web/src/lib/buildBrowserErrorsPrompt.ts b/web/src/lib/buildBrowserErrorsPrompt.ts index 1df6a59..c43ebf2 100644 --- a/web/src/lib/buildBrowserErrorsPrompt.ts +++ b/web/src/lib/buildBrowserErrorsPrompt.ts @@ -16,10 +16,12 @@ import { const TYPE_LABELS: Record = { console: 'Console error', exception: 'Uncaught exception', + failed_request: 'Failed network request', }; const TYPE_PRIORITY: Record = { exception: 'High', + failed_request: 'High', console: 'Medium', }; diff --git a/web/src/strings.json b/web/src/strings.json index 1d8b6c6..9730db1 100644 --- a/web/src/strings.json +++ b/web/src/strings.json @@ -556,6 +556,12 @@ "exceptionTotal": { "body": "Total uncaught JavaScript exceptions captured during this crawl." }, + "failedPages": { + "body": "Pages where at least one network request failed during browser render." + }, + "failedTotal": { + "body": "Total failed network requests captured during JavaScript rendering." + }, "renderMode": { "body": "Crawl render mode for this audit: static HTML only, JavaScript rendering, or auto." } @@ -2808,26 +2814,33 @@ "emptyStaticBody": "Browser console and runtime errors are only captured when the crawl uses JavaScript or Auto rendering. Re-run the audit with JavaScript rendering enabled and console capture turned on.", "runAudit": "Run audit", "emptyCleanTitle": "No JavaScript errors detected", - "emptyCleanBody": "No console errors or uncaught exceptions were logged during JavaScript rendering for this crawl.", + "emptyCleanBody": "No console errors, uncaught exceptions, or failed network requests were logged during JavaScript rendering for this crawl.", "emptyFiltered": "No errors match the current filters or search.", "consolePagesCard": "Pages with console errors", "consoleTotalCard": "Total console errors", "exceptionPagesCard": "Pages with uncaught exceptions", "exceptionTotalCard": "Total uncaught exceptions", + "failedPagesCard": "Pages with failed requests", + "failedTotalCard": "Total failed requests", "renderMode": "Render mode", "topRecurring": "Top recurring console errors", "topRecurringHint": "Grouped by message text across the crawl", + "topRecurringExceptions": "Top recurring uncaught exceptions", + "topRecurringExceptionsHint": "Grouped by exception message across the crawl", "thMessage": "Message", "thCount": "Count", + "thOccurrences": "Occurrences", "thSampleUrls": "Sample URLs", "allErrors": "All errors", - "allErrorsHint": "Console errors and uncaught exceptions with source location where available", + "allErrorsHint": "Console errors, uncaught exceptions, and failed network requests with source location where available", + "consoleLevelsNote": "Only console errors are listed here. Warnings, info, and debug messages appear in the page inspector.", "thUrl": "URL", "thType": "Type", "thSource": "Source", "thActions": "Actions", "typeConsole": "Console", "typeException": "Exception", + "typeFailedRequest": "Failed request", "typeAll": "All types", "viewDetails": "View details", "expandStack": "Show stack trace", diff --git a/web/src/types/report.ts b/web/src/types/report.ts index 21e89d7..dea9237 100644 --- a/web/src/types/report.ts +++ b/web/src/types/report.ts @@ -1022,9 +1022,12 @@ export interface TopConsoleMessage { export interface BrowserDiagnosticsAggregate { pages_with_console_errors?: number; pages_with_page_errors?: number; + pages_with_failed_requests?: number; total_console_errors?: number; total_page_errors?: number; + total_failed_requests?: number; top_console_messages?: TopConsoleMessage[]; + top_page_errors?: TopConsoleMessage[]; } export interface PageAnalysis { diff --git a/web/src/views/JavaScriptErrors.tsx b/web/src/views/JavaScriptErrors.tsx index a80714d..e72cb9c 100644 --- a/web/src/views/JavaScriptErrors.tsx +++ b/web/src/views/JavaScriptErrors.tsx @@ -15,11 +15,12 @@ import { paginateSlice, PAGE_SIZE } from '@/components/google/tableUtils'; import type { ViewTabItem } from '../components'; import type { ViewProps } from '@/types'; import { - buildTopConsoleSummary, - flattenBrowserErrorsForTable, + buildTopConsoleSummaryFromRows, + buildTopExceptionSummaryFromRows, + computeBrowserErrorStats, formatBrowserErrorSource, + formatPagesAffectedStat, getBrowserDiagnosticsScope, - getLinksWithBrowserErrors, linksInspectHref, type FlatBrowserErrorRow, } from '@/lib/browserErrors'; @@ -27,10 +28,77 @@ import AiSuggestionButton from '@/components/ai/AiSuggestionButton'; import BrowserErrorsPromptGenerator from '@/components/issues/BrowserErrorsPromptGenerator'; import { buildBrowserErrorContext, buildBrowserErrorSummaryContext } from '@/lib/fixSuggestionContext'; -type TypeFilter = 'All' | 'console' | 'exception'; +type TypeFilter = 'All' | FlatBrowserErrorRow['type']; const JS_ERRORS_TABS = ['summary', 'errors'] as const; type JsErrorsTabId = (typeof JS_ERRORS_TABS)[number]; +function typeLabel(type: FlatBrowserErrorRow['type'], vj: typeof strings.views.javascriptErrors): string { + if (type === 'console') return vj.typeConsole; + if (type === 'exception') return vj.typeException; + return vj.typeFailedRequest; +} + +function TopSummaryTable({ + title, + hint, + rows, + trailingQuery, + vj, +}: { + title: string; + hint: string; + rows: ReturnType; + trailingQuery: string; + vj: typeof strings.views.javascriptErrors; +}) { + return ( + +

{title}

+

{hint}

+
+ + + + {vj.thMessage} + {vj.thOccurrences} + {vj.thSampleUrls} + + + + {rows.map((row) => ( + + +
+ {row.text} + +
+ + {row.count.toLocaleString()} + +
    + {row.sample_urls.map((url) => ( +
  • + + {url} + +
    • + ))} +
+ + + ))} + +
+
+ + ); +} + export default function JavaScriptErrors({ searchQuery = '' }: ViewProps) { const { data } = useReport(); const domain = data?.site_name || ''; @@ -48,12 +116,10 @@ export default function JavaScriptErrors({ searchQuery = '' }: ViewProps) { const q = (searchQuery || '').toLowerCase().trim(); const scopeInfo = useMemo(() => getBrowserDiagnosticsScope(data), [data]); - const errorLinks = useMemo(() => getLinksWithBrowserErrors(data?.links), [data?.links]); - const allRows = useMemo(() => flattenBrowserErrorsForTable(data?.links), [data?.links]); - const topMessages = useMemo( - () => buildTopConsoleSummary(scopeInfo.browserDiagnostics), - [scopeInfo.browserDiagnostics], - ); + const stats = useMemo(() => computeBrowserErrorStats(data?.links), [data?.links]); + const { allRows } = stats; + const topMessages = useMemo(() => buildTopConsoleSummaryFromRows(allRows), [allRows]); + const topExceptions = useMemo(() => buildTopExceptionSummaryFromRows(allRows), [allRows]); const filteredRows = useMemo(() => { let rows: FlatBrowserErrorRow[] = allRows; @@ -92,36 +158,28 @@ export default function JavaScriptErrors({ searchQuery = '' }: ViewProps) { setExpandedRow(null); }, [typeFilter, q]); + const summaryBadgeCount = topMessages.length + topExceptions.length; + const tabItems = useMemo((): ViewTabItem[] => [ { id: 'summary', label: vj.tabs.summary, icon: , - badge: topMessages.length > 0 ? topMessages.length : null, + badge: summaryBadgeCount > 0 ? summaryBadgeCount : null, }, { id: 'errors', label: vj.tabs.errors, icon: , - badge: filteredRows.length > 0 ? filteredRows.length : null, + badge: allRows.length > 0 ? allRows.length : null, }, - ], [vj.tabs, topMessages.length, filteredRows.length]); + ], [vj.tabs, summaryBadgeCount, allRows.length]); if (!linksReady) { return ; } - const agg = scopeInfo.browserDiagnostics; - const pagesWithConsole = Number(agg?.pages_with_console_errors ?? 0); - const totalConsole = Number(agg?.total_console_errors ?? 0); - const pagesWithExceptions = Number(agg?.pages_with_page_errors ?? 0); - const totalExceptions = Number(agg?.total_page_errors ?? 0); - const hasAnyErrors = - allRows.length > 0 - || totalConsole > 0 - || totalExceptions > 0 - || pagesWithConsole > 0 - || pagesWithExceptions > 0; + const hasAnyErrors = allRows.length > 0; if (!scopeInfo.usesBrowser) { return ( @@ -158,7 +216,7 @@ export default function JavaScriptErrors({ searchQuery = '' }: ViewProps) { title={vj.title} subtitle={`${vj.subtitle} ${format(vj.subtitleCount, { count: allRows.length, - pages: errorLinks.length, + pages: stats.affectedPages, })}`} actions={ -
- - - - +
+ + + + + + {scopeInfo.renderMode}} hint={metricHelpHint('views.jsErrors.renderMode')} />
{topMessages.length > 0 ? ( - -

{vj.topRecurring}

-

{vj.topRecurringHint}

-
- - - - {vj.thMessage} - {vj.thCount} - {vj.thSampleUrls} - - - - {topMessages.map((row) => ( - - -
- {row.text} - -
- - {row.count.toLocaleString()} - -
    - {row.sample_urls.map((url) => ( -
  • - - {url} - -
    • - ))} -
- - - ))} - -
-
- - ) : ( + + ) : null} + + {topExceptions.length > 0 ? ( + + ) : null} + + {topMessages.length === 0 && topExceptions.length === 0 ? ( {vj.emptyFiltered} - )} + ) : null} )} @@ -245,6 +292,7 @@ export default function JavaScriptErrors({ searchQuery = '' }: ViewProps) {

{vj.allErrors}

{vj.allErrorsHint}

+

{vj.consoleLevelsNote}

@@ -286,6 +335,7 @@ export default function JavaScriptErrors({ searchQuery = '' }: ViewProps) { type="button" className="p-1 text-muted-foreground hover:text-foreground" aria-expanded={expanded} + aria-label={expanded ? vj.collapseStack : vj.expandStack} onClick={() => setExpandedRow(expanded ? null : row.id)} > {expanded ? ( @@ -307,8 +357,8 @@ export default function JavaScriptErrors({ searchQuery = '' }: ViewProps) { - - {row.type === 'console' ? vj.typeConsole : vj.typeException} + + {typeLabel(row.type, vj)}
From f79268559f49febc0e187933858e95fe843246b6 Mon Sep 17 00:00:00 2001 From: PrashantUnity Date: Sat, 27 Jun 2026 20:04:02 +0530 Subject: [PATCH 6/8] let add easy way to visulaize --- .../overview/OverviewContentQuality.tsx | 208 ++++++++++++------ 1 file changed, 143 insertions(+), 65 deletions(-) diff --git a/web/src/components/overview/OverviewContentQuality.tsx b/web/src/components/overview/OverviewContentQuality.tsx index 2593eda..e589311 100644 --- a/web/src/components/overview/OverviewContentQuality.tsx +++ b/web/src/components/overview/OverviewContentQuality.tsx @@ -15,10 +15,11 @@ import { strings, format } from '@/lib/strings'; import { metricHelpHint } from '@/lib/metricHelp'; import HelpHint from '@/components/HelpHint'; import { Card, StatCard } from '@/components'; -import { CompactBarChart } from '@/components/charts/compact'; +import { CompactBarChart, CompactDonut } from '@/components/charts/compact'; import { buildKeywordsTabHref } from './overviewKeywordOpportunities'; import { buildLanguageBarChartData, + buildLanguageMixSegments, buildViewHref, duplicateGroupsBand, duplicateMemberCount, @@ -40,18 +41,43 @@ export interface OverviewContentQualityProps { keywordsHref: string; } -function LanguageMixCharts({ counts }: { counts: Record }) { +function LanguageMixVisualization({ + counts, + singleLanguage, +}: { + counts: Record; + singleLanguage: boolean; +}) { const barChart = useMemo(() => buildLanguageBarChartData(counts), [counts]); + const donutSegments = useMemo(() => buildLanguageMixSegments(counts), [counts]); + const primaryShare = useMemo(() => languageShares(counts, 1)[0], [counts]); + + if (barChart) { + return ( + row.height)} + labels={barChart.map((row) => row.label)} + colors={barChart.map((row) => row.color)} + /> + ); + } - if (!barChart) return null; + if (donutSegments.length === 0) return null; return ( - row.height)} - labels={barChart.map((row) => row.label)} - colors={barChart.map((row) => row.color)} - /> +
+ + {singleLanguage ? ( +

{vo.contentQualitySingleLanguageSite}

+ ) : null} +
); } @@ -69,15 +95,15 @@ function ContentQualityColumn({ children: ReactNode; }) { return ( -
-
+
+

{title}

{viewAllLabel}
-
{statCard}
-
{children}
+ {statCard} + {children}
); } @@ -157,8 +183,12 @@ export function OverviewContentQuality({ data, querySuffix, keywordsHref }: Over if (!shouldShowContentQuality(data)) return null; + const showDuplicates = duplicateGroupCount > 0; + const showLanguages = languagesDetected > 0; + const languageOnly = showLanguages && !showDuplicates; + return ( - +
@@ -216,8 +246,10 @@ export function OverviewContentQuality({ data, querySuffix, keywordsHref }: Over ) : null}
-
- {duplicateGroupCount > 0 ? ( +
+ {showDuplicates ? ( } label={vo.contentQualityGroupsCount} @@ -243,7 +274,7 @@ export function OverviewContentQuality({ data, querySuffix, keywordsHref }: Over /> } > -
+

{vo.contentQualityLargestClusters}

@@ -275,53 +306,102 @@ export function OverviewContentQuality({ data, querySuffix, keywordsHref }: Over ) : null} - {languagesDetected > 0 ? ( - } - label={vo.contentQualityLocaleCount} - value={languagesDetected.toLocaleString()} - sub={ - dominantLanguage - ? format(vo.contentQualityDominantLanguage, { - lang: dominantLanguage.lang, - pct: dominantLanguage.pct, - }) - : undefined - } - band={mixedLanguage ? vo.mixedLanguage : vo.metricBandGood} - bandClassName={mixedLanguage ? bandClassName('fair') : bandClassName('good')} - className={ - mixedLanguage ? 'border-amber-500/20 ring-1 ring-inset ring-amber-500/10' : 'border-cyan-500/15' - } - hint={metricHelpHint('views.overview.contentQualityLocales')} - /> - } - > -
-
-

- {vo.contentQualityLanguageMix} -

- - {vo.contentQualityOpenContentAnalytics} + {showLanguages ? ( + languageOnly ? ( +
+
+

{vo.languagesSampled}

+ + {vo.contentQualityOpenTextAnalysis}
- {mixedLanguage ? ( -

- {vo.contentQualityMixedLanguageHint} -

- ) : null} - +
+ } + label={vo.contentQualityLocaleCount} + value={languagesDetected.toLocaleString()} + sub={ + dominantLanguage + ? format(vo.contentQualityDominantLanguage, { + lang: dominantLanguage.lang, + pct: dominantLanguage.pct, + }) + : undefined + } + band={mixedLanguage ? vo.mixedLanguage : vo.metricBandGood} + bandClassName={mixedLanguage ? bandClassName('fair') : bandClassName('good')} + className={ + mixedLanguage ? 'border-amber-500/20 ring-1 ring-inset ring-amber-500/10' : 'border-cyan-500/15' + } + hint={metricHelpHint('views.overview.contentQualityLocales')} + /> +
+
+

+ {vo.contentQualityLanguageMix} +

+ + {vo.contentQualityOpenContentAnalytics} + +
+ {mixedLanguage ? ( +

+ {vo.contentQualityMixedLanguageHint} +

+ ) : null} + +
+
- + ) : ( + } + label={vo.contentQualityLocaleCount} + value={languagesDetected.toLocaleString()} + sub={ + dominantLanguage + ? format(vo.contentQualityDominantLanguage, { + lang: dominantLanguage.lang, + pct: dominantLanguage.pct, + }) + : undefined + } + band={mixedLanguage ? vo.mixedLanguage : vo.metricBandGood} + bandClassName={mixedLanguage ? bandClassName('fair') : bandClassName('good')} + className={ + mixedLanguage ? 'border-amber-500/20 ring-1 ring-inset ring-amber-500/10' : 'border-cyan-500/15' + } + hint={metricHelpHint('views.overview.contentQualityLocales')} + /> + } + > +
+
+

+ {vo.contentQualityLanguageMix} +

+ + {vo.contentQualityOpenContentAnalytics} + +
+ {mixedLanguage ? ( +

+ {vo.contentQualityMixedLanguageHint} +

+ ) : null} + +
+ + ) ) : null}
@@ -330,11 +410,10 @@ export function OverviewContentQuality({ data, querySuffix, keywordsHref }: Over

{vo.contentQualityAdvancedInsights}

-
+
{semanticTopics > 0 ? ( } label={vo.parentTopics} @@ -345,7 +424,6 @@ export function OverviewContentQuality({ data, querySuffix, keywordsHref }: Over {hasNer ? ( } label={vo.namedEntities} From f0f978cc91f1022fd8051b74b36460b8611a785b Mon Sep 17 00:00:00 2001 From: PrashantUnity Date: Sat, 27 Jun 2026 21:28:10 +0530 Subject: [PATCH 7/8] Hello Smart Ways --- AGENT.md | 49 ++- AGENTS.md | 3 + web/src/components/Card.tsx | 9 +- web/src/components/ChartCard.tsx | 7 +- web/src/components/DevCopyJsonButton.tsx | 48 +++ .../content/RichResultsValidationPanel.tsx | 5 +- .../components/google/UrlCoverageDoughnut.tsx | 5 +- .../components/google/UrlGapListsPanel.tsx | 6 +- web/src/components/issues/IssueTaskBoard.tsx | 24 +- web/src/components/issues/IssueTrendChart.tsx | 13 +- .../components/issues/MobileDesktopDelta.tsx | 23 +- .../CompetitorKeywordGapPanel.tsx | 7 +- .../ContentTemplatesPanel.tsx | 11 +- .../keywordsExplorer/KeywordCharts.tsx | 14 +- .../KeywordExplorerChrome.tsx | 6 +- .../keywordsExplorer/KeywordOverviewPanel.tsx | 26 +- .../keywordsExplorer/KeywordPanels.tsx | 45 ++- .../keywordsExplorer/TopicMapPanel.tsx | 7 +- web/src/components/links/InspectorTabs.tsx | 17 +- .../components/links/LinkAttributesPanel.tsx | 42 ++- .../explorer/LinksExplorerSummaryCharts.tsx | 31 +- .../links/explorer/LinksExplorerTableTab.tsx | 20 +- .../components/overview/OverviewChartsTab.tsx | 168 +++++++++- .../overview/OverviewContentQuality.tsx | 141 +++++++- .../overview/OverviewCrawlMetrics.tsx | 135 +++++++- .../overview/OverviewExecutiveSummary.tsx | 61 +++- .../components/overview/OverviewHealthTab.tsx | 74 ++++- .../OverviewKeywordOpportunitiesCard.tsx | 155 ++++++++- .../components/overview/OverviewPagesTab.tsx | 49 ++- .../overview/OverviewSummaryTab.tsx | 34 +- .../overview/PortfolioBenchmarkCard.tsx | 67 +++- .../searchPerformance/GscCharts.tsx | 32 +- .../siteStructure/SiteStructureLinkGraph.tsx | 19 +- web/src/components/traffic/Ga4Charts.tsx | 34 +- web/src/lib/dashboard/widgets/WidgetFrame.tsx | 16 + web/src/strings.json | 3 + web/src/types/components.ts | 1 + web/src/views/Accessibility.tsx | 70 +++- web/src/views/Backlinks.tsx | 173 +++++++++- web/src/views/Contacts.tsx | 73 ++++- web/src/views/Content.tsx | 72 ++++- web/src/views/ContentAnalytics.tsx | 303 ++++++++++++++++-- web/src/views/GeoReadiness.tsx | 243 ++++++++++++-- web/src/views/ImageSeo.tsx | 57 +++- web/src/views/Indexation.tsx | 47 ++- web/src/views/Issues.tsx | 128 +++++++- web/src/views/JavaScriptErrors.tsx | 88 ++++- web/src/views/KeywordsExplorer.tsx | 207 +++++++++++- web/src/views/Lighthouse.tsx | 183 ++++++++++- web/src/views/Links.tsx | 80 ++++- web/src/views/Redirects.tsx | 32 +- web/src/views/SearchPerformance.tsx | 195 ++++++++++- web/src/views/Security.tsx | 78 ++++- web/src/views/SiteStructure.tsx | 122 ++++++- web/src/views/Subdomains.tsx | 58 +++- web/src/views/TechStack.tsx | 51 ++- web/src/views/TextContentAnalysis.tsx | 166 +++++++++- web/src/views/Traffic.tsx | 189 ++++++++++- 58 files changed, 3727 insertions(+), 295 deletions(-) create mode 100644 web/src/components/DevCopyJsonButton.tsx diff --git a/AGENT.md b/AGENT.md index be7359b..62fc061 100644 --- a/AGENT.md +++ b/AGENT.md @@ -61,6 +61,7 @@ Developer reference for agents and contributors. User-facing overview: [README.m | Browser API client | `web/src/lib/publicBase.ts` (`apiUrl`, `apiFetch`, `VITE_BFF_BASE_URL`) | | D3 charts (custom / compare / overview) | `web/src/components/charts/d3/`, `web/src/lib/viz/` | | Chart.js charts (standard bar/line/doughnut) | `web/src/utils/chartJsDefaults.ts`, `react-chartjs-2` in views under `web/src/views/`, `web/src/components/searchPerformance/`, `web/src/components/traffic/` | +| Dev widget JSON copy (report cards + dashboards) | `web/src/components/Card.tsx` (`devData`), `web/src/components/DevCopyJsonButton.tsx`, `web/src/lib/dashboard/widgets/WidgetFrame.tsx` — see **Dev widget JSON copy** below | Schema changes: add Alembic migration (`alembic revision`). @@ -95,6 +96,52 @@ The web UI uses **both** Chart.js and D3.js. Pick the library that fits each cha - Keep chart-library types out of data-prep: use neutral shapes (`BarChartData`, `DualSeriesChartData` in `web/src/lib/viz/types.ts` and `web/src/lib/compareChartData.ts`); convert at the render layer via `web/src/lib/viz/adapters.ts` when needed. - Migrate page-by-page when D3 is the better fit; do not remove `chart.js` from `package.json` until all consumers are migrated. +**Dev widget JSON copy (local dev only)** + +In **`npm run dev`** (`import.meta.env.DEV`), report **widgets** (cards, panels, stat blocks) and custom **dashboard widgets** expose a top-right **`{ }`** button on hover. Clicking copies pretty-printed JSON of what that widget displays. Production builds omit the button entirely (dead-code eliminated at build time). + +| Piece | Path | +|-------|------| +| Overlay button | `web/src/components/DevCopyJsonButton.tsx` | +| Report card hook | `web/src/components/Card.tsx` — optional `devData?: unknown` | +| Dashboard grid widgets | `web/src/lib/dashboard/widgets/WidgetFrame.tsx` — passes `devData` when `status === 'loaded'` | +| Tooltip copy | `strings.json` → `components.devCopyJson.title` | + +**When adding or touching a widget, wire `devData`.** Goal: every user-visible widget in report views eventually has copy support. + +**Report views — use `Card` `devData`** + +Pass a **view-model object** (what the widget renders), not necessarily a single raw report key. Derived UI (computed counts, API-fetched trends, filtered rows) belongs in the payload. + +```tsx +const widgetDevData = useMemo( + () => ({ + widget: 'views.overview.executiveSummary.healthHero', // stable id for debugging + currentHealth, + healthDelta, + topIssues: topIssues.slice(0, 5), + raw: { executive_summary: data.executive_summary }, // optional source slices + }), + [currentHealth, healthDelta, topIssues, data.executive_summary], +); + +… +``` + +**Reference implementation:** `web/src/components/overview/OverviewExecutiveSummary.tsx` (health hero, AI summary, text summary cards). + +**`StatCard` and non-`Card` wrappers:** wrap with `` only when it fits layout; otherwise wrap the section in a `relative group/dev-card` container and render `` directly (same hover behaviour). + +**Dashboard widgets:** `WidgetFrame` already copies `{ widget, status, result }` when loaded. No extra work unless you add a new widget shell outside `WidgetFrame`. + +**Conventions** + +- Always include a stable `widget: 'view.section.name'` string id. +- Copy **displayed** values (including async-fetched state once available); add `raw: { … }` when source report slices help backend/debug work. +- Use `useMemo` for `devData` when the payload depends on props/state/effects. +- Do **not** gate on a custom env var unless staging preview also needs copy — default is `import.meta.env.DEV` only. +- Async widgets: omit `devData` until data is ready, or include partial payload plus flags like `historyLoaded: false`. + **Company standards:** UI copy in `web/src/strings.json` (Site Audit, Properties, Run audit). Data provenance on `report_meta` in report payload. Docs: `docs/COMPANY_STANDARDS.md`, `docs/GLOSSARY.md`. Migration `003_company_standards` (properties, pipeline_jobs, audit_log). **Export:** PDF/workbook via FileService (`FILE_SERVICE_URL` on MCP; `REPORT_API_URL` on FileService); CSV/JSON via `GET /api/report/export` and `src/website_profiling/tools/export_audit.py`. **Common footguns (check before finishing web or DB work)** @@ -169,4 +216,4 @@ These recur when adding features. Verify explicitly — do not assume tests caug - **Do:** Every API service enables `ValidateOnBuild` + `ValidateScopes` in `Program.cs` and ships `ServiceRegistrationValidationTests` (`WebApplicationFactory`). Shared env helper: `services/Shared/WebsiteProfiling.Testing/`. - **Don't:** Add scoped dependencies to singletons without resolving via `IServiceScopeFactory`, or skip the registration test when adding new host services. -**Checklist:** new report page uses `ReportShell` · no duplicate local imports in long functions · new `fetchone()` uses `_row_field` · `./local-test` passes all three coverage gates · new tools coverage test file listed in CI + both local-test scripts · `runpy` main-guard tests pop `sys.modules` first · new .NET API service has DI validation test +**Checklist:** new report page uses `ReportShell` · report/card widgets pass `Card` `devData` (see [AGENT.md](AGENT.md) § Dev widget JSON copy) · no duplicate local imports in long functions · new `fetchone()` uses `_row_field` · `./local-test` passes all three coverage gates · new tools coverage test file listed in CI + both local-test scripts · `runpy` main-guard tests pop `sys.modules` first · new .NET API service has DI validation test diff --git a/AGENTS.md b/AGENTS.md index 77eec83..fb1f98c 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -48,8 +48,11 @@ python -m src # Run audit pipeline | GEO / AEO / Agent readiness | `src/website_profiling/tools/audit_tools/geo/geo_tools.py`, `geo/agent_readiness.py` | | DB schema | `alembic/versions/` | | UI | `web/src/views/`, `web/src/pages/`, `web/src/AppRoutes.tsx` | +| Report/card widgets (dev JSON copy) | `Card` `devData` prop — see [AGENT.md](AGENT.md) § Dev widget JSON copy; reference: `web/src/components/overview/OverviewExecutiveSummary.tsx` | | Charts | D3: `web/src/components/charts/d3/`, `web/src/lib/viz/` · Chart.js: GSC/GA4/Links etc. — see [AGENT.md](AGENT.md) § Charts | **Charts:** Use **both** Chart.js and D3 — choose per chart (Overview/Compare → D3; standard GSC/GA4 bars → Chart.js). Full rules in [AGENT.md](AGENT.md). +**Dev widget JSON copy:** In local dev, each report card/panel should pass `devData` on `Card` so agents/devs can copy the widget’s JSON from the top-right `{ }` button. Wire on every widget you add or touch; full conventions in [AGENT.md](AGENT.md) § Dev widget JSON copy. + **Common pitfalls:** See [AGENT.md](AGENT.md) for the full footguns checklist (React context, Python local imports, psycopg dict rows, coverage gates). diff --git a/web/src/components/Card.tsx b/web/src/components/Card.tsx index b8ecdf1..2f53a2e 100644 --- a/web/src/components/Card.tsx +++ b/web/src/components/Card.tsx @@ -1,4 +1,5 @@ import type { MouseEventHandler, ReactNode } from 'react'; +import DevCopyJsonButton from './DevCopyJsonButton'; type CardProps = { children?: ReactNode; @@ -9,6 +10,8 @@ type CardProps = { /** Adds hover elevation + pointer affordance (for clickable cards). */ interactive?: boolean; onClick?: MouseEventHandler; + /** Dev only: JSON copied when the top-right overlay button is clicked. */ + devData?: unknown; }; /** @@ -24,16 +27,20 @@ export default function Card({ overflowHidden = false, interactive = false, onClick, + devData, }: CardProps) { + const showDevCopy = import.meta.env.DEV && devData != null; const paddingClass = padding === 'none' ? '' : padding === 'tight' ? 'p-4' : 'p-5'; const shadowClass = shadow ? 'shadow-sm' : ''; const overflowClass = overflowHidden ? 'overflow-hidden' : ''; const interactiveClass = interactive ? 'hover-lift cursor-pointer' : ''; + const devClass = showDevCopy ? 'relative group/dev-card' : ''; return (
+ {showDevCopy ? : null} {children}
); diff --git a/web/src/components/ChartCard.tsx b/web/src/components/ChartCard.tsx index 31613ff..6a15ba2 100644 --- a/web/src/components/ChartCard.tsx +++ b/web/src/components/ChartCard.tsx @@ -1,6 +1,7 @@ import type { ReactNode } from 'react'; import HelpHint, { normalizeHintContent, type HelpHintContent } from './HelpHint'; +import DevCopyJsonButton from './DevCopyJsonButton'; export interface ChartCardProps { title: string; @@ -9,6 +10,7 @@ export interface ChartCardProps { heightClass?: string; children?: ReactNode; className?: string; + devData?: unknown; } export default function ChartCard({ @@ -18,11 +20,14 @@ export default function ChartCard({ heightClass = 'h-56', children, className = '', + devData, }: ChartCardProps) { const hintContent = normalizeHintContent(hint); + const showDevCopy = import.meta.env.DEV && devData != null; return ( -
+
+ {showDevCopy ? : null}

{title}

{hintContent ? ( diff --git a/web/src/components/DevCopyJsonButton.tsx b/web/src/components/DevCopyJsonButton.tsx new file mode 100644 index 0000000..ad1fc77 --- /dev/null +++ b/web/src/components/DevCopyJsonButton.tsx @@ -0,0 +1,48 @@ +import { useMemo, useState, type MouseEvent } from 'react'; +import { Braces, Check } from 'lucide-react'; +import { strings } from '@/lib/strings'; + +export interface DevCopyJsonButtonProps { + data: unknown; + className?: string; +} + +function serializeDevJson(data: unknown): string { + try { + return JSON.stringify(data, null, 2); + } catch { + return JSON.stringify({ error: 'Could not serialize widget data' }); + } +} + +/** Dev-only overlay button — copies widget JSON to the clipboard. Stripped from production builds. */ +export default function DevCopyJsonButton({ data, className = '' }: DevCopyJsonButtonProps) { + const [copied, setCopied] = useState(false); + const text = useMemo(() => serializeDevJson(data), [data]); + + if (!import.meta.env.DEV) return null; + + const copy = (event: MouseEvent): void => { + event.stopPropagation(); + void navigator.clipboard.writeText(text).then(() => { + setCopied(true); + setTimeout(() => setCopied(false), 1500); + }); + }; + + return ( + + ); +} diff --git a/web/src/components/content/RichResultsValidationPanel.tsx b/web/src/components/content/RichResultsValidationPanel.tsx index 1339cde..fcf0849 100644 --- a/web/src/components/content/RichResultsValidationPanel.tsx +++ b/web/src/components/content/RichResultsValidationPanel.tsx @@ -13,6 +13,7 @@ import { buildRichResultsContext } from '@/lib/fixSuggestionContext'; interface RichResultsValidationPanelProps { rows: RichResultsValidationRow[]; meta?: RichResultsMeta | null; + devData?: unknown; } function statusBadgeVariant(status: string): string { @@ -29,7 +30,7 @@ function statusBadgeVariant(status: string): string { } } -export default function RichResultsValidationPanel({ rows, meta }: RichResultsValidationPanelProps) { +export default function RichResultsValidationPanel({ rows, meta, devData }: RichResultsValidationPanelProps) { const vca = strings.views.contentAnalytics; const columns = useMemo((): TableColumn[] => [ { key: 'url', label: vca.richResultsColUrl }, @@ -60,7 +61,7 @@ export default function RichResultsValidationPanel({ rows, meta }: RichResultsVa (meta.api_count ?? 0) === 0; return ( - +

{vca.richResultsTitle}

diff --git a/web/src/components/google/UrlCoverageDoughnut.tsx b/web/src/components/google/UrlCoverageDoughnut.tsx index bfdfdc7..996e5e9 100644 --- a/web/src/components/google/UrlCoverageDoughnut.tsx +++ b/web/src/components/google/UrlCoverageDoughnut.tsx @@ -12,9 +12,10 @@ ChartJS.register(ArcElement, Tooltip, Legend); interface UrlCoverageDoughnutProps { urlJoin: UrlJoinData | null | undefined; + devData?: unknown; } -export default function UrlCoverageDoughnut({ urlJoin }: UrlCoverageDoughnutProps) { +export default function UrlCoverageDoughnut({ urlJoin, devData }: UrlCoverageDoughnutProps) { const sp = strings.views.searchPerformance; const chart = useMemo(() => { @@ -52,6 +53,7 @@ export default function UrlCoverageDoughnut({ urlJoin }: UrlCoverageDoughnutProp hint={sp.charts.coverageHint} ariaLabel={sp.charts.coverageAria} heightClass="h-48" + devData={devData} >
{strings.common.notEnoughData} @@ -66,6 +68,7 @@ export default function UrlCoverageDoughnut({ urlJoin }: UrlCoverageDoughnutProp hint={sp.charts.coverageHint} ariaLabel={sp.charts.coverageAria} heightClass="h-48" + devData={devData} > diff --git a/web/src/components/google/UrlGapListsPanel.tsx b/web/src/components/google/UrlGapListsPanel.tsx index 7ab95c7..8ad1052 100644 --- a/web/src/components/google/UrlGapListsPanel.tsx +++ b/web/src/components/google/UrlGapListsPanel.tsx @@ -6,6 +6,7 @@ import { buildLinksInspectHref } from '../../lib/reportNav'; import { filterBySearch, exportCsv } from './tableUtils'; import GoogleTableToolbar from './GoogleTableToolbar'; import SortablePaginatedTable from './SortablePaginatedTable'; +import DevCopyJsonButton from '@/components/DevCopyJsonButton'; import type { TableColumn, UrlGapRow, UrlJoinData } from '@/types/components'; interface UrlGapSegment { @@ -21,6 +22,7 @@ interface UrlGapListsPanelProps { showGsc?: boolean; showGa4?: boolean; showCrawl?: boolean; + devData?: unknown; } /** @@ -32,6 +34,7 @@ export default function UrlGapListsPanel({ showGsc = true, showGa4 = true, showCrawl = true, + devData, }: UrlGapListsPanelProps) { const sp = strings.views.searchPerformance; const cg = strings.components.urlGapLists; @@ -156,7 +159,8 @@ export default function UrlGapListsPanel({ if (!segments.length) return null; return ( -
+
+ {devData != null ? : null}
{segments.map((seg) => (