diff --git a/src/MUI.Catalog/Facets.cs b/src/MUI.Catalog/Facets.cs new file mode 100644 index 0000000..5340de3 --- /dev/null +++ b/src/MUI.Catalog/Facets.cs @@ -0,0 +1,594 @@ +namespace MUI.Catalog; + +/// +/// The name of every facet, in the one spelling the query, the read API and the site's GET form all +/// use (spec §9). +/// +/// +/// These are querystring parameter names as much as they are facet identifiers, and that is the +/// point: q and archived were public on /games before there was a panel, and an +/// API that invented search beside them would have given one question two spellings and let +/// them drift. Naming them once, here, is what makes "the page and the API agree" a fact about the +/// code rather than a convention somebody has to remember. +/// +public static class FacetKeys +{ + public const string Text = "q"; + + public const string Archived = "archived"; + + public const string Band = "band"; + + public const string LastSeen = "seen"; + + public const string Protocol = "protocol"; + + public const string Tls = "tls"; + + public const string Charset = "charset"; + + public const string Language = "language"; + + public const string Codebase = "codebase"; + + public const string Family = "family"; + + public const string Genre = "genre"; +} + +/// +/// Which side of §3.1 a facet reads. Rendered beside every group, never inferred by the reader. +/// +/// +/// The distinction is the product. A measured facet answers "we watched this happen"; a declared one +/// answers "the game typed this into mush.cnf, possibly in 2017". Both are worth filtering on +/// and they are not the same question, so a panel that presented them identically would be making +/// the exact claim this site exists to stop making. +/// +public enum FacetEvidence +{ + Measured, + + Declared, +} + +/// How a facet combines with itself. +public enum FacetKind +{ + /// One value at a time; choosing another replaces it. + Choice, + + /// + /// A set of things we observed, intersected. Checking two asks for games that offered both. + /// + /// + /// There is deliberately no way to ask for the complement. "Games that do not offer GMCP" is a + /// question the data cannot answer: a capability is written true when it was observed and + /// is otherwise not written at all (see FieldObservations.Measured), because this client + /// requests only some options and a server that was never asked has not declined. A checkbox + /// whose unchecked state meant "no" would publish our own instrumentation as a fact about + /// somebody's game. + /// + Presence, +} + +/// +/// One choice-facet selection: a value the data carries, or the absence of one. +/// +/// +/// The absence is a first-class member rather than an empty string, because the whole site turns on +/// unknown and no being different facts. "Games whose codebase we could not identify" is a +/// real and useful question; it is not "games with no codebase", and neither is it a games-with-any +/// filter left blank. Modelling it as a value would let one be typed where the other was meant. +/// +public sealed record FacetChoice(string? Value) +{ + /// + /// The querystring spelling of the absence. Tilde-prefixed so it cannot collide with a real + /// value: a game may legitimately be called none and none may legitimately be a genre. + /// + public const string UnknownToken = "~unknown"; + + /// Games for which this facet has no value at all. + public static readonly FacetChoice Unknown = new((string?)null); + + public static FacetChoice Of(string value) => new(value); + + public bool IsUnknown => Value is null; + + /// What this selection is called in a URL. + public string Token => Value ?? UnknownToken; + + public static FacetChoice Parse(string token) => + string.Equals(token, UnknownToken, StringComparison.Ordinal) ? Unknown : Of(token); + + /// Whether a game whose value for this facet is matches. + public bool Matches(string? actual) => + IsUnknown ? actual is null : string.Equals(actual, Value, StringComparison.OrdinalIgnoreCase); +} + +/// +/// The last-seen facet (spec §9), measured from game.last_reachable_at. +/// +/// +/// The first three nest — a game seen in the last hour is in all of them — because "seen within a +/// week" is the question people actually have, and cutting it into exclusive rings would make the +/// common case two clicks that cannot both be made. The counts stay honest under nesting: each says +/// exactly how many games choosing it returns. +/// +/// is a value rather than an unknown, and that is the whole reason it exists +/// separately from . A game we have listed and never once reached is a different +/// fact from one we reached in 2023, and rendering the two the same way would let our own crawl +/// history read as somebody's outage. +/// +/// +public enum LastSeenBand +{ + Day, + + Week, + + Month, + + Older, + + Never, +} + +/// +/// One value of one facet, with how many games choosing it returns. +/// +/// +/// is not decoration and is not an estimate: it is computed from the same pass +/// that produced the listing beside it (see ), so a facet cannot promise +/// results it will not deliver. A value nothing matches is never offered at all — the one exception +/// is a value that is currently selected, which stays visible at zero so it can be seen and undone. +/// +public sealed record FacetValue(string Token, int Count, bool IsSelected, bool IsUnknown); + +/// One facet, ready to render: what it is called, what it reads, and what it offers. +/// +/// is what dropping this facet's own selection returns — the number an "any" +/// option produces. It is carried rather than summed from because an +/// open-ended facet offers only its commonest values, so the sum is short of the truth by however +/// long the tail is, and a control labelled with a number smaller than the set it selects is the +/// same lie in the other direction. +/// +public sealed record FacetGroup( + string Key, + FacetEvidence Evidence, + FacetKind Kind, + int Total, + IReadOnlyList Values) +{ + /// Whether anything is selected here, which is what a "clear this" affordance needs. + public bool IsFiltered => Values.Any(v => v.IsSelected); +} + +/// The listing and the facets that describe it, from one pass over one set of games. +/// +/// They are returned together rather than fetched separately on purpose. Two queries would be two +/// answers to two slightly different questions, and the first time they disagreed the panel would be +/// advertising a count the listing could not produce. +/// +public sealed record GameListing(IReadOnlyList Games, IReadOnlyList Facets) +{ + public static readonly GameListing Empty = new([], []); +} + +/// +/// One game reduced to the values every facet reads, so the facets are computed once from one shape. +/// +/// +/// +/// Assembling this is each implementation's job — Postgres builds it from +/// game_field rows and the presence digest, the demo fixture builds it from constants — and +/// filtering and counting it is 's, once. That split is what stops the +/// fixture and the database from quietly answering the same filter differently, which they already +/// did for band=archived before this type existed. +/// +/// +/// is the negotiated charset and never the game's MSSP claim about +/// one, and is an endpoint we actually completed a TLS connection to +/// rather than an SSL line in a self-description. Both are named for what they are so that a +/// later reader wiring them up cannot reach for the declared column by accident. +/// +/// +public sealed record GameFacetRow( + GameSummary Summary, + ActivityBand Band, + LastSeenBand LastSeen, + bool TlsMeasured, + string? Charset, + string? Language, + string? Codebase, + string? Family, + string? Genre); + +/// +/// Turns a filter and a set of games into the listing plus every facet's counts. +/// +/// +/// +/// Counts are measured against the set each choice would actually return. A +/// facet replaces its own selection, so its values are counted with +/// that selection lifted and every other filter still applied — the number beside quiet is +/// how many games you get by clicking quiet, not how many quiet games exist. A +/// facet intersects, so its values are counted against the current +/// results — the number beside GMCP is how many of the games on screen also offered it. The +/// two denominators differ because the two gestures differ, and both answer the same question: what +/// happens if I click this. +/// +/// +/// A value with no games is not offered. That is not tidiness — a facet that can be clicked into an +/// empty listing is a facet lying about the catalogue, and the cheapest way to make that impossible +/// is to never draw the click. +/// +/// +public static class FacetedSearch +{ + /// + /// How many values an open-ended facet offers. Codebases are versioned strings and there are + /// hundreds of them; the tail is reachable by search and by URL, and the panel says as much. + /// + public const int MaxValues = 12; + + public static GameListing Search(IReadOnlyList rows, GameFilter filter) + { + ArgumentNullException.ThrowIfNull(rows); + ArgumentNullException.ThrowIfNull(filter); + + // Archived games leave the default listing and nothing else (spec §7.5). Asking for the + // archived band *is* asking for them, so the toggle does not also have to be set — the + // database and the demo fixture disagreed about that until this became one function. + var wantsArchived = filter.IncludeArchived || filter.Band is ActivityBand.Archived; + + var baseRows = rows + .Where(r => (wantsArchived || r.Band is not ActivityBand.Archived) && MatchesText(r, filter.Text)) + .ToList(); + + var results = baseRows.Where(r => Chosen(r, filter, null) && Present(r, filter)).ToList(); + + var groups = new List(); + + foreach (var facet in Choices) + { + // This facet's own selection lifted, so a count is what choosing the value returns. + var domain = baseRows.Where(r => Chosen(r, filter, facet.Key) && Present(r, filter)).ToList(); + var values = facet.Bounded is { } vocabulary + ? Bounded(domain, facet, vocabulary, filter) + : Open(domain, facet, filter); + + if (values.Count > 0) + { + groups.Add(new FacetGroup( + facet.Key, facet.Evidence, FacetKind.Choice, domain.Count, values)); + } + } + + groups.AddRange(Presence(results, filter)); + + return new GameListing([.. results.Select(r => r.Summary)], groups); + } + + /// + /// The last-seen band a game is in, given when it was last reachable. + /// + /// + /// Null is and never the oldest bucket: a game we have listed + /// and never once reached has no last-seen date, and dating it from our own ignorance would be + /// the same error as painting an unprobed hour as an outage. + /// + public static LastSeenBand LastSeenOf(DateTimeOffset? lastReachableAt, DateTimeOffset now) => + lastReachableAt is not { } seen ? LastSeenBand.Never + : now - seen <= TimeSpan.FromDays(1) ? LastSeenBand.Day + : now - seen <= TimeSpan.FromDays(7) ? LastSeenBand.Week + : now - seen <= TimeSpan.FromDays(30) ? LastSeenBand.Month + : LastSeenBand.Older; + + /// + /// A game matches the text box on its name, its own one-line tagline, or its codebase. + /// + /// + /// Here rather than in SQL because the facet counts are computed over the same set the listing + /// is, and a search term applied in one place and counted in another is two answers to one + /// question. The cost is a pass over the catalogue per request, which is what every count in the + /// panel already costs; the point at which that stops being affordable is a GROUP BY per + /// facet in the database, and the counts would then need pinning against the listing rather than + /// being the same arithmetic by construction. + /// + private static bool MatchesText(GameFacetRow row, string? text) + { + if (string.IsNullOrWhiteSpace(text)) + { + return true; + } + + var needle = text.Trim(); + + return row.Summary.Name.Contains(needle, StringComparison.OrdinalIgnoreCase) + || (row.Summary.Tagline?.Contains(needle, StringComparison.OrdinalIgnoreCase) ?? false) + || (row.Codebase?.Contains(needle, StringComparison.OrdinalIgnoreCase) ?? false); + } + + private static bool Chosen(GameFacetRow row, GameFilter filter, string? except) + { + foreach (var facet in Choices) + { + if (string.Equals(facet.Key, except, StringComparison.Ordinal)) + { + continue; + } + + if (facet.SelectionOf(filter) is { } selection + && !facet.TokensOf(row).Any(selection.Matches)) + { + return false; + } + } + + return true; + } + + /// + /// The presence facets, which intersect. Every one of them reads a measurement — a protocol the + /// handshake offered, or an endpoint we completed a TLS connection to. + /// + private static bool Present(GameFacetRow row, GameFilter filter) => + filter.MeasuredProtocols.All( + p => row.Summary.MeasuredProtocols.Contains(p, StringComparer.OrdinalIgnoreCase)) + && (!filter.Tls || row.TlsMeasured); + + private static IEnumerable Presence(IReadOnlyList results, GameFilter filter) + { + var protocols = results + .SelectMany(r => r.Summary.MeasuredProtocols) + .GroupBy(p => p, StringComparer.OrdinalIgnoreCase) + .Select(g => new { Name = g.Key, Count = g.Count() }) + .ToList(); + + // A selected protocol has already narrowed the results, so every remaining game has it: its + // count is the listing's own size, which is what unchecking it would leave in place. + var values = protocols + .Select(p => new FacetValue( + p.Name, + Selected(p.Name) ? results.Count : p.Count, + Selected(p.Name), + IsUnknown: false)) + .Concat(filter.MeasuredProtocols + .Where(p => !protocols.Any(known => string.Equals(known.Name, p, StringComparison.OrdinalIgnoreCase))) + .Select(p => new FacetValue(p, 0, IsSelected: true, IsUnknown: false))) + .OrderByDescending(v => v.Count) + .ThenBy(v => v.Token, StringComparer.Ordinal) + .ToList(); + + if (values.Count > 0) + { + yield return new FacetGroup( + FacetKeys.Protocol, FacetEvidence.Measured, FacetKind.Presence, results.Count, values); + } + + var tls = filter.Tls ? results.Count : results.Count(r => r.TlsMeasured); + + // Rendered only when something was measured. Nothing writes a TLS endpoint today — the + // crawler dials plaintext — so this group is normally absent, which is the honest rendering + // of a measurement nobody has taken. It must never be filled in from MSSP's SSL line. + if (tls > 0 || filter.Tls) + { + yield return new FacetGroup( + FacetKeys.Tls, + FacetEvidence.Measured, + FacetKind.Presence, + results.Count, + [new FacetValue("yes", tls, filter.Tls, IsUnknown: false)]); + } + + bool Selected(string protocol) => + filter.MeasuredProtocols.Contains(protocol, StringComparer.OrdinalIgnoreCase); + } + + /// A fixed vocabulary, kept in its declared order because that order is a scale. + private static List Bounded( + IReadOnlyList domain, + ChoiceFacet facet, + IReadOnlyList vocabulary, + GameFilter filter) + { + var selection = facet.SelectionOf(filter); + var counts = Counts(domain, facet); + + return + [ + .. vocabulary + .Select(token => new FacetValue( + token, + counts.GetValueOrDefault(token), + selection?.Matches(token) ?? false, + IsUnknown: false)) + .Where(v => v.Count > 0 || v.IsSelected), + ]; + } + + /// + /// An open-ended vocabulary — codebases, genres — ordered by how much of the catalogue each + /// covers, capped, with the unknown bucket kept whatever it weighs. + /// + /// + /// The unknown bucket survives the cap deliberately. "Games whose codebase we could not + /// identify" is a measurement of our own reach and one of the more useful things in the panel, + /// and it is exactly the value a popularity cap would delete on a well-covered catalogue. + /// + private static List Open( + IReadOnlyList domain, + ChoiceFacet facet, + GameFilter filter) + { + var selection = facet.SelectionOf(filter); + var counts = Counts(domain, facet); + + var named = counts + .Where(c => !string.Equals(c.Key, FacetChoice.UnknownToken, StringComparison.Ordinal)) + .Select(c => new FacetValue( + c.Key, c.Value, selection?.Matches(c.Key) ?? false, IsUnknown: false)) + .OrderByDescending(v => v.IsSelected) + .ThenByDescending(v => v.Count) + .ThenBy(v => v.Token, StringComparer.Ordinal) + .Take(MaxValues) + .OrderByDescending(v => v.Count) + .ThenBy(v => v.Token, StringComparer.Ordinal) + .ToList(); + + var unknown = counts.GetValueOrDefault(FacetChoice.UnknownToken); + var unknownSelected = selection?.IsUnknown ?? false; + + if (unknown > 0 || unknownSelected) + { + named.Add(new FacetValue(FacetChoice.UnknownToken, unknown, unknownSelected, IsUnknown: true)); + } + + return named; + } + + private static Dictionary Counts(IReadOnlyList domain, ChoiceFacet facet) + { + var counts = new Dictionary(StringComparer.OrdinalIgnoreCase); + + foreach (var row in domain) + { + foreach (var token in facet.TokensOf(row)) + { + var key = token ?? FacetChoice.UnknownToken; + counts[key] = counts.GetValueOrDefault(key) + 1; + } + } + + return counts; + } + + /// + /// One choice facet: which of its values a game is in, and what the filter says about it. + /// + /// + /// returns a list because one facet's values nest: a game + /// reached an hour ago is in "the last 24 hours" and in "the last 7 days" both, and the question + /// people have is the second. Every other facet returns one token, or one null for a game the + /// facet has no value for. + /// + private sealed record ChoiceFacet( + string Key, + FacetEvidence Evidence, + Func> TokensOf, + Func SelectionOf, + IReadOnlyList? Bounded = null); + + /// + /// Every choice facet, in the order the panel shows them: what we measured first, then what the + /// game says about itself. The order is editorial and the labelling is not — a reader has to be + /// able to see which half of the panel is evidence. + /// + private static readonly ChoiceFacet[] Choices = + [ + new( + FacetKeys.Band, + FacetEvidence.Measured, + r => [FacetTokens.Of(r.Band)], + f => f.Band is { } band ? FacetChoice.Of(FacetTokens.Of(band)) : null, + FacetTokens.Bands), + new( + FacetKeys.LastSeen, + FacetEvidence.Measured, + r => FacetTokens.Reaching(r.LastSeen), + f => f.LastSeen is { } seen ? FacetChoice.Of(FacetTokens.Of(seen)) : null, + FacetTokens.LastSeenBands), + new(FacetKeys.Charset, FacetEvidence.Measured, r => [r.Charset], f => f.Charset), + new(FacetKeys.Codebase, FacetEvidence.Declared, r => [r.Codebase], f => f.Codebase), + new(FacetKeys.Family, FacetEvidence.Declared, r => [r.Family], f => f.Family), + new(FacetKeys.Genre, FacetEvidence.Declared, r => [r.Genre], f => f.Genre), + new(FacetKeys.Language, FacetEvidence.Declared, r => [r.Language], f => f.Language), + ]; +} + +/// +/// How the two derived facets' values are spelled in a URL. +/// +/// +/// The spelling lives beside the enums rather than in whichever surface parses a querystring, +/// because the facet panel emits these tokens and the filter binding reads them: if the two had +/// separate tables, the panel could offer a value its own parser would reject. +/// +public static class FacetTokens +{ + public static IReadOnlyList Bands { get; } = + [.. Enum.GetValues().Select(Of)]; + + public static IReadOnlyList LastSeenBands { get; } = + [.. Enum.GetValues().Select(Of)]; + + /// The three windows that nest, widest last. + private static readonly string?[] Nested = + [Of(LastSeenBand.Day), Of(LastSeenBand.Week), Of(LastSeenBand.Month)]; + + /// + /// Every last-seen value a game in answers to. + /// + /// + /// The first three nest: a game reached an hour ago is in the last 24 hours, the last 7 days and + /// the last 30. Cutting them into exclusive rings would make "seen within a week" — the question + /// people actually have — two clicks that cannot both be made. The tails do not nest, because + /// is not a longer version of : + /// a game we have never once reached has no date, and lending it the oldest one would publish + /// our own ignorance as its outage. + /// + public static IReadOnlyList Reaching(LastSeenBand band) => band switch + { + LastSeenBand.Day => Nested[..3], + LastSeenBand.Week => Nested[1..3], + LastSeenBand.Month => Nested[2..3], + LastSeenBand.Older => [Of(LastSeenBand.Older)], + _ => [Of(LastSeenBand.Never)], + }; + + public static string Of(ActivityBand band) => Camel(band.ToString()); + + public static string Of(LastSeenBand band) => Camel(band.ToString()); + + public static bool TryBand(string? text, out ActivityBand band) => TryRead(text, out band); + + public static bool TryLastSeen(string? text, out LastSeenBand band) => TryRead(text, out band); + + /// + /// Reads one of the derived vocabularies, forgivingly about separators and strictly about + /// everything else. + /// + /// + /// Hyphens and underscores are stripped so active-this-week, active_this_week and + /// activeThisWeek are one facet rather than three near misses. Digits are refused + /// outright: also accepts the + /// underlying number, which would make band=0 a synonym for whichever member happens to + /// be declared first — a facet that silently re-points itself the day somebody reorders the + /// enum. Only the names are public. + /// + private static bool TryRead(string? text, out TEnum value) + where TEnum : struct, Enum + { + value = default; + + if (string.IsNullOrWhiteSpace(text)) + { + return false; + } + + var normalised = text.Trim() + .Replace("-", string.Empty, StringComparison.Ordinal) + .Replace("_", string.Empty, StringComparison.Ordinal); + + if (normalised.Length == 0 || normalised.All(char.IsAsciiDigit)) + { + return false; + } + + return Enum.TryParse(normalised, ignoreCase: true, out value) && Enum.IsDefined(value); + } + + private static string Camel(string name) => char.ToLowerInvariant(name[0]) + name[1..]; +} diff --git a/src/MUI.Catalog/Persistence/NpgsqlGameQueries.cs b/src/MUI.Catalog/Persistence/NpgsqlGameQueries.cs index b2d231e..34d8558 100644 --- a/src/MUI.Catalog/Persistence/NpgsqlGameQueries.cs +++ b/src/MUI.Catalog/Persistence/NpgsqlGameQueries.cs @@ -81,7 +81,28 @@ public sealed class NpgsqlGameQueries(NpgsqlDataSource source, IFieldRegistry? r /// public Func Clock { get; init; } = () => DateTimeOffset.UtcNow; - public async Task> ListAsync( + /// + /// The listing and its facets (spec §9), from one pass over one set of games. + /// + /// + /// + /// The database narrows on the one thing that is not a facet — the archive toggle — and + /// everything else is decided by over . + /// That is deliberate rather than laziness about SQL: a facet count has to be measured against + /// the same set the listing came from, and a WHERE clause that filtered here beside a + /// GROUP BY that counted there would be two answers to one question. Sharing the + /// arithmetic also means the demo fixture and this class cannot disagree about what a filter + /// means, which they already did for band=archived. + /// + /// + /// The cost is a pass over the unarchived catalogue and its fields per listing request — the + /// same order as before, since FieldsForAsync already read every field of every listed + /// game. The point at which that stops being affordable is aggregation in the database, and the + /// counts would then need pinning against the listing rather than being the same arithmetic by + /// construction. + /// + /// + public async Task SearchAsync( GameFilter filter, CancellationToken cancellationToken = default) { @@ -91,9 +112,11 @@ public async Task> ListAsync( await using var connection = await source.OpenConnectionAsync(cancellationToken); - var capabilityFields = filter.MeasuredProtocols - .Select(CapabilityFields.Measured) - .ToArray(); + // Archived games leave the default listing and nothing else (spec §7.5) — and asking for the + // archived band is asking for them, so it lifts the exclusion by itself. Without that the one + // facet value naming the archive returned nothing at all, while the fixture returned the + // archive: one filter, two answers, and only one of them was tested. + var includeArchived = filter.IncludeArchived || filter.Band is ActivityBand.Archived; var rows = (await connection.QueryAsync(new CommandDefinition( """ @@ -101,47 +124,30 @@ public async Task> ListAsync( g.state AS State, g.is_claimed AS IsClaimed, g.last_reachable_at AS LastReachableAt FROM game g WHERE (@includeArchived OR g.state <> 'archived') - AND (@text IS NULL OR g.name ILIKE @text) - AND (cardinality(@capabilityFields::text[]) = 0 OR ( - SELECT count(DISTINCT f.field) - FROM game_field f - WHERE f.game_id = g.id - AND f.field = ANY(@capabilityFields) - AND f.value = 'true') = cardinality(@capabilityFields::text[])) ORDER BY g.name """, - new - { - includeArchived = filter.IncludeArchived, - text = string.IsNullOrWhiteSpace(filter.Text) ? null : $"%{filter.Text.Trim()}%", - capabilityFields, - }, + new { includeArchived }, cancellationToken: cancellationToken))).ToList(); if (rows.Count == 0) { - return []; + return GameListing.Empty; } var ids = rows.Select(row => row.Id).ToArray(); var fields = await FieldsForAsync(connection, ids, cancellationToken); var presence = await PresenceDigestAsync(connection, ids, now, cancellationToken); + var tls = await TlsEndpointsAsync(connection, ids, cancellationToken); - var summaries = new List(rows.Count); + var facetRows = new List(rows.Count); foreach (var row in rows) { var forGame = fields.TryGetValue(row.Id, out var list) ? list : []; var digest = presence.TryGetValue(row.Id, out var found) ? found : PresenceDigest.None; var state = SqlEnums.ToLifecycleState(row.State); - var band = BandOf(state, row.LastReachableAt, digest, now); - - if (filter.Band is { } wanted && band != wanted) - { - continue; - } - summaries.Add(new GameSummary( + var summary = new GameSummary( row.Id, row.Slug, row.Name, @@ -150,10 +156,72 @@ ORDER BY g.name row.IsClaimed, digest.CountNow, Winner(forGame, "CODEBASE")?.Value, - MeasuredProtocolsOf(forGame))); + MeasuredProtocolsOf(forGame), + row.LastReachableAt); + + facetRows.Add(new GameFacetRow( + summary, + BandOf(state, row.LastReachableAt, digest, now), + FacetedSearch.LastSeenOf(row.LastReachableAt, now), + TlsMeasured: tls.Contains(row.Id), + Charset: NegotiatedCharset(forGame), + Language: Winner(forGame, "LANGUAGE")?.Value, + Codebase: summary.Codebase, + Family: Winner(forGame, "FAMILY")?.Value, + Genre: Winner(forGame, "GENRE")?.Value)); } - return summaries; + return FacetedSearch.Search(facetRows, filter); + } + + /// A listing with no panel — the same query, projected. + public async Task> ListAsync( + GameFilter filter, + CancellationToken cancellationToken = default) => + (await SearchAsync(filter, cancellationToken)).Games; + + /// + /// The encoding CHARSET settled on, and never the game's MSSP claim about one. + /// + /// + /// Deliberately not the precedence winner. CHARSET is one of the few fields both a + /// handshake and MSSP write, so the winner is the handshake's when there is one and + /// silently the game's own assertion when there is not — which would make a facet advertised as + /// measured answer from the declared column for every server that never negotiates, without + /// saying so anywhere. Games with no measurement belong in the unknown bucket, which is a + /// different answer and an honest one. + /// + private static string? NegotiatedCharset(IReadOnlyList fields) => + fields.FirstOrDefault(f => + string.Equals(f.Field, "CHARSET", StringComparison.Ordinal) + && f.Source is FieldSource.Handshake)?.Value; + + /// + /// The games we have completed a TLS connection to. + /// + /// + /// An endpoint row, not a capability claim. capability.ssl.declared exists and says only + /// that somebody typed SSL 4202 into their configuration; an endpoint of kind tls + /// says a socket was opened. Nothing writes one yet — CatalogueBinder records what it + /// dialled and the crawler dials plaintext — so this comes back empty and the facet does not + /// render at all, which is the honest rendering of a measurement nobody has taken. It becomes a + /// real facet the day the crawler takes it, with no change here. + /// + private static async Task> TlsEndpointsAsync( + NpgsqlConnection connection, + Guid[] ids, + CancellationToken cancellationToken) + { + var rows = await connection.QueryAsync(new CommandDefinition( + """ + SELECT DISTINCT game_id + FROM game_endpoint + WHERE game_id = ANY(@ids) AND kind = 'tls' AND state <> 'gone' + """, + new { ids }, + cancellationToken: cancellationToken)); + + return [.. rows]; } public async Task FindAsync(string slug, CancellationToken cancellationToken = default) @@ -197,7 +265,8 @@ FROM game row.IsClaimed, digest.CountNow, Winner(fields, "CODEBASE")?.Value, - MeasuredProtocolsOf(fields)); + MeasuredProtocolsOf(fields), + row.LastReachableAt); return new GamePage( summary, diff --git a/src/MUI.Catalog/Views.cs b/src/MUI.Catalog/Views.cs index a9aefbc..148ea17 100644 --- a/src/MUI.Catalog/Views.cs +++ b/src/MUI.Catalog/Views.cs @@ -78,6 +78,12 @@ public sealed record ActivityCell(int DayOfWeek, int Hour, int? Count, bool Prob } /// A game as the listing shows it. +/// +/// is carried because the last-seen facet (spec §9) filters on it, and +/// a facet whose value cannot be read off the rows it returned is one a reader has to take on trust. +/// Null means we have never once reached the game, which is a different fact from "reachable a long +/// time ago" and is never rendered as the older of the two. +/// public sealed record GameSummary( Guid Id, string Slug, @@ -87,7 +93,8 @@ public sealed record GameSummary( bool IsClaimed, int? PlayersNow, string? Codebase, - IReadOnlyList MeasuredProtocols); + IReadOnlyList MeasuredProtocols, + DateTimeOffset? LastReachableAt = null); /// A game as its own page shows it. /// @@ -116,15 +123,49 @@ public sealed record GameEndpointView(string Host, int Port, string Kind, bool T public sealed record ChangeEntry(DateTimeOffset At, string Summary); /// What the listing was asked for. A plain GET form's worth of state and nothing more. +/// +/// +/// Every member here is one control on the panel and one querystring parameter, named in +/// . That correspondence is what makes a filtered listing linkable, the back +/// button work and the read API answer the same question the page does — there is no filter state +/// anywhere else, and nothing here needs a session to mean something. +/// +/// +/// and read observations; , +/// , and read what a game says about +/// itself. is the odd one and is deliberately on the measured side: it is what +/// CHARSET settled on in the handshake, never the game's MSSP claim about an encoding. +/// +/// public sealed record GameFilter { public string? Text { get; init; } public bool IncludeArchived { get; init; } + /// + /// Protocols the handshake was observed offering, intersected. Never what MSSP declared — + /// capability.*.measured and capability.*.declared are two fields for exactly this + /// reason, and a facet reading the second would be the central lie of the project. + /// public IReadOnlyList MeasuredProtocols { get; init; } = []; + /// An endpoint we completed a TLS connection to — not an SSL line in MSSP. + public bool Tls { get; init; } + public ActivityBand? Band { get; init; } + + public LastSeenBand? LastSeen { get; init; } + + public FacetChoice? Charset { get; init; } + + public FacetChoice? Codebase { get; init; } + + public FacetChoice? Family { get; init; } + + public FacetChoice? Genre { get; init; } + + public FacetChoice? Language { get; init; } } /// @@ -152,7 +193,27 @@ public enum ActivityBand /// public interface IGameQueries { - Task> ListAsync(GameFilter filter, CancellationToken cancellationToken = default); + /// + /// The listing and the facet counts that describe it, from one pass (spec §9). + /// + /// + /// One method rather than a listing call and a counts call, because a facet must not be able to + /// lie about what a click will produce. Two calls are two answers to two slightly different + /// questions, and the first time they disagreed the panel would be advertising a count the + /// listing could not deliver. + /// + Task SearchAsync(GameFilter filter, CancellationToken cancellationToken = default); + + /// + /// Just the games — for the callers that want a listing and no panel. + /// + /// + /// Every implementation answers it by projecting , so there is no route + /// by which a caller that does not want facets gets a different listing from one that does. + /// + Task> ListAsync( + GameFilter filter, + CancellationToken cancellationToken = default); Task FindAsync(string slug, CancellationToken cancellationToken = default); diff --git a/src/MUI.Web/Api/ApiMapper.cs b/src/MUI.Web/Api/ApiMapper.cs index 0188bb8..e4377be 100644 --- a/src/MUI.Web/Api/ApiMapper.cs +++ b/src/MUI.Web/Api/ApiMapper.cs @@ -28,9 +28,30 @@ public static class ApiMapper Counted(game.PlayersNow), game.Codebase, game.MeasuredProtocols, + game.LastReachableAt, ApiRoutes.Page(game.Slug), ApiRoutes.Game(game.Id)); + /// + /// One facet, carried across exactly as the catalogue counted it. + /// + /// + /// Nothing is recomputed, re-ordered or trimmed here. A count is only trustworthy because it + /// came from the same pass as the listing beside it, and a mapper that adjusted one would break + /// that with no surface left to say so. + /// + public static FacetGroupView Facet(FacetGroup group) + { + ArgumentNullException.ThrowIfNull(group); + + return new FacetGroupView( + group.Key, + group.Evidence, + group.Kind, + group.Total, + [.. group.Values.Select(v => new FacetValueView(v.Token, v.Count, v.IsSelected, v.IsUnknown))]); + } + public static GameView Game( GamePage page, IReadOnlyList availability, diff --git a/src/MUI.Web/Api/ApiModels.cs b/src/MUI.Web/Api/ApiModels.cs index a4da374..2bb5755 100644 --- a/src/MUI.Web/Api/ApiModels.cs +++ b/src/MUI.Web/Api/ApiModels.cs @@ -142,6 +142,10 @@ public sealed record GameSummaryView( PlayerCountState PlayersNowState, string? Codebase, IReadOnlyList MeasuredProtocols, + + // Null means never once reached, which is not "reached a long time ago" — the last-seen facet + // (spec §9) keeps them apart and a consumer reading only the listing has to be able to as well. + DateTimeOffset? LastReachableAt, string Url, string ApiUrl); @@ -170,17 +174,71 @@ public sealed record GameView( string Url, string ApiUrl); -/// What the listing was asked for, echoed so a cached response says what it answers. +/// +/// What the listing was asked for, echoed so a cached response says what it answers. +/// +/// +/// One property per facet, named for the querystring parameter that sets it, built by +/// from the filter itself rather than from the raw query — so an echo cannot claim +/// a filter the query did not apply, and a facet added to and forgotten +/// here fails to compile rather than silently vanishing from the answer. +/// public sealed record FilterView( string? Q, bool IncludeArchived, IReadOnlyList Protocol, - ActivityBand? Band); + bool Tls, + ActivityBand? Band, + LastSeenBand? Seen, + string? Charset, + string? Codebase, + string? Family, + string? Genre, + string? Language) +{ + public static FilterView Of(GameFilter filter) + { + ArgumentNullException.ThrowIfNull(filter); + + return new FilterView( + filter.Text, + filter.IncludeArchived, + filter.MeasuredProtocols, + filter.Tls, + filter.Band, + filter.LastSeen, + filter.Charset?.Token, + filter.Codebase?.Token, + filter.Family?.Token, + filter.Genre?.Token, + filter.Language?.Token); + } +} + +/// One facet value as the API publishes it, with what choosing it returns. +public sealed record FacetValueView(string Value, int Count, bool Selected, bool Unknown); + +/// +/// One facet: its querystring name, whether it reads a measurement or a claim, and its values. +/// +/// +/// evidence is published rather than left for a consumer to infer, for the same reason the +/// page prints it: a facet reading capability.gmcp.measured and one reading MSSP's +/// GENRE are not the same kind of statement, and a client that presented them identically +/// would be making the claim this whole site exists to stop making. +/// +public sealed record FacetGroupView( + string Key, + FacetEvidence Evidence, + FacetKind Kind, + int Total, + IReadOnlyList Values); public sealed record GameListView( string ApiVersion, DateTimeOffset GeneratedAt, FilterView Filter, + IReadOnlyList Facets, int Total, int Limit, int Offset, diff --git a/src/MUI.Web/Api/GameEndpoints.cs b/src/MUI.Web/Api/GameEndpoints.cs index 9b7f39b..8890c46 100644 --- a/src/MUI.Web/Api/GameEndpoints.cs +++ b/src/MUI.Web/Api/GameEndpoints.cs @@ -29,14 +29,18 @@ public static async Task ListAsync(HttpContext http, IGameQueries queries, TimeP return; } - var matched = await queries.ListAsync(query.Filter, http.RequestAborted); - var page = matched.Skip(query.Offset).Take(query.Limit).Select(ApiMapper.Summary).ToList(); + // The facets come back with the listing rather than from a second call, so a consumer + // building a filter UI over this endpoint gets counts that describe the page it was handed + // — the same guarantee the site's own panel has, for the same reason. + var matched = await queries.SearchAsync(query.Filter, http.RequestAborted); + var page = matched.Games.Skip(query.Offset).Take(query.Limit).Select(ApiMapper.Summary).ToList(); await ApiResponse.WriteJsonAsync(http, new GameListView( ApiVersion.Current, ApiClock.Now(clock), query.Echo, - Total: matched.Count, + [.. matched.Facets.Select(ApiMapper.Facet)], + Total: matched.Games.Count, query.Limit, query.Offset, Count: page.Count, diff --git a/src/MUI.Web/Api/GameFilterBinding.cs b/src/MUI.Web/Api/GameFilterBinding.cs index 0186fb4..dd0ae51 100644 --- a/src/MUI.Web/Api/GameFilterBinding.cs +++ b/src/MUI.Web/Api/GameFilterBinding.cs @@ -1,3 +1,6 @@ +using Microsoft.AspNetCore.WebUtilities; +using Microsoft.Extensions.Primitives; + using MUI.Catalog; using MUI.Web.Components; @@ -7,19 +10,21 @@ namespace MUI.Web.Api; public sealed record GameQuery(GameFilter Filter, FilterView Echo, int Limit, int Offset); /// -/// Querystring to , in the spelling the site's own facet panel uses. +/// Querystring to — the one parser, with two callers. /// /// /// -/// The panel is a plain GET form, so its field names are the public query language: -/// q and archived already mean something on /games, and an API that invented -/// search and include=archived beside them would give one question two spellings and -/// let them drift. protocol and band are the remaining two members of -/// , named the way the panel will name them when it grows the controls. +/// The facet panel is a plain GET form, so its field names are the public query language, +/// and the page reads its own URL through this same function rather than binding each parameter for +/// itself. That is not tidiness: /games?band=quiet and /api/games?band=quiet have to +/// mean one thing, and a second binder is exactly how they stop meaning one thing. Every name comes +/// from , so the facet a query returns and the parameter that selects it +/// cannot be spelled differently. /// /// -/// An unrecognised band is a 400 rather than a silent empty filter. A consumer who typoed a -/// facet should be told, not handed the unfiltered catalogue and left to wonder. +/// An unrecognised band or seen is refused rather than ignored. A consumer who typoed +/// a facet should be told, not handed the unfiltered catalogue and left to read it as the answer — +/// the same rule as everywhere else here: our own silence must not be published as a fact. /// /// public static class GameFilterBinding @@ -28,42 +33,64 @@ public static class GameFilterBinding public const int MaxLimit = 500; + /// Reads a request's querystring — the API's caller. public static bool TryRead(IQueryCollection query, out GameQuery result, out string? error) + { + ArgumentNullException.ThrowIfNull(query); + + return TryRead(name => query[name], out result, out error); + } + + /// + /// Reads a raw querystring — the page's caller, which has a URL rather than an + /// because a static-SSR component is handed neither the request + /// nor a bound model it could share with the API. + /// + public static bool TryRead(string? queryString, out GameQuery result, out string? error) + { + var parsed = QueryHelpers.ParseQuery(queryString ?? string.Empty); + + return TryRead( + name => parsed.TryGetValue(name, out var values) ? values : StringValues.Empty, + out result, + out error); + } + + private static bool TryRead( + Func read, + out GameQuery result, + out string? error) { result = null!; - error = null; - ActivityBand? band = null; - var bandText = query["band"].ToString(); - if (!string.IsNullOrWhiteSpace(bandText)) + if (!TryBand(read, out var band, out error) || !TryLastSeen(read, out var seen, out error)) { - if (!TryBand(bandText, out var parsed)) - { - error = $"'{bandText}' is not an activity band. " - + "Accepted: playersNow, activeThisWeek, quiet, dark, archived."; - return false; - } - - band = parsed; + return false; } - var protocols = Protocols(query); - var text = query["q"].ToString(); - var includeArchived = Truthy.Is(query["archived"]); + var protocols = Protocols(read); + var text = read(FacetKeys.Text).ToString(); var filter = new GameFilter { Text = string.IsNullOrWhiteSpace(text) ? null : text, - IncludeArchived = includeArchived, + IncludeArchived = Truthy.Is(read(FacetKeys.Archived)), MeasuredProtocols = protocols, + Tls = Truthy.Is(read(FacetKeys.Tls)), Band = band, + LastSeen = seen, + Charset = Choice(read, FacetKeys.Charset), + Codebase = Choice(read, FacetKeys.Codebase), + Family = Choice(read, FacetKeys.Family), + Genre = Choice(read, FacetKeys.Genre), + Language = Choice(read, FacetKeys.Language), }; result = new GameQuery( filter, - new FilterView(filter.Text, includeArchived, protocols, band), - Bounded(query["limit"], DefaultLimit, 1, MaxLimit), - Bounded(query["offset"], 0, 0, int.MaxValue)); + FilterView.Of(filter), + Bounded(read("limit"), DefaultLimit, 1, MaxLimit), + Bounded(read("offset"), 0, 0, int.MaxValue)); return true; } @@ -72,33 +99,72 @@ public static bool TryRead(IQueryCollection query, out GameQuery result, out str /// Repeatable and comma-separated both work, because both are what people type. Every value is /// a measured protocol — a game's own claim never satisfies this facet (spec §3.1). /// - private static string[] Protocols(IQueryCollection query) => + private static string[] Protocols(Func read) => [ - .. query["protocol"] + .. read(FacetKeys.Protocol) .SelectMany(v => (v ?? string.Empty).Split(',', StringSplitOptions.RemoveEmptyEntries)) .Select(p => p.Trim()) .Where(p => p.Length > 0) .Distinct(StringComparer.OrdinalIgnoreCase), ]; - private static bool TryBand(string value, out ActivityBand band) + /// + /// An open-ended facet's selection, which may name the absence of a value. + /// + /// + /// A blank parameter is no selection at all; ~unknown is a selection of the games that + /// have no value. Folding the two together would make "games whose codebase we could not + /// identify" — a measurement of our own reach, and one of the more useful filters here — + /// unaskable, and would quietly re-point any URL that asked it. + /// + private static FacetChoice? Choice(Func read, string key) + { + var value = read(key).ToString(); + + return string.IsNullOrWhiteSpace(value) ? null : FacetChoice.Parse(value.Trim()); + } + + private static bool TryBand(Func read, out ActivityBand? band, out string? error) { - // Hyphens and underscores are stripped so active-this-week, active_this_week and - // activeThisWeek are one facet rather than three near misses. - var normalised = value.Replace("-", string.Empty, StringComparison.Ordinal) - .Replace("_", string.Empty, StringComparison.Ordinal); - - // Enum.TryParse also accepts the underlying number, which would make band=0 a synonym for - // whichever member happens to be declared first — a facet that silently re-points itself the - // day somebody reorders the enum. Only the names are public. - if (normalised.Length == 0 || normalised.All(char.IsAsciiDigit)) + band = null; + error = null; + var text = read(FacetKeys.Band).ToString(); + + if (string.IsNullOrWhiteSpace(text)) + { + return true; + } + + if (!FacetTokens.TryBand(text, out var parsed)) { - band = default; + error = $"'{text}' is not an activity band. Accepted: {string.Join(", ", FacetTokens.Bands)}."; return false; } - return Enum.TryParse(normalised, ignoreCase: true, out band) - && Enum.IsDefined(band); + band = parsed; + return true; + } + + private static bool TryLastSeen(Func read, out LastSeenBand? seen, out string? error) + { + seen = null; + error = null; + var text = read(FacetKeys.LastSeen).ToString(); + + if (string.IsNullOrWhiteSpace(text)) + { + return true; + } + + if (!FacetTokens.TryLastSeen(text, out var parsed)) + { + error = $"'{text}' is not a last-seen band. " + + $"Accepted: {string.Join(", ", FacetTokens.LastSeenBands)}."; + return false; + } + + seen = parsed; + return true; } private static int Bounded(string? value, int fallback, int min, int max) diff --git a/src/MUI.Web/Components/FacetPanel.razor b/src/MUI.Web/Components/FacetPanel.razor new file mode 100644 index 0000000..2115076 --- /dev/null +++ b/src/MUI.Web/Components/FacetPanel.razor @@ -0,0 +1,97 @@ +@* + The facet panel: one plain GET form, no script. + + The querystring *is* the state, so a filtered listing is linkable, the back button works, and the + server recomputes every count on every request. Nothing here is a control that remembers + something — there is no state anywhere but the URL, which is what makes a shared link the same + page for the person who receives it. + + Every count beside a value came from the same pass that produced the listing below it (see + FacetedSearch), so a choice cannot promise results it will not deliver, and a value nothing + matches is never drawn at all. +*@ + + + +@code { + [Parameter, EditorRequired] public IReadOnlyList Facets { get; set; } = []; + + [Parameter, EditorRequired] public GameFilter Filter { get; set; } = new(); + + private static string Evidence(FacetGroup group) => + group.Evidence is FacetEvidence.Measured ? "measured" : "declared"; +} diff --git a/src/MUI.Web/Components/FacetWords.cs b/src/MUI.Web/Components/FacetWords.cs new file mode 100644 index 0000000..c22c7b2 --- /dev/null +++ b/src/MUI.Web/Components/FacetWords.cs @@ -0,0 +1,109 @@ +using MUI.Catalog; + +namespace MUI.Web.Components; + +/// +/// The words the facets are shown in — on the rendered panel and in plain text alike. +/// +/// +/// +/// Wording lives here rather than beside the query because MUI.Catalog is UI-agnostic and a +/// facet's name is not its label: seen is a querystring parameter and "last seen" is +/// a phrase in English. It lives in one place rather than two because the graphical panel and the +/// plain surface are the same facts with different renderers, and a value called one thing in a +/// <select> and another in an 80-column list is two vocabularies again. +/// +/// +/// is the load-bearing one. Every facet spells its own absence, and none of +/// them spells it as a no — "not identified" is a fact about our reach, "not declared" is a +/// fact about what a game published, and neither is a fact about the game lacking the thing. +/// +/// +public static class FacetWords +{ + /// What a facet is called on the page. + public static string Group(string key) => key switch + { + FacetKeys.Band => "activity", + FacetKeys.LastSeen => "last seen", + FacetKeys.Protocol => "protocols offered", + FacetKeys.Tls => "encrypted", + FacetKeys.Charset => "encoding negotiated", + FacetKeys.Codebase => "codebase", + FacetKeys.Family => "family", + FacetKeys.Genre => "genre", + FacetKeys.Language => "language", + _ => key, + }; + + /// + /// How a facet's evidence is described, in three words a reader can act on. + /// + /// + /// Never abbreviated to a symbol. The difference between something we watched happen and + /// something a game typed into mush.cnf in 2017 is the product, and a legend a reader has + /// to learn is a difference they will not read. + /// + public static string Evidence(FacetEvidence evidence) => evidence switch + { + FacetEvidence.Measured => "we measured this", + _ => "the game says so", + }; + + /// One value's label. Open-ended facets are their own labels; the derived ones are not. + public static string Value(string key, FacetValue value) + { + ArgumentNullException.ThrowIfNull(value); + + if (value.IsUnknown) + { + return Unknown(key); + } + + return key switch + { + FacetKeys.Band => Band(value.Token), + FacetKeys.LastSeen => LastSeen(value.Token), + FacetKeys.Tls => "connected over TLS", + _ => value.Token, + }; + } + + /// + /// What "we have no value for this game" is called, per facet. + /// + /// + /// Three different sentences because they are three different facts. A codebase we could not + /// identify is a limit of our parsers; a genre nobody declared is a limit of what the game + /// published; an encoding nothing negotiated is a limit of the handshake. Rendering all three as + /// "unknown" would be true and would throw away the only part of the answer worth having. + /// + public static string Unknown(string key) => key switch + { + FacetKeys.Charset => "nothing negotiated", + FacetKeys.Codebase => "not identified", + _ => "not declared", + }; + + private static string Band(string token) => token switch + { + "playersNow" => "players on now", + "activeThisWeek" => "active this week", + "quiet" => "quiet — reachable, nobody counted", + "dark" => "dark — not reached in a month", + _ => "archived", + }; + + private static string LastSeen(string token) => token switch + { + "day" => "in the last 24 hours", + "week" => "in the last 7 days", + "month" => "in the last 30 days", + "older" => "longer ago", + + // Never reached, and deliberately not the oldest bucket: a game we have listed and never + // once got an answer from has no last-seen date at all, and dating it from our own ignorance + // would read as its outage. + _ => "never reached", + }; +} diff --git a/src/MUI.Web/Components/Layout/MainLayout.razor b/src/MUI.Web/Components/Layout/MainLayout.razor index a68a4d2..91e6554 100644 --- a/src/MUI.Web/Components/Layout/MainLayout.razor +++ b/src/MUI.Web/Components/Layout/MainLayout.razor @@ -24,6 +24,7 @@