From 47b73f24467f163720b9c59fee7dcf906457329d Mon Sep 17 00:00:00 2001 From: paoloredis Date: Mon, 17 Aug 2026 12:28:53 +0200 Subject: [PATCH] DOC-6979 Publish the versioned pages in sitemap.xml 3,134 live docs pages appear in no sitemap: every page under a versioned directory (operate/rs, operate/kubernetes, develop/ai/redisvl). The published sitemap carries 2,682 URLs where a full build renders 5,816, and the gap is exactly the versioned set -- verified by diffing the deployed file against a local unrestricted build, which reconciles to the URL with no remainder. Nothing was broken in the generator. The file fell between two jobs: - The `latest` build `rm -rf`s the version directories before Hugo runs, so its sitemap cannot list a versioned page. - Each versioned matrix build *does* render them, but its deploy uploads only the versioned subdirectory (`output/operate/rs/${version}`), and Hugo writes sitemap.xml at the root of `output/` -- one level above anything that ships. So both halves exist at build time and neither reaches the bucket. Fix, in two parts: - generate_version_sitemap.py runs in each versioned build and filters that build's sitemap down to its own subtree, writing the result *into* the directory that already deploys. No deploy command changes. - merge_sitemaps.py runs in the new deploy_complete_sitemap job, unions the 39 versioned sitemaps with the latest build's, and overwrites the published sitemap.xml at both mirrors deploy_latest writes. Taking URLs from Hugo's own output rather than deriving them from `url:` frontmatter is load-bearing. All 3,152 versioned .md files carry an explicit `url:`, but 18 are drafts; deriving would have published 18 URLs that 404. Copying whole elements also preserves the git-derived lastmod. Two guards, both because a silent partial sitemap would recreate this bug: - generate_version_sitemap.py exits non-zero when a subtree matches nothing, which is what a regression in the versioned `url:` scheme would look like. - merge_sitemaps.py refuses to write unless it sees one sitemap per discovered version plus the latest build's, so a failed matrix build leaves the published file alone instead of trimming it. Subtree matching is segment-anchored, not substring. operate/kubernetes/8.0 and operate/kubernetes/8.0.18 are both live version directories, so a substring match folds 78 pages of 8.0.18 into 8.0 and publishes them under the wrong version. test_sitemaps.py pins that case. One flat urlset rather than a sitemap index: the SEO team's file is itself a sitemap index and the protocol forbids nesting one inside another. Overwriting the address they already reference also means no change on their side. Note for whoever picks up the related SEO report: the 2,108 URLs it lists as "missing from the sitemap" are a different problem. Spot-checked, they are mostly anchor URLs and pre-restructure paths served by Hugo alias stubs, which return 200 with a meta-refresh -- which is why their liveness check passed. Redirects do not belong in a sitemap; this change does not address them, and the 200-instead-of-301 alias behaviour is worth its own ticket. Co-Authored-By: Claude Opus 5 (1M context) --- .github/workflows/main.yml | 174 ++++++++++++++++++++++++ build/generate_version_sitemap.py | 130 ++++++++++++++++++ build/merge_sitemaps.py | 118 ++++++++++++++++ build/test_sitemaps.py | 215 ++++++++++++++++++++++++++++++ 4 files changed, 637 insertions(+) create mode 100644 build/generate_version_sitemap.py create mode 100644 build/merge_sitemaps.py create mode 100644 build/test_sitemaps.py diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 7cbfef381d..07cdd44227 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -218,6 +218,17 @@ jobs: hugo -d "output" + # Hugo writes one sitemap at the root of the build output, but deploy_kubernetes + # uploads only the versioned subdirectory, so that file never reaches the + # bucket -- and the latest build deletes these version directories before it + # builds, so its sitemap cannot list them either. Result: no sitemap anywhere + # listed a versioned page. Write a subtree sitemap into the directory that + # does ship, so the existing rsync picks it up with no deploy changes. + python3 build/generate_version_sitemap.py \ + --sitemap output/sitemap.xml \ + --subtree "operate/kubernetes/${version}" \ + --output "output/operate/kubernetes/${version}/sitemap.xml" + - name: Upload artifact uses: actions/upload-artifact@v4 with: @@ -225,6 +236,15 @@ jobs: path: output/ retention-days: 1 + # Uploaded separately as well so deploy_version_sitemaps can collect every + # version's sitemap without downloading four dozen full site builds. + - name: Upload Kubernetes ${{ matrix.version }} sitemap + uses: actions/upload-artifact@v4 + with: + name: sitemap-kubernetes-${{ matrix.version }} + path: output/operate/kubernetes/${{ matrix.version }}/sitemap.xml + retention-days: 1 + # Build RS versions in parallel build_rs: name: Build RS ${{ matrix.version }} @@ -289,6 +309,13 @@ jobs: hugo -d "output" + # See the Kubernetes build: the root sitemap is never deployed, so write one + # into the versioned subdirectory that is. + python3 build/generate_version_sitemap.py \ + --sitemap output/sitemap.xml \ + --subtree "operate/rs/${version}" \ + --output "output/operate/rs/${version}/sitemap.xml" + - name: Upload artifact uses: actions/upload-artifact@v4 with: @@ -296,6 +323,13 @@ jobs: path: output/ retention-days: 1 + - name: Upload RS ${{ matrix.version }} sitemap + uses: actions/upload-artifact@v4 + with: + name: sitemap-rs-${{ matrix.version }} + path: output/operate/rs/${{ matrix.version }}/sitemap.xml + retention-days: 1 + # Build RDI versions in parallel build_rdi: name: Build RDI ${{ matrix.version }} @@ -360,6 +394,13 @@ jobs: hugo -d "output" + # See the Kubernetes build: the root sitemap is never deployed, so write one + # into the versioned subdirectory that is. + python3 build/generate_version_sitemap.py \ + --sitemap output/sitemap.xml \ + --subtree "integrate/redis-data-integration/${version}" \ + --output "output/integrate/redis-data-integration/${version}/sitemap.xml" + - name: Upload artifact uses: actions/upload-artifact@v4 with: @@ -367,6 +408,13 @@ jobs: path: output/ retention-days: 1 + - name: Upload RDI ${{ matrix.version }} sitemap + uses: actions/upload-artifact@v4 + with: + name: sitemap-rdi-${{ matrix.version }} + path: output/integrate/redis-data-integration/${{ matrix.version }}/sitemap.xml + retention-days: 1 + # Build RedisVL versions in parallel build_redisvl: name: Build RedisVL ${{ matrix.version }} @@ -431,6 +479,13 @@ jobs: hugo -d "output" + # See the Kubernetes build: the root sitemap is never deployed, so write one + # into the versioned subdirectory that is. + python3 build/generate_version_sitemap.py \ + --sitemap output/sitemap.xml \ + --subtree "develop/ai/redisvl/${version}" \ + --output "output/develop/ai/redisvl/${version}/sitemap.xml" + - name: Upload artifact uses: actions/upload-artifact@v4 with: @@ -438,6 +493,13 @@ jobs: path: output/ retention-days: 1 + - name: Upload RedisVL ${{ matrix.version }} sitemap + uses: actions/upload-artifact@v4 + with: + name: sitemap-redisvl-${{ matrix.version }} + path: output/develop/ai/redisvl/${{ matrix.version }}/sitemap.xml + retention-days: 1 + # Deploy latest build to GCS deploy_latest: name: Deploy latest @@ -790,6 +852,118 @@ jobs: fi # Deploy custom 404 page (only for the production latest build) + # Republish /sitemap.xml with the versioned pages folded in. + # + # deploy_latest publishes the sitemap Hugo rendered for the latest build, which by + # construction cannot list a versioned page: that build deletes the version + # directories before Hugo runs. The versioned builds do render those pages, but + # only their versioned subdirectory is deployed, so their sitemaps never ship. + # This job runs after both and overwrites the published file with the union -- + # ~2,700 latest URLs plus ~3,100 versioned ones -- so the address the SEO team + # already references gains the missing pages with no change on their side. + # + # One flat urlset rather than a sitemap index, deliberately: their file is itself + # a sitemap index, and the protocol does not allow one index to nest inside + # another. At ~5,800 URLs this is well inside the 50,000 URL / 50 MB ceiling. + deploy_complete_sitemap: + name: Deploy complete sitemap + needs: + - discover_versions + - build_kubernetes + - build_rs + - build_rdi + - build_redisvl + # deploy_latest mirrors public -> docs/ with `-d`, which would delete + # this file. Same gating reason as the versioned deploy jobs. + - deploy_latest + # A build job is skipped when its product has no version directories (RDI has + # none today), and a skipped `needs` would skip this job too -- hence + # `!cancelled()` rather than a plain dependency. Only deploy_latest is + # load-bearing; the rest contribute artifacts when they run. The last clause + # stands the job down entirely when nothing is versioned, so that an empty + # artifact set is a no-op instead of a failed merge. + if: >- + ${{ !cancelled() + && needs.deploy_latest.result == 'success' + && !(needs.discover_versions.outputs.kubernetes_versions == '[]' + && needs.discover_versions.outputs.rs_versions == '[]' + && needs.discover_versions.outputs.rdi_versions == '[]' + && needs.discover_versions.outputs.redisvl_versions == '[]') }} + runs-on: ubuntu-latest + permissions: + contents: 'read' + id-token: 'write' + env: + PROD_PROJECT_ID: ${{ secrets.GCP_PROJECT_PROD }} + PROD_SERVICE_ACCOUNT: ${{ secrets.PROD_SERVICE_ACCOUNT }} + PROD_WORKLOAD_IDENTITY_PROVIDER: ${{ secrets.PROD_WORKLOAD_IDENTITY_PROVIDER }} + + steps: + - name: Check the branch out + uses: actions/checkout@v4 + + - name: Download the versioned sitemaps + uses: actions/download-artifact@v4 + with: + pattern: sitemap-* + path: sitemaps/ + + # The latest build's own sitemap is the other half of the union. Taken from its + # artifact rather than from the bucket so the merge reflects the build being + # deployed rather than whatever a concurrent run happened to leave published. + - name: Download the latest build's sitemap + uses: actions/download-artifact@v4 + with: + name: build-latest + path: latest-build/ + + - name: Merge every sitemap into one + run: | + mkdir -p sitemaps/latest + cp latest-build/sitemap.xml sitemaps/latest/sitemap.xml + + # One sitemap per discovered version, plus the latest build's. Fewer means a + # matrix build failed or dropped its artifact, and overwriting the published + # file with a partial one would silently drop those pages from search again -- + # the exact regression this job exists to prevent. + expected=$( ( + echo '${{ needs.discover_versions.outputs.kubernetes_versions }}' + echo '${{ needs.discover_versions.outputs.rs_versions }}' + echo '${{ needs.discover_versions.outputs.rdi_versions }}' + echo '${{ needs.discover_versions.outputs.redisvl_versions }}' + ) | jq -s 'map(length) | add') + expected=$((expected + 1)) + echo "Expecting ${expected} sitemaps (versions + latest)" + + python3 build/merge_sitemaps.py sitemaps \ + --output sitemap.xml \ + --expect "$expected" + + - name: 'Google auth' + uses: 'google-github-actions/auth@v2' + with: + project_id: '${{ env.PROD_PROJECT_ID }}' + service_account: '${{ env.PROD_SERVICE_ACCOUNT }}' + workload_identity_provider: '${{ env.PROD_WORKLOAD_IDENTITY_PROVIDER }}' + + - name: 'Set up Cloud SDK' + uses: 'google-github-actions/setup-gcloud@v2' + with: + project_id: '${{ env.PROD_PROJECT_ID }}' + version: '>= 363.0.0' + + - name: Deploy the complete sitemap to GCS + run: | + bucket_path="${{ needs.discover_versions.outputs.bucket_path }}" + + # Mirrors the versioned deploy jobs, which act on latest and staging only. + if [[ "$bucket_path" == "latest" || "$bucket_path" == staging/* ]]; then + # Both destinations deploy_latest mirrors to, so the two copies of + # sitemap.xml do not disagree about which pages exist. + gsutil cp sitemap.xml "gs://${BUCKET}/${bucket_path}/sitemap.xml" + gsutil cp sitemap.xml "gs://${BUCKET}/docs/${bucket_path}/sitemap.xml" + fi + deploy_404: name: Deploy 404 page needs: diff --git a/build/generate_version_sitemap.py b/build/generate_version_sitemap.py new file mode 100644 index 0000000000..86e1f005ec --- /dev/null +++ b/build/generate_version_sitemap.py @@ -0,0 +1,130 @@ +#!/usr/bin/env python3 +"""Write a sitemap for one versioned subtree, inside the directory that ships. + +Every versioned matrix build in ``.github/workflows/main.yml`` renders a *whole* +site: the version's content rsynced up to the unversioned content path, plus the +rest of the docs. Hugo therefore writes one ``sitemap.xml`` at the root of the +build output, listing all ~5,800 pages. + +But the deploy step for a versioned build uploads only the versioned +subdirectory:: + + gsutil -m rsync -r -c -j html -d \\ + "output/operate/rs/${version}" \\ + "gs://${BUCKET}/docs/${bucket_path}/operate/rs/${version}" + +``output/sitemap.xml`` sits one or more levels *above* that directory, so it is +never uploaded by any job. The ``latest`` build deletes the version directories +before building, so its sitemap cannot list those pages either. The result is +that no sitemap anywhere lists a versioned page -- 3,134 live URLs as of this +writing, measured against a local unrestricted build. + +This script closes that gap without touching the deploy commands: it filters the +rendered sitemap down to the pages under one versioned subtree and writes the +result *into* the directory that already ships, so the existing rsync picks it up. +``merge_sitemaps.py`` then folds every version's file, plus the latest build's, into +the single published ``sitemap.xml``. + +The locs need no rewriting. Every page under a version directory carries explicit +``url:`` frontmatter pinning its versioned path (all 435 files under +``content/operate/rs/7.8`` do), so Hugo already computes the correct public +permalink -- ``https://redis.io/docs/latest/operate/rs/7.8/...`` -- even though +the content was rsynced to the unversioned path before the build. We copy whole +```` elements, so ``lastmod`` comes through as Hugo computed it from git. + +Exits non-zero when the subtree matches nothing. That is the regression guard: if +a future change stops those pages being emitted at their versioned URLs, this +fails the build rather than silently shipping an empty sitemap. + +Run with ``pytest build/test_sitemaps.py`` for the tests. +""" + +import argparse +import logging +import os +import sys +import xml.etree.ElementTree as ET +from urllib.parse import urlparse + +SITEMAP_NS = "http://www.sitemaps.org/schemas/sitemap/0.9" +XHTML_NS = "http://www.w3.org/1999/xhtml" + +logger = logging.getLogger("generate_version_sitemap") + + +def parse_args() -> argparse.Namespace: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--sitemap", default="output/sitemap.xml", + help="sitemap Hugo rendered at the build root") + parser.add_argument("--subtree", required=True, + help="versioned path to keep, e.g. operate/rs/7.8") + parser.add_argument("--output", required=True, + help="where to write the filtered sitemap") + parser.add_argument("--allow-empty", action="store_true", + help="warn instead of failing when nothing matches") + return parser.parse_args() + + +def in_subtree(loc: str, subtree: str) -> bool: + """Is ``loc`` the subtree root or a page beneath it? + + Compared on the URL *path* so that a baseURL which itself contains the + subtree string cannot widen the match, and segment-anchored so that + ``operate/rs/7.8`` does not swallow a future ``operate/rs/7.8-rc1``. + """ + path = urlparse(loc).path + marker = "/" + subtree.strip("/") + return path.rstrip("/").endswith(marker) or (marker + "/") in path + + +def filter_sitemap(xml_text: str, subtree: str) -> tuple[str, int]: + """Return a sitemap holding only the ```` entries under ``subtree``.""" + ET.register_namespace("", SITEMAP_NS) + ET.register_namespace("xhtml", XHTML_NS) + + source = ET.fromstring(xml_text) + kept = ET.Element(f"{{{SITEMAP_NS}}}urlset") + + for url in source.findall(f"{{{SITEMAP_NS}}}url"): + loc = url.find(f"{{{SITEMAP_NS}}}loc") + if loc is not None and loc.text and in_subtree(loc.text, subtree): + kept.append(url) + + ET.indent(kept, space=" ") + body = ET.tostring(kept, encoding="unicode") + header = '\n' + return header + body + "\n", len(kept) + + +def main() -> int: + logging.basicConfig(level=logging.INFO, format="%(name)s: %(message)s") + args = parse_args() + + if not os.path.isfile(args.sitemap): + logger.error("no sitemap at %s -- did hugo run?", args.sitemap) + return 1 + + with open(args.sitemap, encoding="utf-8") as handle: + xml_text = handle.read() + + document, count = filter_sitemap(xml_text, args.subtree) + + if not count: + message = "%s matched no URLs in %s" + if not args.allow_empty: + logger.error(message, args.subtree, args.sitemap) + logger.error("versioned pages are not being emitted at their " + "versioned URLs -- check the url: frontmatter") + return 1 + logger.warning(message, args.subtree, args.sitemap) + + os.makedirs(os.path.dirname(args.output) or ".", exist_ok=True) + with open(args.output, "w", encoding="utf-8") as handle: + handle.write(document) + + logger.info("wrote %d URLs for %s to %s", count, args.subtree, args.output) + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/build/merge_sitemaps.py b/build/merge_sitemaps.py new file mode 100644 index 0000000000..e934efc742 --- /dev/null +++ b/build/merge_sitemaps.py @@ -0,0 +1,118 @@ +#!/usr/bin/env python3 +"""Merge the latest and per-version sitemaps into the one published sitemap.xml. + +No single build can produce a complete sitemap. The ``latest`` build deletes the +version directories before Hugo runs, so its sitemap lists ~2,700 URLs and no +versioned page. Each versioned build does render its own pages, but only its +versioned subdirectory is deployed, so the sitemap Hugo writes at the root of that +build never ships. The union is ~5,800 URLs and lives in neither place. + +``generate_version_sitemap.py`` runs inside each versioned build and emits that +version's subtree sitemap. This script runs after all of them, in a job that +collects their (tiny) artifacts plus the latest build's sitemap, and concatenates +everything into one flat ```` that overwrites the published file. + +Flat urlset rather than a sitemap index, deliberately: the SEO team's own file is +already a sitemap index, and the protocol does not allow one index to nest inside +another. Overwriting the address they already reference also means they need change +nothing. At ~5,800 URLs this is well inside the 50,000 URL / 50 MB ceiling. + +Deduplicates on loc. Nothing should collide -- the latest build and each version +cover disjoint paths -- so a duplicate means two builds claimed the same URL, which +is worth knowing about and is logged. +""" + +import argparse +import logging +import os +import sys +import xml.etree.ElementTree as ET + +SITEMAP_NS = "http://www.sitemaps.org/schemas/sitemap/0.9" +XHTML_NS = "http://www.w3.org/1999/xhtml" + +logger = logging.getLogger("merge_sitemaps") + + +def parse_args() -> argparse.Namespace: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("input_dir", + help="directory holding the downloaded sitemap artifacts") + parser.add_argument("--output", required=True, + help="where to write the merged sitemap") + parser.add_argument("--expect", type=int, default=None, + help="number of sitemaps that must be present (versions plus " + "the latest build); fail if fewer") + return parser.parse_args() + + +def find_sitemaps(input_dir: str) -> list[str]: + """Every ``sitemap.xml`` under ``input_dir``, in a stable order.""" + found = [] + for root, _dirs, files in os.walk(input_dir): + for name in files: + if name == "sitemap.xml": + found.append(os.path.join(root, name)) + return sorted(found) + + +def merge(paths: list[str]) -> tuple[str, int]: + """Concatenate the ```` entries of every sitemap in ``paths``.""" + ET.register_namespace("", SITEMAP_NS) + ET.register_namespace("xhtml", XHTML_NS) + + merged = ET.Element(f"{{{SITEMAP_NS}}}urlset") + seen: set[str] = set() + + for path in paths: + source = ET.parse(path).getroot() + added = 0 + for url in source.findall(f"{{{SITEMAP_NS}}}url"): + loc = url.find(f"{{{SITEMAP_NS}}}loc") + if loc is None or not loc.text: + continue + if loc.text in seen: + logger.warning("duplicate loc %s (from %s)", loc.text, path) + continue + seen.add(loc.text) + merged.append(url) + added += 1 + logger.info("%s contributed %d URLs", path, added) + + ET.indent(merged, space=" ") + body = ET.tostring(merged, encoding="unicode") + header = '\n' + return header + body + "\n", len(merged) + + +def main() -> int: + logging.basicConfig(level=logging.INFO, format="%(name)s: %(message)s") + args = parse_args() + + paths = find_sitemaps(args.input_dir) + if not paths: + logger.error("no sitemap.xml found under %s", args.input_dir) + return 1 + + # Refuse to overwrite the published sitemap with a partial one. A missing + # artifact means a build failed, and quietly dropping its pages out of the file + # is the exact failure this whole change exists to fix. + if args.expect is not None and len(paths) < args.expect: + logger.error("found %d sitemaps but expected %d", len(paths), args.expect) + logger.error("a build's sitemap is missing -- keeping the published " + "sitemap rather than shipping an incomplete one") + return 1 + + document, count = merge(paths) + + os.makedirs(os.path.dirname(args.output) or ".", exist_ok=True) + with open(args.output, "w", encoding="utf-8") as handle: + handle.write(document) + + logger.info("wrote %d URLs from %d sitemaps to %s", + count, len(paths), args.output) + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/build/test_sitemaps.py b/build/test_sitemaps.py new file mode 100644 index 0000000000..839f5a67ac --- /dev/null +++ b/build/test_sitemaps.py @@ -0,0 +1,215 @@ +#!/usr/bin/env python3 +"""Tests for generate_version_sitemap and merge_sitemaps. + +The load-bearing logic is ``in_subtree``. It decides which of the ~5,800 URLs in a +versioned build's sitemap belong to that build's version, and it has to be +segment-anchored: ``operate/kubernetes/8.0`` and ``operate/kubernetes/8.0.18`` are +both live version directories today, so a substring match would fold 78 pages of +8.0.18 into 8.0's sitemap and publish them under the wrong version. + +Verified end to end against a real 5,816-URL build: the 39 version subtrees filter +to 3,134 URLs with no duplicates, which is exactly the count of versioned locs in +that sitemap and exactly the shortfall between it and the 2,678 URLs published at +redis.io/docs/latest/sitemap.xml. + +Run with ``pytest build/test_sitemaps.py`` or directly. +""" + +import os +import sys +import tempfile + +sys.path.insert(0, os.path.dirname(__file__)) + +from generate_version_sitemap import filter_sitemap, in_subtree # noqa: E402 +from merge_sitemaps import find_sitemaps, merge # noqa: E402 + + +def sitemap(*entries: str) -> str: + urls = "".join( + f"{loc}{mod}" + for loc, mod in (e.split(" ") for e in entries) + ) + return ( + '' + '' + f"{urls}" + ) + + +def locs(xml_text: str) -> list[str]: + import xml.etree.ElementTree as ET + ns = "{http://www.sitemaps.org/schemas/sitemap/0.9}" + root = ET.fromstring(xml_text) + return [u.findtext(f"{ns}loc") for u in root.findall(f"{ns}url")] + + +# --------------------------------------------------------------------------- # +# in_subtree +# --------------------------------------------------------------------------- # + +def test_matches_pages_beneath_the_subtree(): + loc = "https://redis.io/docs/latest/operate/rs/7.8/clusters/maintenance-mode/" + assert in_subtree(loc, "operate/rs/7.8") + + +def test_matches_the_subtree_root_itself(): + # content/operate/rs/7.8/_index.md carries url: '/operate/rs/7.8/', so the + # version landing page is in the sitemap and must not be dropped. + assert in_subtree("https://redis.io/docs/latest/operate/rs/7.8/", "operate/rs/7.8") + assert in_subtree("https://redis.io/docs/latest/operate/rs/7.8", "operate/rs/7.8") + + +def test_longer_version_is_not_swallowed_by_a_shorter_prefix(): + # The real trap: operate/kubernetes/8.0 and .../8.0.18 both ship today. + loc = "https://redis.io/docs/latest/operate/kubernetes/8.0.18/quickstart/" + assert in_subtree(loc, "operate/kubernetes/8.0.18") + assert not in_subtree(loc, "operate/kubernetes/8.0") + + +def test_unversioned_sibling_is_excluded(): + # Every versioned build also renders the unversioned tree; those URLs belong + # to the latest build's sitemap, not this one's. + loc = "https://redis.io/docs/latest/operate/rs/clusters/maintenance-mode/" + assert not in_subtree(loc, "operate/rs/7.8") + + +def test_match_is_on_the_path_not_the_whole_url(): + # A host or query string echoing the subtree must not widen the match. + assert not in_subtree("https://operate/rs/7.8/example.com/page/", "operate/rs/7.8") + assert not in_subtree("https://redis.io/x/?p=/operate/rs/7.8/", "operate/rs/7.8") + + +def test_subtree_slashes_are_tolerated(): + loc = "https://redis.io/docs/latest/operate/rs/7.8/clusters/" + assert in_subtree(loc, "/operate/rs/7.8/") + + +# --------------------------------------------------------------------------- # +# filter_sitemap +# --------------------------------------------------------------------------- # + +def test_filter_keeps_only_the_subtree_and_reports_the_count(): + document, count = filter_sitemap(sitemap( + "https://redis.io/docs/latest/operate/rs/7.8/ 2026-01-01T00:00:00Z", + "https://redis.io/docs/latest/operate/rs/7.8/clusters/ 2026-01-02T00:00:00Z", + "https://redis.io/docs/latest/operate/rs/7.4/clusters/ 2026-01-03T00:00:00Z", + "https://redis.io/docs/latest/develop/ 2026-01-04T00:00:00Z", + ), "operate/rs/7.8") + + assert count == 2 + assert locs(document) == [ + "https://redis.io/docs/latest/operate/rs/7.8/", + "https://redis.io/docs/latest/operate/rs/7.8/clusters/", + ] + + +def test_filter_preserves_lastmod(): + # lastmod comes from git via Hugo's enableGitInfo. Copying whole + # elements keeps it, rather than re-deriving dates the build cannot see. + document, _ = filter_sitemap(sitemap( + "https://redis.io/docs/latest/operate/rs/7.8/ 2026-01-01T00:00:00Z", + ), "operate/rs/7.8") + assert "2026-01-01T00:00:00Z" in document + + +def test_filter_output_declares_the_sitemap_namespace(): + document, _ = filter_sitemap(sitemap( + "https://redis.io/docs/latest/operate/rs/7.8/ 2026-01-01T00:00:00Z", + ), "operate/rs/7.8") + assert 'xmlns="http://www.sitemaps.org/schemas/sitemap/0.9"' in document + assert document.startswith(' str: + """Drop ``text`` at ``root/name/filename``, mimicking a downloaded artifact.""" + directory = os.path.join(root, name) + os.makedirs(directory, exist_ok=True) + path = os.path.join(directory, filename) + with open(path, "w", encoding="utf-8") as handle: + handle.write(text) + return path + + +def test_merge_concatenates_every_subtree(): + with tempfile.TemporaryDirectory() as tmp: + write(tmp, "rs-7.8", "sitemap.xml", sitemap( + "https://redis.io/docs/latest/operate/rs/7.8/ 2026-01-01T00:00:00Z")) + write(tmp, "rs-7.4", "sitemap.xml", sitemap( + "https://redis.io/docs/latest/operate/rs/7.4/ 2026-01-02T00:00:00Z")) + + document, count = merge(find_sitemaps(tmp)) + + assert count == 2 + assert sorted(locs(document)) == [ + "https://redis.io/docs/latest/operate/rs/7.4/", + "https://redis.io/docs/latest/operate/rs/7.8/", + ] + + +def test_merge_deduplicates_on_loc(): + # Should never happen -- the inputs cover disjoint subtrees -- so a collision + # means two builds claimed one URL. Drop the repeat, keep the file valid. + duplicate = "https://redis.io/docs/latest/operate/rs/7.8/ 2026-01-01T00:00:00Z" + with tempfile.TemporaryDirectory() as tmp: + write(tmp, "a", "sitemap.xml", sitemap(duplicate)) + write(tmp, "b", "sitemap.xml", sitemap(duplicate)) + + _, count = merge(find_sitemaps(tmp)) + + assert count == 1 + + +def test_find_sitemaps_is_ordered_and_recursive(): + with tempfile.TemporaryDirectory() as tmp: + write(tmp, "b-version", "sitemap.xml", sitemap( + "https://redis.io/docs/latest/operate/rs/7.4/ 2026-01-01T00:00:00Z")) + write(tmp, "a-version", "sitemap.xml", sitemap( + "https://redis.io/docs/latest/operate/rs/7.8/ 2026-01-01T00:00:00Z")) + + found = find_sitemaps(tmp) + + assert len(found) == 2 + assert found == sorted(found) + + +def test_find_sitemaps_ignores_other_files(): + # download-artifact unpacks each version's artifact into its own directory, + # so the only thing distinguishing our file is its name. + with tempfile.TemporaryDirectory() as tmp: + write(tmp, "rs-7.8", "sitemap.xml", sitemap( + "https://redis.io/docs/latest/operate/rs/7.8/ 2026-01-01T00:00:00Z")) + write(tmp, "rs-7.8", "index.html", "") + + found = find_sitemaps(tmp) + + assert [os.path.basename(p) for p in found] == ["sitemap.xml"] + + +if __name__ == "__main__": + failures = 0 + for name, fn in sorted(list(globals().items())): + if name.startswith("test_") and callable(fn): + try: + fn() + print(f" ok {name}") + except AssertionError as exc: + failures += 1 + print(f" FAIL {name}: {exc}") + print(f"\n{failures} failure(s)") + sys.exit(1 if failures else 0)