From 9614bbdc83e8f9ca9cf855aefc1b23bcfeff463d Mon Sep 17 00:00:00 2001 From: Codey Schoettle Date: Sun, 16 Aug 2026 20:39:19 -0400 Subject: [PATCH] Add CDN logic, mTLS ability for CDNs, and Branding for webpage --- .gitignore | 3 + CHANGELOG.md | 9 + Makefile | 5 +- README.md | 127 +++++++++- cmd/config_test.go | 32 +-- cmd/serve.go | 24 +- cmd/serve_test.go | 51 ---- configs/pkgproxy.yaml | 13 + docs/architecture.md | 18 +- openspec/specs/http-landing-page/spec.md | 55 ++-- pkg/pkgproxy/landing.go | 104 +++++++- pkg/pkgproxy/landing_test.go | 169 ++++++++++--- pkg/pkgproxy/mtls_test.go | 305 +++++++++++++++++++++++ pkg/pkgproxy/proxy.go | 161 ++++++++++-- pkg/pkgproxy/proxy_test.go | 50 +++- pkg/pkgproxy/repository.go | 84 ++++++- pkg/pkgproxy/repository_test.go | 164 ++++++++++++ test/e2e/e2e_test.go | 74 +++++- 18 files changed, 1234 insertions(+), 214 deletions(-) create mode 100644 pkg/pkgproxy/mtls_test.go diff --git a/.gitignore b/.gitignore index 1081f0a..2195632 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,6 @@ +# local mTLS client certificates (e.g. Red Hat entitlements) +*.pem + # build and testing artifacts /bin/ /cache/ diff --git a/CHANGELOG.md b/CHANGELOG.md index 45fde4f..ad5a577 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,13 +8,22 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/). ### Added +- Per-repository `cdn` config field to proxy vendor CDNs that publish no public mirrors +- Per-repository `mtls` config field with client `cert`, `key` and optional `ca` for CDNs using mutual TLS +- Support for proxying entitled Red Hat content from `cdn.redhat.com` without client-side certificates +- Client config snippet for Red Hat Enterprise Linux on the landing page and in the README +- Top-level `branding` config field to customize the landing page title and description +- Landing page now shows the running pkgproxy version - Container image now runs `serve` by default and loads bundled config from `$KO_DATA_PATH` - `PKGPROXY_TRUST_PROXY` env var (and `--trust-proxy` flag) to opt in to X-Forwarded-For trust - `PKGPROXY_HOST` env var to set the listen address without passing `--host` on the command line ### Changed +- Repositories must now define exactly one of `mirrors` or `cdn`; setting both is rejected +- Upstream URLs are validated at startup: they must be absolute and use `http` or `https` - **Breaking:** `remote_ip` in access logs now reflects the direct connecting peer by default; set `PKGPROXY_TRUST_PROXY` to restore XFF-based IP extraction when running behind a reverse proxy +- **Breaking:** Removed the `--public-host` flag and `PKGPROXY_PUBLIC_HOST` env var; the landing page now fills in config snippet hostnames automatically — server-side from the request's `Host` header (works for `curl` too), further corrected client-side to the browser's own URL when that differs (e.g. behind a TLS-terminating reverse proxy) - Upgraded Echo web framework to v5.1.1 - Config-file errors now list all default paths attempted, not just the last one diff --git a/Makefile b/Makefile index cd6f0a7..9f230f9 100644 --- a/Makefile +++ b/Makefile @@ -113,7 +113,8 @@ $(if $(filter debian,$(1)),TestDebian,\ $(if $(filter ubuntu,$(1)),TestUbuntu,\ $(if $(filter archlinux,$(1)),TestArch,\ $(if $(filter gentoo,$(1)),TestGentoo,\ -$(error Unknown DISTRO: $(1). Use one of: fedora centos-stream almalinux rockylinux debian ubuntu archlinux gentoo)))))))))) +$(if $(filter rhel,$(1)),TestRHEL,\ +$(error Unknown DISTRO: $(1). Use one of: fedora centos-stream almalinux rockylinux debian ubuntu archlinux gentoo rhel))))))))))) endef .PHONY: e2e @@ -157,7 +158,7 @@ run: format vet generate ## Run the application from your host $(info *************************************************) $(info ********** EXECUTING 'run' MAKE TARGET **********) $(info *************************************************) - PKGPROXY_CONFIG=./configs/pkgproxy.yaml PKGPROXY_PUBLIC_HOST=$(shell hostname):8080 CGO_ENABLED=$(CGO_ENABLED) go run . serve --host 0.0.0.0 --debug + PKGPROXY_CONFIG=./configs/pkgproxy.yaml CGO_ENABLED=$(CGO_ENABLED) go run . serve --host 0.0.0.0 --debug PLATFORMS := $(shell echo $(ARCHS) | sed 's/,/ /g' | sed 's/[^ ]\+/linux\/&/g' | tr ' ' ',') diff --git a/README.md b/README.md index c0be5b3..17ad307 100644 --- a/README.md +++ b/README.md @@ -33,12 +33,32 @@ podman run --rm -p 8080:8080 -e PKGPROXY_HOST=0.0.0.0 --volume ./cache:/ko-app/c | `--cachedir` | | `cache` | Path to the local cache directory | | `--host` | `PKGPROXY_HOST` | `localhost` | Listen address | | `--port` | | `8080` | Listen port | -| `--public-host` | `PKGPROXY_PUBLIC_HOST` | | Public hostname (or `host:port`) shown in landing page config snippets. When set, the listen port is not appended. Useful when running behind a reverse proxy. | | `--trust-proxy` | `PKGPROXY_TRUST_PROXY` | | Comma-separated list of trusted proxy sources for X-Forwarded-For. Accepted values: `none`, `loopback`, `private`, a CIDR (e.g. `10.0.0.0/8`), or a bare IP (promoted to `/32`/`/128`). Unset or empty means no XFF trust. | | `--debug` | | `false` | Enable debug logging | Any flag with an env variable listed above can be set via the environment instead of passing the flag. +### Landing page hostname + +The config snippets shown on the landing page (`GET /`) need pkgproxy's own +address, e.g. `baseurl=http:///fedora/...`. Rather than relying on a +server-side setting, this is filled in automatically, with no configuration +needed: + +- **Server-side, from the request's `Host` header.** Every response — including + `curl` and other non-browser clients — already contains a working address + built from the `Host` header the request itself carried (the same header a + reverse proxy forwards by default). No JavaScript required. +- **Client-side, from the page's own URL.** In a browser, a small inline script + additionally corrects the address to `window.location.origin` if it differs + from the server-rendered one — which matters behind a reverse proxy that + changes the scheme (e.g. TLS termination), since the `Host` header alone + can't reveal that. + +If a reverse proxy in front of pkgproxy does not forward the original `Host` +header, `curl` (or a browser with JavaScript disabled) will see whatever host +pkgproxy itself observed instead. + ### Trusting X-Forwarded-For By default pkgproxy ignores the `X-Forwarded-For` header and uses the direct connecting IP address for the `remote_ip` access-log field. This is the safe behavior when pkgproxy faces the internet directly or runs in a container without a reverse proxy in front of it. @@ -66,8 +86,94 @@ Each repository supports the following options: |-----|----------|-------------| | `suffixes` | yes | File suffixes that are eligible for caching (e.g. `.rpm`, `.deb`). Use `"*"` to cache all files. | | `exclude` | no | List of file names to exclude from caching, even when they match a suffix. Useful with the `"*"` wildcard suffix. | -| `mirrors` | yes | Ordered list of upstream mirror URLs | -| `retries` | no | Number of attempts per mirror before moving to the next one (default: `1`) | +| `mirrors` | yes* | Ordered list of upstream mirror URLs | +| `cdn` | yes* | Single upstream CDN URL, used instead of `mirrors` | +| `mtls` | no | Client certificate (`cert`), private key (`key`) and optional CA bundle (`ca`) used with a `cdn` requiring mutual TLS | +| `retries` | no | Number of attempts per upstream before moving to the next one (default: `1`) | + +\* Each repository must define exactly one of `mirrors` or `cdn`; setting both is rejected. + +### Landing page branding + +The top-level `branding` key customizes the title and description shown on the +landing page (and the HTML ``) served at `/`: + +```yaml +branding: + title: Acme Package Mirror + description: Internal package cache for Acme Corp. + +repositories: + ... +``` + +Both fields are optional and independent — omitting `branding` entirely, or +leaving one of the two fields unset, falls back to the default "pkgproxy" title +and "Caching forward proxy for Linux package repositories." description. The +landing page also always shows the running pkgproxy version below the +description. + +### CDN upstreams + +Some vendors do not publish public mirrors and serve their packages from a single +CDN instead. Use `cdn` in place of `mirrors` for those repositories: + +```yaml +repositories: + rhel: + suffixes: + - .rpm + cdn: https://cdn.redhat.com/ +``` + +Requests are mapped the same way as for mirrors: the repository name is stripped +from the request path and the remainder is appended to the CDN URL, so +`/rhel/content/dist/rhel9/9/x86_64/baseos/os/` is fetched from +`https://cdn.redhat.com/content/dist/rhel9/9/x86_64/baseos/os/`. + +### CDN client certificates (mTLS) + +When the CDN requires mutual TLS — as the Red Hat CDN does for entitled content — +add an `mtls` block with the client certificate and its private key. pkgproxy +presents them during the TLS handshake with the CDN: + +```yaml +repositories: + rhel: + suffixes: + - .drpm + - .rpm + cdn: https://cdn.redhat.com/ + mtls: + cert: /etc/pki/entitlement/1234567890123456789.pem + key: /etc/pki/entitlement/1234567890123456789-key.pem + ca: /etc/rhsm/ca/redhat-uep.pem +``` + +On a subscribed Red Hat host the entitlement certificate and its key are the +`.pem` file pair in `/etc/pki/entitlement/`. + +The optional `ca` points at a CA bundle used to verify the CDN's *own* server +certificate, and is added to the system trust store rather than replacing it. +It is required for `cdn.redhat.com`, whose certificate is issued by a private +Red Hat CA that public trust stores do not contain — without it every request +fails with `x509: certificate signed by unknown authority`. + +Notes: + +- `mtls` is only valid together with `cdn`, and both `cert` and `key` are required. +- Relative paths are resolved against the working directory of the pkgproxy + process. Prefer absolute paths, especially for container deployments. +- pkgproxy **refuses to start** if the certificate, key or CA bundle cannot be + loaded. Proxying without them would only produce opaque TLS or authorization + errors from the CDN. +- The certificate is scoped to the configured CDN host. If the CDN redirects to a + different host, the redirect is followed **without** the client certificate so + the credential is never sent elsewhere. +- Clients talking to pkgproxy need no entitlement certificate of their own — this + is the point of proxying an entitled CDN for a local network. Protect access to + pkgproxy accordingly, since it will serve entitled content to anyone who can + reach it. ### Mirror retries @@ -214,6 +320,21 @@ For Enterprise distributions the URL suffix `epel-$releasever-$basearch` must be baseurl=http://<pkgproxy>:8080/rockylinux/$releasever/BaseOS/$basearch/os/ ``` +### Red Hat Enterprise Linux + +Requires a `rhel` repository configured with `cdn` and `mtls` (see [CDN client +certificates](#cdn-client-certificates-mtls)). Disable the subscription-manager +managed repositories, then e.g. `/etc/yum.repos.d/rhel.repo` (adjust other +repositories accordingly): +``` +[rhel-baseos-rpms] +# baseurl=https://cdn.redhat.com/content/dist/rhel$releasever/$releasever/$basearch/baseos/os +baseurl=http://<pkgproxy>:8080/rhel/content/dist/rhel$releasever/$releasever/$basearch/baseos/os +``` + +The client needs no `sslclientcert`/`sslclientkey` of its own — pkgproxy holds the +entitlement certificate and authenticates against the CDN on the client's behalf. + ### Ubuntu E.g. Ubuntu 24.04 Noble Numbat: `/etc/apt/sources.list` (substitute your release codename): diff --git a/cmd/config_test.go b/cmd/config_test.go index 796514d..c1844aa 100644 --- a/cmd/config_test.go +++ b/cmd/config_test.go @@ -24,20 +24,20 @@ func writeConfig(t *testing.T, dir, name string) string { func TestResolveConfigPath(t *testing.T) { tests := []struct { - name string - localExists bool - localIsDir bool - koDataSet bool - koFileExists bool - wantPath func(koDir string) string - wantCandidates func(koDir string) []string + name string + localExists bool + localIsDir bool + koDataSet bool + koFileExists bool + wantPath func(koDir string) string + wantCandidates func(koDir string) []string }{ { - name: "local file wins over ko fallback", - localExists: true, - koDataSet: true, - koFileExists: true, - wantPath: func(_ string) string { return defaultConfigPath }, + name: "local file wins over ko fallback", + localExists: true, + koDataSet: true, + koFileExists: true, + wantPath: func(_ string) string { return defaultConfigPath }, wantCandidates: func(_ string) []string { return []string{defaultConfigPath} }, }, { @@ -51,10 +51,10 @@ func TestResolveConfigPath(t *testing.T) { }, }, { - name: "both missing returns default path", - localExists: false, - koDataSet: false, - wantPath: func(_ string) string { return defaultConfigPath }, + name: "both missing returns default path", + localExists: false, + koDataSet: false, + wantPath: func(_ string) string { return defaultConfigPath }, wantCandidates: func(_ string) []string { return []string{defaultConfigPath} }, }, { diff --git a/cmd/serve.go b/cmd/serve.go index dc3c0d7..3bfacd7 100644 --- a/cmd/serve.go +++ b/cmd/serve.go @@ -25,7 +25,6 @@ import ( var ( listenAddress string listenPort uint16 - publicHost string trustProxy string ipExtractor echo.IPExtractor resolvedTrustProxy string @@ -35,7 +34,6 @@ const ( defaultAddress = "localhost" defaultPort = 8080 hostEnvVar = "PKGPROXY_HOST" - publicHostEnvVar = "PKGPROXY_PUBLIC_HOST" trustProxyEnvVar = "PKGPROXY_TRUST_PROXY" ) @@ -59,7 +57,6 @@ func newServeCommand() *cobra.Command { } c.PersistentFlags().StringVar(&listenAddress, "host", defaultAddress, "listen address of the pkgproxy.") c.PersistentFlags().Uint16Var(&listenPort, "port", defaultPort, "listen port of the pkgproxy.") - c.PersistentFlags().StringVar(&publicHost, "public-host", "", "public hostname (or host:port) shown in landing page config snippets; overrides PKGPROXY_PUBLIC_HOST.") c.PersistentFlags().StringVar(&trustProxy, "trust-proxy", "", "comma-separated list of trusted proxy addresses for X-Forwarded-For: none, loopback, private, CIDR, or IP; overrides PKGPROXY_TRUST_PROXY.") return c @@ -76,19 +73,6 @@ func resolveListenHost(flagChanged bool, flagValue, envValue string) string { return defaultAddress } -// resolvePublicAddr determines the address rendered in landing page config snippets. -// The CLI flag takes precedence over the environment variable. If neither is set, -// the listen host:port is used. -func resolvePublicAddr(flagValue string, listenAddr string, port uint16) string { - if flagValue != "" { - return flagValue - } - if v := os.Getenv(publicHostEnvVar); v != "" { - return v - } - return fmt.Sprintf("%s:%d", listenAddr, port) -} - // resolveTrustProxy determines the trust-proxy value using flag → env var → default precedence. func resolveTrustProxy(flagChanged bool, flagValue, envValue string) string { if flagChanged { @@ -221,12 +205,14 @@ func startServer(_ *cobra.Command, _ []string) error { }) app.Use(middleware.Recover()) - pkgProxy := pkgproxy.New(&pkgproxy.PkgProxyConfig{ + pkgProxy, err := pkgproxy.New(&pkgproxy.PkgProxyConfig{ CacheBasePath: cacheDir, RepositoryConfig: &repoConfig, }) - publicAddr := resolvePublicAddr(publicHost, listenAddress, listenPort) - app.GET("/", pkgproxy.LandingHandler(&repoConfig, publicAddr)) + if err != nil { + return fmt.Errorf("unable to initialize proxy: %w", err) + } + app.GET("/", pkgproxy.LandingHandler(&repoConfig, Version)) app.Use(pkgProxy.Cache) app.Use(pkgProxy.ForwardProxy) diff --git a/cmd/serve_test.go b/cmd/serve_test.go index a5e3d3d..7a10a7a 100644 --- a/cmd/serve_test.go +++ b/cmd/serve_test.go @@ -275,54 +275,3 @@ func TestParseTrustProxy(t *testing.T) { }) } } - -func TestResolvePublicAddr(t *testing.T) { - tests := []struct { - name string - flagValue string - envValue string - listenAddr string - listenPort uint16 - want string - }{ - { - name: "flag takes precedence over env var", - flagValue: "myproxy.lan", - envValue: "other.host", - listenAddr: "localhost", - listenPort: 8080, - want: "myproxy.lan", - }, - { - name: "env var used when flag is empty", - flagValue: "", - envValue: "myproxy.lan", - listenAddr: "localhost", - listenPort: 8080, - want: "myproxy.lan", - }, - { - name: "defaults to listen host:port when neither is set", - flagValue: "", - envValue: "", - listenAddr: "localhost", - listenPort: 8080, - want: "localhost:8080", - }, - { - name: "flag with embedded port used verbatim", - flagValue: "myproxy.lan:9090", - envValue: "", - listenAddr: "localhost", - listenPort: 8080, - want: "myproxy.lan:9090", - }, - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - t.Setenv(publicHostEnvVar, tt.envValue) - got := resolvePublicAddr(tt.flagValue, tt.listenAddr, tt.listenPort) - assert.Equal(t, tt.want, got) - }) - } -} diff --git a/configs/pkgproxy.yaml b/configs/pkgproxy.yaml index f94aeff..c62fbfa 100644 --- a/configs/pkgproxy.yaml +++ b/configs/pkgproxy.yaml @@ -1,4 +1,8 @@ --- +branding: + title: Pkgproxy Application + description: Caching forward proxy for Linux package repositories + repositories: almalinux: suffixes: @@ -87,6 +91,15 @@ repositories: - https://mirror.init7.net/rockylinux/ - https://mirror.puzzle.ch/rockylinux/ - https://dl.rockylinux.org/pub/rocky/ + rhel: + suffixes: + - .drpm + - .rpm + cdn: https://cdn.redhat.com/ + mtls: + cert: entitlement.pem + key: entitlement-key.pem + ca: /etc/rhsm/ca/redhat-uep.pem ubuntu: suffixes: - .deb diff --git a/docs/architecture.md b/docs/architecture.md index 561b818..a82c2ce 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -16,14 +16,22 @@ The **first path segment** of the URL is the repository name (e.g. `/fedora/...` ## Key Types -- `pkgProxy` (`pkg/pkgproxy/proxy.go`) — holds `upstreams` map (repo name → mirrors + cache instance), `transport`, and `retryBaseDelay`. The `PkgProxy` interface exposes only `Cache` and `ForwardProxy` middleware funcs. -- `upstream` — per-repository struct bundling a `FileCache`, a list of parsed mirror `*url.URL`s, and the retry count. +- `pkgProxy` (`pkg/pkgproxy/proxy.go`) — holds `upstreams` map (repo name → targets + cache instance), `transport`, and `retryBaseDelay`. The `PkgProxy` interface exposes only `Cache` and `ForwardProxy` middleware funcs. `New` returns an error, so misconfiguration (notably an unloadable mTLS key pair) aborts startup instead of failing per request. +- `upstream` — per-repository struct bundling a `FileCache`, the parsed upstream `targets` (`*url.URL`), an optional repository-scoped `transport`, and the retry count. - `FileCache` (`pkg/cache/cache.go`) — interface backed by a filesystem cache. Uses atomic write (temp file + `os.Rename`) to prevent partial reads. Path traversal is prevented in `resolvedFilePath`. -- `RepoConfig` / `Repository` (`pkg/pkgproxy/repository.go`) — YAML-loaded config: each repository has `mirrors`, `suffixes` (cache candidates), and optional `retries`. +- `RepoConfig` / `Repository` (`pkg/pkgproxy/repository.go`) — YAML-loaded config: each repository has `suffixes` (cache candidates), exactly one of `mirrors` or `cdn`, and optional `mtls` and `retries`. -## Mirror Failover & Retry (`tryMirrors`) +## Upstream Kinds -Mirrors are tried in order. Per mirror, up to `retries` attempts are made (default 1). Exponential backoff (`retryBaseDelay * 2^(attempt-2)`, starting at 1 s) is triggered only on 5xx responses. A single redirect (301/302/303/307/308) is followed per attempt. Connection-level errors skip immediately to the next mirror. The first 200 response wins; otherwise the last non-nil response is returned. +A repository is backed either by an ordered list of `mirrors` or by a single `cdn`; the two are mutually exclusive and validated at config load. Both are normalized into the same `targets` slice, so failover, retry, redirect handling and path mapping are one code path. + +A `cdn` may carry an `mtls` block. `New` loads the key pair and clones the proxy-wide `*http.Transport`, adding the client certificate to the clone's `TLSClientConfig`. An optional `mtls.ca` is appended to the system trust store (or to the base transport's existing `RootCAs`) so CDNs signed by a private CA — such as `cdn.redhat.com` — verify. The clone is stored on the `upstream`, which scopes both the credential and the extra trust to that one repository; `transportFor` falls back to the shared transport for everything else. Cloning preserves proxy env vars and timeouts. + +## Mirror Failover & Retry (`tryUpstreams`) + +Upstream targets are tried in order. Per target, up to `retries` attempts are made (default 1). Exponential backoff (`retryBaseDelay * 2^(attempt-2)`, starting at 1 s) is triggered only on 5xx responses. A single redirect (301/302/303/307/308) is followed per attempt. Connection-level errors skip immediately to the next target. The first 200 response wins; otherwise the last non-nil response is returned. + +When a repository uses a repository-scoped (mTLS) transport and the redirect points at a different host, the redirect is followed with the shared transport instead, so the client certificate is never presented to a host other than the configured CDN. ## Cache Write Path diff --git a/openspec/specs/http-landing-page/spec.md b/openspec/specs/http-landing-page/spec.md index c3d4b62..3a0d851 100644 --- a/openspec/specs/http-landing-page/spec.md +++ b/openspec/specs/http-landing-page/spec.md @@ -16,7 +16,7 @@ pkgproxy SHALL serve an HTML landing page at `GET /` that lists all configured r - **THEN** each upstream mirror URL is rendered as an HTML anchor (`<a href="...">`) that opens the mirror in the browser ### Requirement: Package manager configuration snippets match README -The landing page SHALL include copy-paste configuration snippets for repositories whose names appear in the project README client configuration section. Snippets MUST match the URL structure from the README including the full URI path suffix after the repository name (e.g. `/$releasever/BaseOS/$basearch/os/`), with `<pkgproxy>` replaced by the configured public address. Repositories not documented in the README SHALL have their snippet omitted entirely. DEB-based snippets (Debian, Ubuntu) SHALL use a `<release>` placeholder instead of hardcoded release codenames, matching the placeholder convention used by the COPR snippet (`<user>`, `<repo>`). The README retains concrete codename examples for readability; the landing page uses placeholders. +The landing page SHALL include copy-paste configuration snippets for repositories whose names appear in the project README client configuration section. Snippets MUST match the URL structure from the README including the full URI path suffix after the repository name (e.g. `/$releasever/BaseOS/$basearch/os/`), with `<pkgproxy>` replaced by an address resolved automatically (see "Automatic hostname substitution"). Repositories not documented in the README SHALL have their snippet omitted entirely. DEB-based snippets (Debian, Ubuntu) SHALL use a `<release>` placeholder instead of hardcoded release codenames, matching the placeholder convention used by the COPR snippet (`<user>`, `<repo>`). The README retains concrete codename examples for readability; the landing page uses placeholders. #### Scenario: Known RPM repository shows dnf/yum baseurl snippet with full path - **WHEN** a repository name matches one documented in the README with `.rpm` suffixes @@ -34,50 +34,39 @@ The landing page SHALL include copy-paste configuration snippets for repositorie - **WHEN** a repository name has no matching entry in the README client configuration section - **THEN** no configuration snippet is shown for that repository -#### Scenario: Snippet uses listen host:port when no public host is set -- **WHEN** no public host is configured and pkgproxy is started with `--host h --port p` -- **THEN** all config snippets on the landing page use `h:p` as the address +### Requirement: Automatic hostname substitution +Configuration snippets SHALL NOT depend on a server-side public-address setting. Instead, the address is resolved automatically in two layers: -#### Scenario: Snippet uses public address verbatim without appending listen port -- **WHEN** a public address is configured via `--public-host` or `PKGPROXY_PUBLIC_HOST` -- **THEN** all config snippets use that value verbatim and the listen port is not appended +1. **Server-side default.** Every response renders snippets using the `Host` header of the incoming request, so any client — including `curl` and other non-browser HTTP clients — receives a working, copy-pasteable address without needing JavaScript. +2. **Client-side correction.** The page additionally includes a small inline script that, once loaded in a browser, compares the server-rendered default against the page's own `window.location.origin` and, if they differ, rewrites the snippets to match it. This corrects cases the `Host` header alone cannot reveal, such as a reverse proxy that terminates TLS (the browser's scheme is `https`, but pkgproxy itself only ever sees plain HTTP). -### Requirement: Configurable public address -pkgproxy SHALL expose a `--public-host` CLI flag (on the `serve` subcommand) and a `PKGPROXY_PUBLIC_HOST` environment variable that set the address rendered in landing page config snippets. The value MAY include a port (e.g. `myproxy.lan:9090`), in which case that port is used as-is. When a public host is set, the listen port is NOT appended. When no public host is set, the listen `host:port` is used. The CLI flag takes precedence over the environment variable when both are set. +This keeps snippets correct with no pkgproxy-side configuration in the common case (reverse proxy forwarding the original `Host` header), and self-corrects in the browser when it doesn't. -#### Scenario: Flag sets the public address without appending listen port -- **WHEN** pkgproxy is started with `--public-host myproxy.lan` -- **THEN** the landing page config snippets use `myproxy.lan` with no port suffix +#### Scenario: Server renders snippets using the request's Host header +- **WHEN** a client sends `GET /` with a given `Host` header +- **THEN** every configuration snippet on the returned page uses that `Host` value as the address, with no placeholder text -#### Scenario: Flag value with embedded port is used verbatim -- **WHEN** pkgproxy is started with `--public-host myproxy.lan:9090` -- **THEN** the landing page config snippets use `myproxy.lan:9090` verbatim +#### Scenario: Inline script corrects the address to the page's origin when it differs +- **WHEN** the landing page is loaded in a browser whose `window.location.origin` differs from the server-rendered default (e.g. behind a TLS-terminating reverse proxy) +- **THEN** an inline script rewrites the address in the rendered snippets to `window.location.origin` -#### Scenario: Environment variable sets the public address -- **WHEN** `PKGPROXY_PUBLIC_HOST=myproxy.lan` is set and no `--public-host` flag is given -- **THEN** the landing page config snippets use `myproxy.lan` with no port suffix +#### Scenario: No client-side change when origins already match +- **WHEN** the landing page is loaded in a browser whose `window.location.origin` matches the server-rendered default +- **THEN** the inline script makes no changes to the rendered snippets -#### Scenario: CLI flag takes precedence over environment variable -- **WHEN** both `--public-host myproxy.lan` and `PKGPROXY_PUBLIC_HOST=other.host` are set -- **THEN** the landing page config snippets use `myproxy.lan` +#### Scenario: Server-rendered address is used verbatim without JavaScript +- **WHEN** the landing page is loaded with JavaScript disabled or unavailable, or fetched with a non-browser client such as `curl` +- **THEN** the configuration snippets show the server-rendered address derived from the request's `Host` header -#### Scenario: Default renders listen address with port -- **WHEN** neither `--public-host` nor `PKGPROXY_PUBLIC_HOST` is set -- **THEN** the landing page config snippets use `<host>:<port>` from the listen configuration - -### Requirement: README documents CLI flags and the public host option -The project README SHALL contain a CLI flags reference table covering all `serve` subcommand flags, including `--public-host` and the `PKGPROXY_PUBLIC_HOST` environment variable with a description of their effect. +### Requirement: README documents CLI flags +The project README SHALL contain a CLI flags reference table covering all `serve` subcommand flags. #### Scenario: README CLI flags table includes all serve flags - **WHEN** a user reads the README - **THEN** they find a table listing all `serve` subcommand flags with their defaults and descriptions -#### Scenario: README CLI flags table includes public-host and its env var -- **WHEN** a user reads the README -- **THEN** they can find `--public-host` and `PKGPROXY_PUBLIC_HOST` with a description of their effect - -### Requirement: No external dependencies for page rendering -The landing page SHALL be rendered using only Go standard library (`html/template`), with no JavaScript or external stylesheet resources. +### Requirement: No external script, stylesheet, or font dependencies +The landing page SHALL be rendered using only the Go standard library (`html/template`) plus a small inline script used solely for client-side hostname correction (see "Automatic hostname substitution"). It SHALL NOT load external JavaScript, stylesheets, or fonts. #### Scenario: Page is self-contained - **WHEN** the landing page HTML is served diff --git a/pkg/pkgproxy/landing.go b/pkg/pkgproxy/landing.go index e4724d8..359e45f 100644 --- a/pkg/pkgproxy/landing.go +++ b/pkg/pkgproxy/landing.go @@ -16,7 +16,7 @@ const landingTemplate = `<!DOCTYPE html> <head> <meta charset="UTF-8"> <meta name="viewport" content="width=device-width, initial-scale=1.0"> -<title>pkgproxy +{{.Title}} -

pkgproxy

-

Caching forward proxy for Linux package repositories.

-{{range .}} +

{{.Title}}

+

{{.Description}}

+

pkgproxy {{.Version}}

+{{range .Repos}}

{{.Name}}

+{{if .CDN}} +

CDN:

+ +{{else}}

Mirrors:

-{{with repoSnippet .Name}} +{{end}} +{{with repoSnippet .Name $.Addr}}

Configuration snippet:

{{.}}
{{end}} {{end}} + ` +// defaultTitle and defaultDescription are used when the config's 'branding' +// block is absent or leaves a field empty. +const ( + defaultTitle = "pkgproxy" + defaultDescription = "Caching forward proxy for Linux package repositories." +) + // snippetFuncs maps known repository names to functions that generate // package manager configuration snippets for the landing page. // Each function takes the public address (host or host:port) and returns @@ -95,6 +119,11 @@ var snippetFuncs = map[string]func(string) string{ "# mirrorlist=https://mirrors.rockylinux.org/mirrorlist?arch=$basearch&repo=BaseOS-$releasever$rltype\n" + "baseurl=http://" + addr + "/rockylinux/$releasever/BaseOS/$basearch/os/" }, + "rhel": func(addr string) string { + return "[rhel-baseos-rpms]\n" + + "# baseurl=https://cdn.redhat.com/content/dist/rhel$releasever/$releasever/$basearch/baseos/os\n" + + "baseurl=http://" + addr + "/rhel/content/dist/rhel$releasever/$releasever/$basearch/baseos/os" + }, "ubuntu": func(addr string) string { return "deb http://" + addr + "/ubuntu main restricted universe multiverse\n" + "deb http://" + addr + "/ubuntu -updates main restricted universe multiverse" @@ -108,6 +137,38 @@ var snippetFuncs = map[string]func(string) string{ type repoEntry struct { Name string Mirrors []string + CDN string +} + +// landingData is the top-level data passed to the landing page template. +type landingData struct { + Title string + Description string + Version string + // Addr is the host[:port] that config snippets are rendered with by + // default, taken from the incoming request's Host header. This makes + // snippets immediately usable for non-JS clients like curl. The inline + // script in landingTemplate additionally corrects it in a browser to + // window.location.origin when that differs (e.g. behind a + // TLS-terminating reverse proxy). + Addr string + Repos []repoEntry +} + +// brandingOrDefault returns the configured title and description, falling +// back to the built-in pkgproxy defaults for whichever field is unset. +func brandingOrDefault(branding *BrandingConfig) (title string, description string) { + title, description = defaultTitle, defaultDescription + if branding == nil { + return title, description + } + if branding.Title != "" { + title = branding.Title + } + if branding.Description != "" { + description = branding.Description + } + return title, description } // sortedRepos returns repository entries sorted alphabetically by name. @@ -120,29 +181,48 @@ func sortedRepos(config *RepoConfig) []repoEntry { entries := make([]repoEntry, 0, len(names)) for _, name := range names { - entries = append(entries, repoEntry{Name: name, Mirrors: config.Repositories[name].Mirrors}) + entries = append(entries, repoEntry{ + Name: name, + Mirrors: config.Repositories[name].Mirrors, + CDN: config.Repositories[name].CDN, + }) } return entries } // LandingHandler returns an Echo handler that renders an HTML overview page -// listing all configured repositories, their mirrors, and package manager snippets. -// publicAddr is the address (host or host:port) rendered in config snippets. -func LandingHandler(config *RepoConfig, publicAddr string) echo.HandlerFunc { +// listing all configured repositories, their mirrors / CDN, and package manager +// snippets. Snippet hostnames default to the incoming request's Host header +// (so plain HTTP clients like curl get a working address), and are further +// corrected client-side to the page's own URL when a browser loads it behind +// a reverse proxy that changes the scheme (see landingData.Addr). version is +// the pkgproxy build version shown on the page. +func LandingHandler(config *RepoConfig, version string) echo.HandlerFunc { funcMap := template.FuncMap{ - "repoSnippet": func(name string) string { + "repoSnippet": func(name string, addr string) string { fn, ok := snippetFuncs[name] if !ok { return "" } - return fn(publicAddr) + return fn(addr) }, } tmpl := template.Must(template.New("landing").Funcs(funcMap).Parse(landingTemplate)) + title, description := brandingOrDefault(config.Branding) + repos := sortedRepos(config) + return func(c *echo.Context) error { + data := landingData{ + Title: title, + Description: description, + Version: version, + Addr: c.Request().Host, + Repos: repos, + } + var buf bytes.Buffer - if err := tmpl.Execute(&buf, sortedRepos(config)); err != nil { + if err := tmpl.Execute(&buf, data); err != nil { return err } c.Response().Header().Set(echo.HeaderContentType, "text/html; charset=UTF-8") diff --git a/pkg/pkgproxy/landing_test.go b/pkg/pkgproxy/landing_test.go index 1ef9bc4..02c5389 100644 --- a/pkg/pkgproxy/landing_test.go +++ b/pkg/pkgproxy/landing_test.go @@ -11,15 +11,34 @@ import ( "github.com/stretchr/testify/assert" ) -func newLandingApp(config *RepoConfig, publicAddr string) *echo.Echo { +// defaultTestHost is the Host httptest.NewRequest assigns to a relative +// target ("/") when the test doesn't set req.Host explicitly. +const defaultTestHost = "example.com" + +func newLandingApp(config *RepoConfig) *echo.Echo { + return newLandingAppWithVersion(config, "1.2.3") +} + +func newLandingAppWithVersion(config *RepoConfig, version string) *echo.Echo { app := echo.New() - app.GET("/", LandingHandler(config, publicAddr)) + app.GET("/", LandingHandler(config, version)) return app } func getLandingBody(t *testing.T, app *echo.Echo) string { + t.Helper() + return getLandingBodyWithHost(t, app, "") +} + +// getLandingBodyWithHost issues GET / with the given Host header (the header +// curl and other HTTP clients send based on the URL they were given). An +// empty host leaves httptest's default ("example.com") in place. +func getLandingBodyWithHost(t *testing.T, app *echo.Echo, host string) string { t.Helper() req := httptest.NewRequest(http.MethodGet, "/", nil) + if host != "" { + req.Host = host + } rec := httptest.NewRecorder() app.ServeHTTP(rec, req) assert.Equal(t, http.StatusOK, rec.Code) @@ -32,7 +51,7 @@ func TestLandingHandlerHTTP(t *testing.T) { "fedora": {CacheSuffixes: []string{".rpm"}, Mirrors: []string{"https://mirror.example.com/fedora/"}}, }, } - app := newLandingApp(config, "localhost:8080") + app := newLandingApp(config) req := httptest.NewRequest(http.MethodGet, "/", nil) rec := httptest.NewRecorder() app.ServeHTTP(rec, req) @@ -49,7 +68,7 @@ func TestLandingHandlerRepoNames(t *testing.T) { "unknown": {CacheSuffixes: []string{".rpm"}, Mirrors: []string{"https://mirror.example.com/unknown/"}}, }, } - body := getLandingBody(t, newLandingApp(config, "localhost:8080")) + body := getLandingBody(t, newLandingApp(config)) assert.Contains(t, body, "fedora") assert.Contains(t, body, "debian") @@ -62,28 +81,104 @@ func TestLandingHandlerMirrorLinks(t *testing.T) { "fedora": {CacheSuffixes: []string{".rpm"}, Mirrors: []string{"https://mirror.example.com/fedora/"}}, }, } - body := getLandingBody(t, newLandingApp(config, "localhost:8080")) + body := getLandingBody(t, newLandingApp(config)) assert.Contains(t, body, `https://mirror.example.com/fedora/`) } +func TestLandingHandlerDefaultBranding(t *testing.T) { + config := &RepoConfig{ + Repositories: map[string]Repository{ + "fedora": {CacheSuffixes: []string{".rpm"}, Mirrors: []string{"https://mirror.example.com/"}}, + }, + } + body := getLandingBody(t, newLandingApp(config)) + + assert.Contains(t, body, "pkgproxy") + assert.Contains(t, body, "

pkgproxy

") + assert.Contains(t, body, "

Caching forward proxy for Linux package repositories.

") +} + +func TestLandingHandlerCustomBranding(t *testing.T) { + config := &RepoConfig{ + Branding: &BrandingConfig{ + Title: "Acme Package Mirror", + Description: "Internal package cache for Acme Corp.", + }, + Repositories: map[string]Repository{ + "fedora": {CacheSuffixes: []string{".rpm"}, Mirrors: []string{"https://mirror.example.com/"}}, + }, + } + body := getLandingBody(t, newLandingApp(config)) + + assert.Contains(t, body, "Acme Package Mirror") + assert.Contains(t, body, "

Acme Package Mirror

") + assert.Contains(t, body, "

Internal package cache for Acme Corp.

") + assert.NotContains(t, body, "pkgproxy") +} + +func TestLandingHandlerBrandingPartialOverride(t *testing.T) { + config := &RepoConfig{ + Branding: &BrandingConfig{Title: "Acme Package Mirror"}, + Repositories: map[string]Repository{ + "fedora": {CacheSuffixes: []string{".rpm"}, Mirrors: []string{"https://mirror.example.com/"}}, + }, + } + body := getLandingBody(t, newLandingApp(config)) + + assert.Contains(t, body, "

Acme Package Mirror

") + // Description falls back to the default when only the title is customized. + assert.Contains(t, body, "

Caching forward proxy for Linux package repositories.

") +} + +func TestLandingHandlerVersion(t *testing.T) { + config := &RepoConfig{ + Repositories: map[string]Repository{ + "fedora": {CacheSuffixes: []string{".rpm"}, Mirrors: []string{"https://mirror.example.com/"}}, + }, + } + body := getLandingBody(t, newLandingAppWithVersion(config, "v0.3.1")) + + assert.Contains(t, body, "

pkgproxy v0.3.1

") +} + +func TestLandingHandlerCDNLink(t *testing.T) { + config := &RepoConfig{ + Repositories: map[string]Repository{ + "rhel": { + CacheSuffixes: []string{".rpm"}, + CDN: "https://cdn.redhat.com/", + MTLS: &MTLSConfig{Cert: "entitlement.pem", Key: "entitlement-key.pem"}, + }, + }, + } + body := getLandingBody(t, newLandingApp(config)) + + assert.Contains(t, body, "CDN:") + assert.Contains(t, body, `https://cdn.redhat.com/`) + assert.NotContains(t, body, "Mirrors:") + assert.Contains(t, body, "baseurl=http://"+defaultTestHost+"/rhel/content/dist/rhel$releasever/$releasever/$basearch/baseos/os") + // The certificate paths are local secrets and must never be rendered. + assert.NotContains(t, body, "entitlement") +} + func TestLandingHandlerKnownSnippets(t *testing.T) { tests := []struct { repo string suffix string wantIn string }{ - {"almalinux", ".rpm", "baseurl=http://localhost:8080/almalinux/$releasever/BaseOS/$basearch/os/"}, - {"archlinux", ".tar.zst", "Server = http://localhost:8080/archlinux/$repo/os/$arch"}, - {"centos", ".rpm", "baseurl=http://localhost:8080/centos/$releasever/os/$basearch/"}, - {"centos-stream", ".rpm", "baseurl=http://localhost:8080/centos-stream/$stream/BaseOS/$basearch/os/"}, - {"debian", ".deb", "deb http://localhost:8080/debian <release> main contrib non-free non-free-firmware"}, - {"debian-security", ".deb", "deb http://localhost:8080/debian-security <release>-security main contrib non-free non-free-firmware"}, - {"epel", ".rpm", "baseurl=http://localhost:8080/epel/$releasever/Everything/$basearch/"}, - {"fedora", ".rpm", "baseurl=http://localhost:8080/fedora/releases/$releasever/Everything/$basearch/os/"}, - {"rockylinux", ".rpm", "baseurl=http://localhost:8080/rockylinux/$releasever/BaseOS/$basearch/os/"}, - {"ubuntu", ".deb", "deb http://localhost:8080/ubuntu <release> main restricted universe multiverse"}, - {"ubuntu-security", ".deb", "deb http://localhost:8080/ubuntu-security <release>-security main restricted universe multiverse"}, + {"almalinux", ".rpm", "baseurl=http://" + defaultTestHost + "/almalinux/$releasever/BaseOS/$basearch/os/"}, + {"archlinux", ".tar.zst", "Server = http://" + defaultTestHost + "/archlinux/$repo/os/$arch"}, + {"centos", ".rpm", "baseurl=http://" + defaultTestHost + "/centos/$releasever/os/$basearch/"}, + {"centos-stream", ".rpm", "baseurl=http://" + defaultTestHost + "/centos-stream/$stream/BaseOS/$basearch/os/"}, + {"debian", ".deb", "deb http://" + defaultTestHost + "/debian <release> main contrib non-free non-free-firmware"}, + {"debian-security", ".deb", "deb http://" + defaultTestHost + "/debian-security <release>-security main contrib non-free non-free-firmware"}, + {"epel", ".rpm", "baseurl=http://" + defaultTestHost + "/epel/$releasever/Everything/$basearch/"}, + {"fedora", ".rpm", "baseurl=http://" + defaultTestHost + "/fedora/releases/$releasever/Everything/$basearch/os/"}, + {"rockylinux", ".rpm", "baseurl=http://" + defaultTestHost + "/rockylinux/$releasever/BaseOS/$basearch/os/"}, + {"ubuntu", ".deb", "deb http://" + defaultTestHost + "/ubuntu <release> main restricted universe multiverse"}, + {"ubuntu-security", ".deb", "deb http://" + defaultTestHost + "/ubuntu-security <release>-security main restricted universe multiverse"}, } for _, tt := range tests { t.Run(tt.repo, func(t *testing.T) { @@ -92,7 +187,7 @@ func TestLandingHandlerKnownSnippets(t *testing.T) { tt.repo: {CacheSuffixes: []string{tt.suffix}, Mirrors: []string{"https://mirror.example.com/"}}, }, } - body := getLandingBody(t, newLandingApp(config, "localhost:8080")) + body := getLandingBody(t, newLandingApp(config)) assert.Contains(t, body, tt.wantIn) }) } @@ -104,33 +199,49 @@ func TestLandingHandlerUnknownRepoNoSnippet(t *testing.T) { "myprivaterepo": {CacheSuffixes: []string{".rpm"}, Mirrors: []string{"https://mirror.example.com/"}}, }, } - body := getLandingBody(t, newLandingApp(config, "localhost:8080")) + body := getLandingBody(t, newLandingApp(config)) assert.NotContains(t, body, "Configuration snippet") assert.NotContains(t, body, "baseurl=") } -func TestLandingHandlerPublicHostNoPort(t *testing.T) { +// TestLandingHandlerUsesRequestHost is the core of curl support: snippets are +// rendered server-side using whatever Host header the client's request +// carried, so a plain HTTP client that never runs JavaScript still gets a +// working, copy-pasteable address — not a placeholder. +func TestLandingHandlerUsesRequestHost(t *testing.T) { config := &RepoConfig{ Repositories: map[string]Repository{ "fedora": {CacheSuffixes: []string{".rpm"}, Mirrors: []string{"https://mirror.example.com/"}}, }, } - body := getLandingBody(t, newLandingApp(config, "myproxy.lan")) + app := newLandingApp(config) + + body := getLandingBodyWithHost(t, app, "myproxy.lan:9090") + assert.Contains(t, body, "baseurl=http://myproxy.lan:9090/fedora/releases/$releasever/Everything/$basearch/os/") - assert.Contains(t, body, "http://myproxy.lan/fedora/") - assert.NotContains(t, body, "myproxy.lan:") + body = getLandingBodyWithHost(t, app, "other.example:8080") + assert.Contains(t, body, "baseurl=http://other.example:8080/fedora/releases/$releasever/Everything/$basearch/os/") } -func TestLandingHandlerPublicHostWithPort(t *testing.T) { +// TestLandingHandlerHostSubstitutionScript verifies the page ships the inline +// script that further corrects the server-rendered address to the browser's +// own URL on load (needed when a reverse proxy changes the scheme, e.g. TLS +// termination, which the Host header alone can't reveal). The Go test suite +// has no JS engine, so this only checks the script is wired up correctly +// rather than executing it. +func TestLandingHandlerHostSubstitutionScript(t *testing.T) { config := &RepoConfig{ Repositories: map[string]Repository{ "fedora": {CacheSuffixes: []string{".rpm"}, Mirrors: []string{"https://mirror.example.com/"}}, }, } - body := getLandingBody(t, newLandingApp(config, "myproxy.lan:9090")) + body := getLandingBodyWithHost(t, newLandingApp(config), "myproxy.lan") - assert.Contains(t, body, "http://myproxy.lan:9090/fedora/") + assert.Contains(t, body, "baseurl=http://myproxy.lan/fedora/") + assert.Contains(t, body, `var defaultOrigin = "http://myproxy.lan";`) + assert.Contains(t, body, "var origin = window.location.origin;") + assert.Contains(t, body, `document.querySelectorAll("pre")`) } func TestLandingHandlerDefaultListenAddr(t *testing.T) { @@ -139,9 +250,9 @@ func TestLandingHandlerDefaultListenAddr(t *testing.T) { "fedora": {CacheSuffixes: []string{".rpm"}, Mirrors: []string{"https://mirror.example.com/"}}, }, } - body := getLandingBody(t, newLandingApp(config, "localhost:8080")) + body := getLandingBody(t, newLandingApp(config)) - assert.Contains(t, body, "http://localhost:8080/fedora/") + assert.Contains(t, body, "http://"+defaultTestHost+"/fedora/") } func TestLandingHandlerSelfContained(t *testing.T) { @@ -150,9 +261,11 @@ func TestLandingHandlerSelfContained(t *testing.T) { "fedora": {CacheSuffixes: []string{".rpm"}, Mirrors: []string{"https://mirror.example.com/"}}, }, } - body := getLandingBody(t, newLandingApp(config, "localhost:8080")) + body := getLandingBody(t, newLandingApp(config)) assert.NotContains(t, body, "https://fonts.") + // The inline hostname-substitution script is allowed; only *external* + // script/stylesheet/font references are not. assert.NotContains(t, body, "