diff --git a/.github/workflows/generate.yaml b/.github/workflows/generate.yaml index 4169f74e0..0352a5f19 100644 --- a/.github/workflows/generate.yaml +++ b/.github/workflows/generate.yaml @@ -25,31 +25,27 @@ jobs: steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + # Pinned exactly, not floated: protoc stamps its version into every generated + # file, so a release bump rewrites the committed output. Moving this is a + # deliberate edit whose header churn is reviewable on its own. - name: Install Protoc uses: arduino/setup-protoc@c65c819552d16ad3c9b72d9dfd5ba5237b9c906b # v3.0.0 with: - version: "35.x" + version: "35.1" repo-token: ${{ secrets.GITHUB_TOKEN }} - name: Set up Go uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v7.0.0 with: - go-version: ">=1.26" + go-version-file: "go.mod" - - name: Go mod tidy - run: go mod tidy - - - name: Install generators - uses: magefile/mage-action@a662bd8c29d8106879588cfff83b2faf6e6f59db # v4.0.0 - with: - version: latest - install-only: true - - - name: Bootstrap - run: mage bootstrap + - name: Download Go modules + run: go mod download + # The protoc plugins and mage itself come from the tool directives in go.mod, so + # generation runs the versions pinned there instead of whatever is latest today. - name: Generate Protobuf - run: mage proto + run: go tool mage proto - uses: pnpm/action-setup@0977fd99725f1db4007ccb2928dbb4e90d06cc86 # v6.0.10 diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml index 2d73673b3..5988312c0 100644 --- a/.github/workflows/release.yaml +++ b/.github/workflows/release.yaml @@ -19,9 +19,13 @@ jobs: steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + # The published JS package is generated at build time by pnpm generate:proto, so + # protoc is a release input. Keep it on the same release as generate.yaml, or the + # published package is generated by a different compiler than the committed Go code. - name: Install Protoc uses: arduino/setup-protoc@c65c819552d16ad3c9b72d9dfd5ba5237b9c906b # v3.0.0 with: + version: "35.1" repo-token: ${{ secrets.GITHUB_TOKEN }} - uses: pnpm/action-setup@0977fd99725f1db4007ccb2928dbb4e90d06cc86 # v6.0.10 diff --git a/bootstrap.sh b/bootstrap.sh deleted file mode 100755 index 0a3d09f07..000000000 --- a/bootstrap.sh +++ /dev/null @@ -1,57 +0,0 @@ -#!/bin/bash -# Copyright 2023 LiveKit, Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - - -if ! command -v protoc &> /dev/null -then - echo "protoc is required and not found. please install" - exit 1 -fi - -if ! command -v mage &> /dev/null -then - pushd /tmp - git clone https://github.com/magefile/mage - cd mage - go run bootstrap.go - rm -rf /tmp/mage - popd -fi - -if ! command -v mage &> /dev/null -then - echo "Ensure `go env GOPATH`/bin is in your \$PATH" - exit 1 -fi - -go mod download - -GO_VERSION=`go version | { read _ _ v _; echo ${v#go}; }` -GO_TARGET_VERSION=1.17 - -function version { echo "$@" | awk -F. '{ printf("%d%03d%03d%03d\n", $1,$2,$3,$4); }'; } - -if [ $(version $GO_VERSION) -ge $(version $GO_TARGET_VERSION) ]; - then - go install github.com/twitchtv/twirp/protoc-gen-twirp@v8.1.3 - go install google.golang.org/protobuf/cmd/protoc-gen-go@v1.31.0 - go install google.golang.org/grpc/cmd/protoc-gen-go-grpc@v1.3 - go install github.com/livekit/psrpc/protoc-gen-psrpc@v0.5.1 - else - go get -u github.com/twitchtv/twirp/protoc-gen-twirp@v8.1.3 - go get -u google.golang.org/protobuf/cmd/protoc-gen-go@v1.30.0 - go get -u google.golang.org/grpc/cmd/protoc-gen-go-grpc@v1.2 - go get -u github.com/livekit/psrpc/protoc-gen-psrpc@v0.3.1 -fi diff --git a/go.mod b/go.mod index cce8163f1..4740fd41a 100644 --- a/go.mod +++ b/go.mod @@ -46,7 +46,7 @@ require ( golang.org/x/text v0.40.0 google.golang.org/genproto/googleapis/rpc v0.0.0-20260526163538-3dc84a4a5aaa google.golang.org/grpc v1.83.0 - google.golang.org/protobuf v1.36.11 + google.golang.org/protobuf v1.36.12 gopkg.in/yaml.v3 v3.0.1 ) @@ -65,6 +65,7 @@ require ( github.com/hashicorp/go-cleanhttp v0.5.2 // indirect github.com/klauspost/compress v1.19.1 // indirect github.com/klauspost/cpuid/v2 v2.3.0 // indirect + github.com/magefile/mage v1.17.2 // indirect github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect github.com/nats-io/nats.go v1.52.0 // indirect github.com/nats-io/nkeys v0.4.16 // indirect @@ -82,6 +83,7 @@ require ( github.com/pion/stun/v3 v3.1.6 // indirect github.com/pion/transport/v4 v4.0.2 // indirect github.com/pion/turn/v5 v5.0.12 // indirect + github.com/pkg/errors v0.9.1 // indirect github.com/prometheus/common v0.70.1 // indirect github.com/wlynxg/anet v0.0.5 // indirect go.opentelemetry.io/auto/sdk v1.2.1 // indirect @@ -95,4 +97,13 @@ require ( golang.org/x/time v0.15.0 // indirect golang.org/x/tools v0.47.0 // indirect google.golang.org/genproto/googleapis/api v0.0.0-20260526163538-3dc84a4a5aaa // indirect + google.golang.org/grpc/cmd/protoc-gen-go-grpc v1.6.2 // indirect +) + +tool ( + github.com/livekit/psrpc/protoc-gen-psrpc + github.com/magefile/mage + github.com/twitchtv/twirp/protoc-gen-twirp + google.golang.org/grpc/cmd/protoc-gen-go-grpc + google.golang.org/protobuf/cmd/protoc-gen-go ) diff --git a/go.sum b/go.sum index b96e9dc88..e7ae5f2e8 100644 --- a/go.sum +++ b/go.sum @@ -89,6 +89,8 @@ github.com/livekit/psrpc v0.7.3 h1:bekuZt/ZQzg8+/M8G6G5jq7bvV9fAKdPHSOZeTwrIIc= github.com/livekit/psrpc v0.7.3/go.mod h1:rAI+m2+/cb4x9RXhLRtUx5ZwdfjjXOl4zi46IjEetaw= github.com/mackerelio/go-osstat v0.2.8 h1:I2duicTaCGWoM53XwAwA9OIe1inu0xnVs8/pqOWWVr4= github.com/mackerelio/go-osstat v0.2.8/go.mod h1:SyS3XxKdoSKJnTGTkN5Yrh6VUQVuAURACfE6y+2DN4k= +github.com/magefile/mage v1.17.2 h1:fyXVu1eadI8Ap1HCCNgEhJ5McIWiYhLR8uol64ZZc40= +github.com/magefile/mage v1.17.2/go.mod h1:Yj51kqllmsgFpvvSzgrZPK9WtluG3kUhFaBUVLo4feA= github.com/mattn/go-colorable v0.1.13 h1:fFA4WZxdEF4tXPZVKMLwD8oUnCTTo08duU7wxecdEvA= github.com/mattn/go-colorable v0.1.13/go.mod h1:7S9/ev0klgBDR4GtXTXX8a3vIGJpMovkB8vQcUbaXHg= github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY= @@ -247,8 +249,10 @@ google.golang.org/genproto/googleapis/rpc v0.0.0-20260526163538-3dc84a4a5aaa h1: google.golang.org/genproto/googleapis/rpc v0.0.0-20260526163538-3dc84a4a5aaa/go.mod h1:4Hqkh8ycfw05ld/3BWL7rJOSfebL2Q+DVDeRgYgxUU8= google.golang.org/grpc v1.83.0 h1:JeNZEKJFbQxArAMl+hiytHauacDNqJUllNfmIMmpqnQ= google.golang.org/grpc v1.83.0/go.mod h1:kDyl6SKsiHKt0uylY5gtn5cEjkrIOhQOGDgIc4JGwzQ= -google.golang.org/protobuf v1.36.11 h1:fV6ZwhNocDyBLK0dj+fg8ektcVegBBuEolpbTQyBNVE= -google.golang.org/protobuf v1.36.11/go.mod h1:HTf+CrKn2C3g5S8VImy6tdcUvCska2kB7j23XfzDpco= +google.golang.org/grpc/cmd/protoc-gen-go-grpc v1.6.2 h1:rgSNvqscFZ1JgV/4wH5GOsZFSFkR2Eua9As3KIr2LlM= +google.golang.org/grpc/cmd/protoc-gen-go-grpc v1.6.2/go.mod h1:iMEtFwDlAhjDU9L5mY6U1XLwlIId/G3h+QcBHDIvrJ8= +google.golang.org/protobuf v1.36.12 h1:pJOKDDOyeXErUroCihFAd5LQuwXBSpVnKGrj5o/fwxc= +google.golang.org/protobuf v1.36.12/go.mod h1:HTf+CrKn2C3g5S8VImy6tdcUvCska2kB7j23XfzDpco= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= diff --git a/magefile.go b/magefile.go index 9a5783d47..5b42cfd6d 100644 --- a/magefile.go +++ b/magefile.go @@ -24,6 +24,7 @@ import ( "os" "os/exec" "path/filepath" + "strings" "github.com/livekit/mageutil" "github.com/livekit/protocol/psrpc" @@ -31,6 +32,20 @@ import ( var Default = Proto +// install the protoc plugins system-wide, for developers who want them on their PATH +// +// This is purely a developer convenience and is not required to generate anything: no +// build, release or CI step invokes it, and none should. `mage proto` resolves each +// plugin through the tool directives in go.mod (see goToolPath) and passes protoc an +// explicit path, so it works on a clean checkout and ignores whatever these installs put +// in GOPATH/bin. +// +// That separation is why @latest is safe here. These binaries are for ad-hoc use outside +// generation — inspecting a descriptor, compiling a scratch .proto — and are not inputs +// to any committed file. Two things keep them out of the committed output: the tool +// directives above, and the Generate workflow, which regenerates and commits the +// generated tree on every push to a non-main branch, so what lands on main always comes +// from the pinned versions regardless of what a contributor had installed locally. func Bootstrap() error { return mageutil.Run(context.Background(), "go install github.com/twitchtv/twirp/protoc-gen-twirp@latest", @@ -113,15 +128,15 @@ func Proto() error { if err != nil { return err } - protocGoPath, err := getToolPath("protoc-gen-go") + protocGoPath, err := goToolPath("protoc-gen-go") if err != nil { return err } - twirpPath, err := getToolPath("protoc-gen-twirp") + twirpPath, err := goToolPath("protoc-gen-twirp") if err != nil { return err } - protocGrpcGoPath, err := getToolPath("protoc-gen-go-grpc") + protocGrpcGoPath, err := goToolPath("protoc-gen-go-grpc") if err != nil { return err } @@ -132,8 +147,8 @@ func Proto() error { "--twirp_out", target, "--go_opt=paths=source_relative", "--twirp_opt=paths=source_relative", - "--plugin=go=" + protocGoPath, - "--plugin=twirp=" + twirpPath, + "--plugin=protoc-gen-go=" + protocGoPath, + "--plugin=protoc-gen-twirp=" + twirpPath, "-I=./protobufs", }, twirpProtoFiles...) cmd := exec.Command(protoc, args...) @@ -146,7 +161,7 @@ func Proto() error { args = append([]string{ "--go_out", target, "--go_opt=paths=source_relative", - "--plugin=go=" + protocGoPath, + "--plugin=protoc-gen-go=" + protocGoPath, "-I=./protobufs", }, protoFiles...) cmd = exec.Command(protoc, args...) @@ -160,7 +175,7 @@ func Proto() error { args := []string{ "--go_out", target, "--go_opt=paths=source_relative", - "--plugin=go=" + protocGoPath, + "--plugin=protoc-gen-go=" + protocGoPath, "-I=./protobufs", } args = append(args, agentProtoFiles...) @@ -181,7 +196,7 @@ func Proto() error { args := []string{ "--go_out", target, "--go_opt=paths=source_relative", - "--plugin=go=" + protocGoPath, + "--plugin=protoc-gen-go=" + protocGoPath, "-I=./protobufs", } for _, protoFile := range protoFiles { @@ -200,8 +215,8 @@ func Proto() error { "--go-grpc_out", ".", "--go_opt=paths=source_relative", "--go-grpc_opt=paths=source_relative", - "--plugin=go=" + protocGoPath, - "--plugin=go-grpc=" + protocGrpcGoPath, + "--plugin=protoc-gen-go=" + protocGoPath, + "--plugin=protoc-gen-go-grpc=" + protocGrpcGoPath, "-I=./protobufs", }, grpcProtoFiles...) cmd = exec.Command(protoc, args...) @@ -216,7 +231,7 @@ func Proto() error { if err != nil { return err } - psrpcPath, err := mageutil.GetToolPath("protoc-gen-psrpc") + psrpcPath, err := goToolPath("protoc-gen-psrpc") if err != nil { return err } @@ -229,8 +244,8 @@ func Proto() error { "--psrpc_out", ".", "--go_opt=paths=source_relative", "--psrpc_opt=paths=source_relative", - "--plugin=go=" + protocGoPath, - "--plugin=psrpc=" + psrpcPath, + "--plugin=protoc-gen-go=" + protocGoPath, + "--plugin=protoc-gen-psrpc=" + psrpcPath, "-I" + psrpcDir + "/protoc-gen-psrpc/options", "-I=./protobufs", }, psrpcProtoFiles...) @@ -252,6 +267,22 @@ func Test() error { // helpers +// goToolPath builds a protoc plugin from the tool directives in go.mod and returns its +// path, so generation uses the pinned version rather than whatever happens to be on PATH. +func goToolPath(name string) (string, error) { + out, err := exec.Command("go", "tool", "-n", name).Output() + if err != nil { + return "", fmt.Errorf("resolving tool %s: %w", name, err) + } + path := strings.TrimSpace(string(out)) + if path == "" { + return "", fmt.Errorf("resolving tool %s: no path returned", name) + } + return path, nil +} + +// getToolPath locates a binary that is not a Go tool, i.e. protoc itself, which CI +// installs with arduino/setup-protoc. func getToolPath(name string) (string, error) { if p, err := exec.LookPath(name); err == nil { return p, nil diff --git a/renovate.json b/renovate.json index fbe269eeb..4835c578c 100644 --- a/renovate.json +++ b/renovate.json @@ -1,25 +1,60 @@ { "$schema": "https://docs.renovatebot.com/renovate-schema.json", - "extends": ["config:base", "helpers:pinGitHubActionDigests"], - "constraints": { - "go": "1.22" - }, - "minimumReleaseAge": "2 weeks", + "extends": [ + "config:recommended", + "helpers:pinGitHubActionDigests" + ], "commitBody": "Generated by renovateBot", + "prConcurrentLimit": 5, + "minimumReleaseAge": "2 weeks", + "vulnerabilityAlerts": { + "enabled": true, + "minimumReleaseAge": "2 days", + "prConcurrentLimit": 0, + "prHourlyLimit": 0, + "schedule": [] + }, "packageRules": [ { "matchManagers": ["github-actions"], "groupName": "github workflows" }, { + "description": "Go modules are never grouped: each module gets its own PR so a bad bump can be reverted on its own", + "matchManagers": ["gomod"], + "groupName": null + }, + { + "description": "First-party deps are grouped and never quarantined: we track unreleased pseudo-versions that change constantly, so one PR per module would flood the queue", "matchManagers": ["gomod"], - "groupName": "go deps" + "groupName": "livekit deps", + "matchPackageNames": [ + "github.com/livekit{/,}**" + ], + "minimumReleaseAge": null + }, + { + "description": "Exception to the rule above: the pion modules are co-released and depend on each other, so bumping them one at a time produces PRs that don't build", + "matchManagers": ["gomod"], + "matchPackageNames": [ + "github.com/pion{/,}**" + ], + "groupName": "pion deps" + }, + { + "description": "Never bump the go/toolchain directives in go.mod: this module is a library, so the go directive is the minimum we ask of consumers and must stay at the lowest version our dependencies require.", + "matchManagers": ["gomod"], + "matchDepTypes": ["golang", "toolchain"], + "enabled": false }, { "matchManagers": ["npm"], "groupName": "npm deps" } ], - "postUpdateOptions": ["gomodTidy"], - "schedule": ["on monday"] + "postUpdateOptions": [ + "gomodTidy" + ], + "schedule": ["before 9am on monday"], + "updateNotScheduled": false }