From c75d223e666df4ad8baaeffb3073262edf968c97 Mon Sep 17 00:00:00 2001 From: Patrik Beqo Date: Wed, 19 Aug 2026 14:20:12 -0400 Subject: [PATCH 01/11] feat: add OpenTelemetry v2 docs --- .../go/best-practices/context-propagation.mdx | 6 +- docs/develop/go/index.mdx | 1 + .../go/integrations/opentelemetry-v2.mdx | 421 ++++++++++++++++++ docs/develop/go/platform/observability.mdx | 18 + sidebars.js | 5 +- .../IntegrationsGrid/integrations-data.json | 9 + 6 files changed, 457 insertions(+), 3 deletions(-) create mode 100644 docs/develop/go/integrations/opentelemetry-v2.mdx diff --git a/docs/develop/go/best-practices/context-propagation.mdx b/docs/develop/go/best-practices/context-propagation.mdx index 62287d6797..c74a6d701b 100644 --- a/docs/develop/go/best-practices/context-propagation.mdx +++ b/docs/develop/go/best-practices/context-propagation.mdx @@ -16,13 +16,15 @@ tags: description: How to propagate custom key-value data across Workflow, Activity, and Child Workflow boundaries using the Temporal Go SDK. --- -Context propagation lets you pass custom key-value data from a Client to Workflows, and from Workflows to Activities and Child Workflows, without threading it through every function signature. Common use cases include propagating tracing IDs, tenant IDs, auth tokens, or other request-scoped metadata. +Context propagation lets you pass custom key-value data from a Client to Workflows, and from Workflows to Activities and Child Workflows, without threading it through every function signature. Common use cases include propagating tenant IDs, auth tokens, or other request-scoped metadata. {/* TODO: Link to /encyclopedia/context-propagation once that page lands */} :::tip -If you want to propagate tracing context, check if there is a [built-in tracing interceptor](/develop/go/platform/observability#tracing) for your library before building a custom context propagator. +For OpenTelemetry tracing, use the [OpenTelemetry v2 integration](/develop/go/integrations/opentelemetry-v2). The plugin +handles OpenTelemetry trace-context propagation automatically, so you do not need a custom Temporal context propagator +for trace identifiers. ::: diff --git a/docs/develop/go/index.mdx b/docs/develop/go/index.mdx index d8ecfb7746..458d033b9c 100644 --- a/docs/develop/go/index.mdx +++ b/docs/develop/go/index.mdx @@ -96,6 +96,7 @@ From there, you can dive deeper into any of the Temporal primitives to start bui ## [Integrations](/develop/go/integrations) - [Google ADK integration](/develop/go/integrations/google-adk) +- [OpenTelemetry v2 integration](/develop/go/integrations/opentelemetry-v2) ## Temporal Go technical resources diff --git a/docs/develop/go/integrations/opentelemetry-v2.mdx b/docs/develop/go/integrations/opentelemetry-v2.mdx new file mode 100644 index 0000000000..76981d38b7 --- /dev/null +++ b/docs/develop/go/integrations/opentelemetry-v2.mdx @@ -0,0 +1,421 @@ +--- +id: opentelemetry-v2 +title: OpenTelemetry v2 integration +sidebar_label: OpenTelemetry v2 +toc_max_heading_level: 3 +description: Configure custom tracing or automatic tracing and metrics for a Temporal Application with the Go SDK OpenTelemetry v2 plugin. +tags: + - Go SDK + - Temporal SDKs + - Integrations + - Observability +--- + +import { ReleaseNoteHeader } from '@site/src/components'; + +The OpenTelemetry v2 integration connects the Temporal Go SDK to OpenTelemetry through the +[Plugin API](/develop/plugins-guide). Use it to propagate application trace context across Temporal calls. You can also +opt in to Temporal SDK operation spans and SDK metrics. + + + +This guide demonstrates two approaches: + +- **Automatic instrumentation:** The Worker plugin creates spans for Temporal SDK operations and reports Temporal SDK + metrics. +- **Custom instrumentation:** The plugin propagates trace context, but you create the spans that describe your + application. + +This guide assumes that you understand OpenTelemetry fundamentals. For OpenTelemetry concepts and exporter options, +refer to the [OpenTelemetry documentation](https://opentelemetry.io/docs/). + +Code snippets in this guide are taken from the +[OpenTelemetry v2 sample](https://github.com/temporalio/samples-go/tree/main/opentelemetry-v2). Refer to the sample for +the complete, runnable code. + +## Prerequisites + +- Set up your local development environment by following + [Set up your local development environment](/develop/go/set-up-your-local-go). +- Leave the Temporal development server running to test the sample locally. +- Install Docker to run Jaeger for the local tracing examples. + +## Install the plugin + +Install the OpenTelemetry v2 plugin: + +```bash +go get go.temporal.io/sdk/contrib/opentelemetry-v2@latest +``` + +The samples also use the OpenTelemetry OTLP gRPC and Prometheus exporters: + +```bash +go get go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc@latest +go get go.opentelemetry.io/otel/exporters/prometheus@latest +go get github.com/prometheus/client_golang@latest +``` + +## Configure tracing + +Create and install a replay-safe tracer provider in every process that creates the plugin or calls +`temporalotel.Tracer`. The replay-safe provider gives Workflow spans consistent identities when Workflow code replays. + + +[opentelemetry-v2/setup.go](https://github.com/temporalio/samples-go/blob/main/opentelemetry-v2/setup.go) +```go +func InitializeGlobalTracerProvider( + ctx context.Context, + serviceName string, +) (*temporalotel.ReplaySafeTracerProvider, error) { + exporter, err := otlptracegrpc.New( + ctx, + otlptracegrpc.WithEndpoint("127.0.0.1:4317"), + otlptracegrpc.WithInsecure(), + ) + if err != nil { + return nil, err + } + + provider := temporalotel.NewReplaySafeTracerProvider( + // WithBatcher performs exporter I/O outside the Workflow goroutine. + sdktrace.WithBatcher(exporter), + sdktrace.WithResource(resource.NewWithAttributes( + semconv.SchemaURL, + semconv.ServiceName(serviceName), + )), + ) + otel.SetTracerProvider(provider) + return provider, nil +} +``` + + +The samples use distinct OpenTelemetry service names for each instrumented process. This makes process boundaries clear +when one trace contains spans from both a Client and a Worker. + +The OTLP exporter uses an insecure loopback connection for this local example. Configure transport security for your +production telemetry backend. + +## Enable automatic instrumentation + +Set `AddTemporalSpans` to `true` to have the plugin create spans for Temporal SDK operations performed by the Worker. +Provide `MetricsHandlerOptions` to install the plugin's Temporal SDK metrics handler. A non-nil metrics configuration +enables the complete SDK metric set. + + +[opentelemetry-v2/automatic-instrumentation/worker/main.go](https://github.com/temporalio/samples-go/blob/main/opentelemetry-v2/automatic-instrumentation/worker/main.go) +```go +plugin, err := temporalotel.NewPlugin(temporalotel.PluginOptions{ + TracerOptions: tracing.TracerOptions{ + AddTemporalSpans: true, + }, + MetricsHandlerOptions: &temporalotel.MetricsHandlerOptions{ + UseMonotonicCounters: true, + }, +}) +if err != nil { + return fmt.Errorf("unable to create plugin: %w", err) +} +``` + + +`UseMonotonicCounters` represents Temporal SDK counters as OpenTelemetry `Int64Counter` instruments instead of +`Int64UpDownCounter` instruments. + +### Export Temporal SDK metrics + +Create and install a meter provider backed by the OpenTelemetry Prometheus exporter: + + +[opentelemetry-v2/setup.go](https://github.com/temporalio/samples-go/blob/main/opentelemetry-v2/setup.go) +```go +func InitializeGlobalMeterProvider(serviceName string) (*sdkmetric.MeterProvider, error) { + exporter, err := otelprometheus.New() + if err != nil { + return nil, err + } + + provider := sdkmetric.NewMeterProvider( + sdkmetric.WithReader(exporter), + sdkmetric.WithResource(resource.NewWithAttributes( + semconv.SchemaURL, + semconv.ServiceName(serviceName), + )), + ) + otel.SetMeterProvider(provider) + return provider, nil +} +``` + + +Start a loopback-only HTTP endpoint that serves the Prometheus handler: + + +[opentelemetry-v2/setup.go](https://github.com/temporalio/samples-go/blob/main/opentelemetry-v2/setup.go) +```go +func StartPrometheusEndpoint() (*http.Server, error) { + mux := http.NewServeMux() + mux.Handle("/metrics", promhttp.Handler()) + + server := &http.Server{ + Addr: "127.0.0.1:9090", + Handler: mux, + ReadHeaderTimeout: 5 * time.Second, + } + listener, err := net.Listen("tcp", server.Addr) + if err != nil { + return nil, err + } + + log.Println("Prometheus metrics available at http://127.0.0.1:9090/metrics") + go func() { + if err := server.Serve(listener); err != nil && !errors.Is(err, http.ErrServerClosed) { + log.Println("Prometheus endpoint failed:", err) + } + }() + return server, nil +} +``` + + +Opening the listener before starting the serving goroutine makes an address conflict fail during Worker startup. The +endpoint at [http://127.0.0.1:9090/metrics](http://127.0.0.1:9090/metrics) returns metrics in Prometheus format for an +external Prometheus server to scrape. + +### Run the automatic instrumentation sample + +From the root of a `samples-go` checkout, start Jaeger and the Worker: + +```bash +docker compose -f opentelemetry-v2/docker-compose.yaml up -d +go run opentelemetry-v2/automatic-instrumentation/worker/main.go +``` + +In another terminal, run the starter: + +```bash +go run opentelemetry-v2/automatic-instrumentation/starter/main.go +``` + +The trace begins with the Worker's automatic spans, since only the Worker installs the plugin in this sample. These +spans describe Temporal SDK operations that the Worker performs, such as running the Workflow and its Activity. + +Open the [Jaeger UI](http://127.0.0.1:16686), select `temporal-otel-v2-automatic-worker`, and inspect the Worker trace. Open +[http://127.0.0.1:9090/metrics](http://127.0.0.1:9090/metrics) to inspect the Temporal SDK metrics. + +## Add custom instrumentation + +By default, the plugin propagates context: spans your application creates remain connected across Clients, Workflows, +Activities, and other Temporal calls. The custom samples focus on that span propagation. + +### Propagate from a Workflow to an Activity + +Use `temporalotel.Tracer` to create replay-safe spans in Workflow code. Use an ordinary OpenTelemetry tracer in Activity +code. Passing the span-derived Workflow context to `workflow.ExecuteActivity` connects the Activity span to the Workflow +span. + + +[opentelemetry-v2/custom-instrumentation/workflow-activity-propagation/opentelemetry.go](https://github.com/temporalio/samples-go/blob/main/opentelemetry-v2/custom-instrumentation/workflow-activity-propagation/opentelemetry.go) +```go +const instrumentationName = "github.com/temporalio/samples-go/opentelemetry-v2/custom-instrumentation/workflow-activity-propagation" + +func Workflow(ctx workflow.Context, name string) (string, error) { + tracer := temporalotel.Tracer(instrumentationName) + ctx, span := tracer.Start(ctx, "workflow-operation") + defer span.End() + + ctx = workflow.WithActivityOptions(ctx, workflow.ActivityOptions{ + StartToCloseTimeout: 10 * time.Second, + }) + + var result string + if err := workflow.ExecuteActivity(ctx, Activity, name).Get(ctx, &result); err != nil { + return "", err + } + + return result, nil +} + +func Activity(ctx context.Context, name string) (string, error) { + _, span := otel.Tracer(instrumentationName).Start(ctx, "activity-operation") + defer span.End() + + return fmt.Sprintf("Hello, %s!", name), nil +} +``` + + +The Worker creates a default plugin so it can propagate the custom Workflow span to the Activity: + + +[opentelemetry-v2/custom-instrumentation/workflow-activity-propagation/worker/main.go](https://github.com/temporalio/samples-go/blob/main/opentelemetry-v2/custom-instrumentation/workflow-activity-propagation/worker/main.go) +```go +plugin, err := temporalotel.NewPlugin(temporalotel.PluginOptions{}) +if err != nil { + return fmt.Errorf("unable to create plugin: %w", err) +} + +c, err := client.Dial(client.Options{Plugins: []client.Plugin{plugin}}) +if err != nil { + return fmt.Errorf("unable to create client: %w", err) +} +defer c.Close() +``` + + +From the root of a `samples-go` checkout, start Jaeger and the Worker: + +```bash +docker compose -f opentelemetry-v2/docker-compose.yaml up -d +go run opentelemetry-v2/custom-instrumentation/workflow-activity-propagation/worker/main.go +``` + +In another terminal, run the starter: + +```bash +go run opentelemetry-v2/custom-instrumentation/workflow-activity-propagation/starter/main.go +``` + +Open the [Jaeger UI](http://127.0.0.1:16686), select +`temporal-otel-v2-custom-workflow-activity-propagation-worker`, and find the trace +containing the `workflow-operation` and `activity-operation` spans created by the sample. + +### Propagate from a Client to an Update + +Install the plugin in both processes when a custom client span must cross a Temporal call. The client-side plugin injects +the span context into the Update headers. The Worker plugin extracts it into the Update handler's Workflow context. + +Create the replay-safe tracer provider before the plugin in both processes, and add the default plugin when you create +each Temporal Client. Keep the `send-update` span open until the Update reaches the completed stage and returns its +result: + + +[opentelemetry-v2/custom-instrumentation/client-update-propagation/starter/main.go](https://github.com/temporalio/samples-go/blob/main/opentelemetry-v2/custom-instrumentation/client-update-propagation/starter/main.go) +```go +func sendUpdate( + ctx context.Context, + c client.Client, + workflowID string, + name string, +) (string, error) { + ctx, span := otel.Tracer(instrumentationName).Start(ctx, "send-update") + defer span.End() + + handle, err := c.UpdateWorkflow(ctx, client.UpdateWorkflowOptions{ + WorkflowID: workflowID, + UpdateName: clientupdate.UpdateName, + WaitForStage: client.WorkflowUpdateStageCompleted, + Args: []interface{}{name}, + }) + if err != nil { + return "", fmt.Errorf("unable to send Workflow Update: %w", err) + } + + var result string + if err := handle.Get(ctx, &result); err != nil { + return "", fmt.Errorf("unable to get Workflow Update result: %w", err) + } + return result, nil +} +``` + + +The Update validator creates a replay-safe `validate-update` span, and the handler creates a `handle-update` span, +changes the Workflow's completion state, and returns the result: + + +[opentelemetry-v2/custom-instrumentation/client-update-propagation/workflow.go](https://github.com/temporalio/samples-go/blob/main/opentelemetry-v2/custom-instrumentation/client-update-propagation/workflow.go) +```go +func Workflow(ctx workflow.Context) (string, error) { + var result string + updateCompleted := false + + err := workflow.SetUpdateHandlerWithOptions( + ctx, + UpdateName, + func(ctx workflow.Context, name string) (string, error) { + _, span := temporalotel.Tracer(instrumentationName).Start(ctx, "handle-update") + defer span.End() + + result = fmt.Sprintf("Hello, %s!", name) + updateCompleted = true + return result, nil + }, + workflow.UpdateHandlerOptions{ + Validator: func(ctx workflow.Context, name string) error { + _, span := temporalotel.Tracer(instrumentationName).Start(ctx, "validate-update") + defer span.End() + + if name == "" { + return fmt.Errorf("name cannot be empty") + } + return nil + }, + }, + ) + if err != nil { + return "", fmt.Errorf("unable to register Update handler: %w", err) + } + + if err := workflow.Await(ctx, func() bool { return updateCompleted && workflow.AllHandlersFinished(ctx) }); err != nil { + return "", err + } + return result, nil +} +``` + + +The Workflow waits for the Update to set its completion state and for `workflow.AllHandlersFinished` to report all +Update handlers done before returning. + +From the root of a `samples-go` checkout, start the Worker: + +```bash +go run opentelemetry-v2/custom-instrumentation/client-update-propagation/worker/main.go +``` + +In another terminal, run the starter: + +```bash +go run opentelemetry-v2/custom-instrumentation/client-update-propagation/starter/main.go +``` + +In Jaeger, select `temporal-otel-v2-custom-client-update-propagation-client` and open the trace containing `send-update`, +`validate-update`, and `handle-update`. The trace crosses into `temporal-otel-v2-custom-client-update-propagation-worker`, +connected by the context the default plugins propagate. + +## Shut down telemetry providers + +Install the tracer and meter providers as OpenTelemetry globals before constructing the plugin or a Workflow tracer, so +they're available to instrument every Temporal call the process makes. + +When the process exits, close the Temporal Client or stop the Worker first, then shut down the Prometheus-format +endpoint, meter provider, and tracer provider, each with its own fresh, bounded context rather than one that might +already be canceled. Shutting down the tracer provider flushes the batch span processor, so spans created earlier in +the process still reach the exporter before it exits. + +## Propagate trace context and baggage + +The plugin automatically propagates OpenTelemetry trace context and baggage across Temporal calls, using a composite +W3C Trace Context and Baggage propagator by default instead of the global OpenTelemetry propagator. Use +`PluginOptions.TextMapPropagator` to provide a different propagator. + +Temporal headers that carry baggage can be persisted in Workflow Event Histories. Do not put credentials, tokens, or +other sensitive data in OpenTelemetry baggage. Set `PluginOptions.DisableBaggage` to `true` to turn off baggage +propagation. + +For other application context, see [Context Propagation](/develop/go/best-practices/context-propagation). + +## Resources + +- [Automatic instrumentation sample](https://github.com/temporalio/samples-go/tree/main/opentelemetry-v2/automatic-instrumentation) + — spans the Worker creates for Temporal SDK operations, plus Temporal SDK metrics. +- [Workflow-to-Activity propagation sample](https://github.com/temporalio/samples-go/tree/main/opentelemetry-v2/custom-instrumentation/workflow-activity-propagation) + — custom spans propagated from a Workflow to an Activity. +- [Client-to-Update propagation sample](https://github.com/temporalio/samples-go/tree/main/opentelemetry-v2/custom-instrumentation/client-update-propagation) + — custom spans propagated from a Client into a Workflow Update. +- [OpenTelemetry v2 Go package](https://pkg.go.dev/go.temporal.io/sdk/contrib/opentelemetry-v2) — the full plugin + reference. +- [Go SDK observability guide](/develop/go/platform/observability) — where tracing and metrics fit among the SDK's + other observability tools. diff --git a/docs/develop/go/platform/observability.mdx b/docs/develop/go/platform/observability.mdx index 0a93e3f6c3..8d75cf6cea 100644 --- a/docs/develop/go/platform/observability.mdx +++ b/docs/develop/go/platform/observability.mdx @@ -41,6 +41,15 @@ This section covers features related to viewing the state of the application, in Each Temporal SDK is capable of emitting an optional set of metrics from either the Client or the Worker process. For a complete list of metrics capable of being emitted, see the [SDK metrics reference](/references/sdk-metrics). +:::tip[Use the OpenTelemetry v2 integration] + +New metrics setups should use the +[OpenTelemetry v2 integration](/develop/go/integrations/opentelemetry-v2#export-temporal-sdk-metrics) instead. It +configures Temporal SDK metrics through the Go SDK Plugin API and is currently in +[Pre-release](/evaluate/development-production-features/release-stages#pre-release). + +::: + - For an overview of Prometheus and Grafana integration, refer to the [Monitoring](/self-hosted-guide/monitoring) guide. - For a list of metrics, see the [SDK metrics reference](/references/sdk-metrics). - For an end-to-end example that exposes metrics with the Go SDK, refer to the [samples-go](https://github.com/temporalio/samples-go/tree/main/metrics) repo. @@ -91,6 +100,15 @@ Negative values can produce invalid or backend-dependent metric data when `UseMo Tracing allows you to view the call graph of a Workflow along with its Activities, Nexus Operations, and Child Workflows. +:::tip[Use the OpenTelemetry v2 integration] + +New tracing setups should use the +[OpenTelemetry v2 integration](/develop/go/integrations/opentelemetry-v2) instead. It uses the Go SDK Plugin API to +propagate OpenTelemetry trace context across Temporal calls and is currently in +[Pre-release](/evaluate/development-production-features/release-stages#pre-release). + +::: + The Go SDK provides tracing interceptors for [OpenTelemetry](https://pkg.go.dev/go.temporal.io/sdk/contrib/opentelemetry), [OpenTracing](https://pkg.go.dev/go.temporal.io/sdk/contrib/opentracing), and [Datadog](https://pkg.go.dev/go.temporal.io/sdk/contrib/datadog/tracing). First, create a tracing interceptor for Client instantiation. diff --git a/sidebars.js b/sidebars.js index 058c884560..6b6449aa5b 100644 --- a/sidebars.js +++ b/sidebars.js @@ -309,7 +309,10 @@ const developGoCategory = { type: 'doc', id: 'develop/go/integrations/index', }, - items: ['develop/go/integrations/google-adk'], + items: [ + 'develop/go/integrations/google-adk', + 'develop/go/integrations/opentelemetry-v2', + ], }, ], }; diff --git a/src/components/IntegrationsGrid/integrations-data.json b/src/components/IntegrationsGrid/integrations-data.json index ee7831a1e7..0dfe910d43 100644 --- a/src/components/IntegrationsGrid/integrations-data.json +++ b/src/components/IntegrationsGrid/integrations-data.json @@ -180,6 +180,15 @@ "sdk": "Python", "href": "https://docs.openbox.ai/getting-started/temporal" }, + { + "name": "OpenTelemetry v2", + "description": "Export tracing and metrics from Temporal Go SDK applications with OpenTelemetry.", + "tags": [ + "Observability" + ], + "sdk": "Go", + "href": "/develop/go/integrations/opentelemetry-v2" + }, { "name": "Parseable", "description": "Stream Temporal Workflow and Activity execution events to Parseable for observability and analysis.", From 0a97f04aa34c2ef2c11e0786c4ee163a4761af52 Mon Sep 17 00:00:00 2001 From: Patrik Beqo Date: Wed, 19 Aug 2026 15:28:08 -0400 Subject: [PATCH 02/11] fix: update docs with the new sample directory structure --- .../go/integrations/opentelemetry-v2.mdx | 34 +++++++++++-------- 1 file changed, 20 insertions(+), 14 deletions(-) diff --git a/docs/develop/go/integrations/opentelemetry-v2.mdx b/docs/develop/go/integrations/opentelemetry-v2.mdx index 76981d38b7..7f92eb70dc 100644 --- a/docs/develop/go/integrations/opentelemetry-v2.mdx +++ b/docs/develop/go/integrations/opentelemetry-v2.mdx @@ -192,10 +192,13 @@ docker compose -f opentelemetry-v2/docker-compose.yaml up -d go run opentelemetry-v2/automatic-instrumentation/worker/main.go ``` -In another terminal, run the starter: +In another terminal, start the Workflow: ```bash -go run opentelemetry-v2/automatic-instrumentation/starter/main.go +temporal workflow execute \ + --task-queue opentelemetry-v2 \ + --type Workflow \ + --input '"Temporal"' ``` The trace begins with the Worker's automatic spans, since only the Worker installs the plugin in this sample. These @@ -216,9 +219,9 @@ code. Passing the span-derived Workflow context to `workflow.ExecuteActivity` co span. -[opentelemetry-v2/custom-instrumentation/workflow-activity-propagation/opentelemetry.go](https://github.com/temporalio/samples-go/blob/main/opentelemetry-v2/custom-instrumentation/workflow-activity-propagation/opentelemetry.go) +[opentelemetry-v2/workflow-activity-propagation/opentelemetry.go](https://github.com/temporalio/samples-go/blob/main/opentelemetry-v2/workflow-activity-propagation/opentelemetry.go) ```go -const instrumentationName = "github.com/temporalio/samples-go/opentelemetry-v2/custom-instrumentation/workflow-activity-propagation" +const instrumentationName = "github.com/temporalio/samples-go/opentelemetry-v2/workflow-activity-propagation" func Workflow(ctx workflow.Context, name string) (string, error) { tracer := temporalotel.Tracer(instrumentationName) @@ -249,7 +252,7 @@ func Activity(ctx context.Context, name string) (string, error) { The Worker creates a default plugin so it can propagate the custom Workflow span to the Activity: -[opentelemetry-v2/custom-instrumentation/workflow-activity-propagation/worker/main.go](https://github.com/temporalio/samples-go/blob/main/opentelemetry-v2/custom-instrumentation/workflow-activity-propagation/worker/main.go) +[opentelemetry-v2/workflow-activity-propagation/worker/main.go](https://github.com/temporalio/samples-go/blob/main/opentelemetry-v2/workflow-activity-propagation/worker/main.go) ```go plugin, err := temporalotel.NewPlugin(temporalotel.PluginOptions{}) if err != nil { @@ -268,13 +271,16 @@ From the root of a `samples-go` checkout, start Jaeger and the Worker: ```bash docker compose -f opentelemetry-v2/docker-compose.yaml up -d -go run opentelemetry-v2/custom-instrumentation/workflow-activity-propagation/worker/main.go +go run opentelemetry-v2/workflow-activity-propagation/worker/main.go ``` -In another terminal, run the starter: +In another terminal, start the Workflow: ```bash -go run opentelemetry-v2/custom-instrumentation/workflow-activity-propagation/starter/main.go +temporal workflow execute \ + --task-queue opentelemetry-v2 \ + --type Workflow \ + --input '"Temporal"' ``` Open the [Jaeger UI](http://127.0.0.1:16686), select @@ -291,7 +297,7 @@ each Temporal Client. Keep the `send-update` span open until the Update reaches result: -[opentelemetry-v2/custom-instrumentation/client-update-propagation/starter/main.go](https://github.com/temporalio/samples-go/blob/main/opentelemetry-v2/custom-instrumentation/client-update-propagation/starter/main.go) +[opentelemetry-v2/client-update-propagation/starter/main.go](https://github.com/temporalio/samples-go/blob/main/opentelemetry-v2/client-update-propagation/starter/main.go) ```go func sendUpdate( ctx context.Context, @@ -325,7 +331,7 @@ The Update validator creates a replay-safe `validate-update` span, and the handl changes the Workflow's completion state, and returns the result: -[opentelemetry-v2/custom-instrumentation/client-update-propagation/workflow.go](https://github.com/temporalio/samples-go/blob/main/opentelemetry-v2/custom-instrumentation/client-update-propagation/workflow.go) +[opentelemetry-v2/client-update-propagation/workflow.go](https://github.com/temporalio/samples-go/blob/main/opentelemetry-v2/client-update-propagation/workflow.go) ```go func Workflow(ctx workflow.Context) (string, error) { var result string @@ -372,13 +378,13 @@ Update handlers done before returning. From the root of a `samples-go` checkout, start the Worker: ```bash -go run opentelemetry-v2/custom-instrumentation/client-update-propagation/worker/main.go +go run opentelemetry-v2/client-update-propagation/worker/main.go ``` In another terminal, run the starter: ```bash -go run opentelemetry-v2/custom-instrumentation/client-update-propagation/starter/main.go +go run opentelemetry-v2/client-update-propagation/starter/main.go ``` In Jaeger, select `temporal-otel-v2-custom-client-update-propagation-client` and open the trace containing `send-update`, @@ -411,9 +417,9 @@ For other application context, see [Context Propagation](/develop/go/best-practi - [Automatic instrumentation sample](https://github.com/temporalio/samples-go/tree/main/opentelemetry-v2/automatic-instrumentation) — spans the Worker creates for Temporal SDK operations, plus Temporal SDK metrics. -- [Workflow-to-Activity propagation sample](https://github.com/temporalio/samples-go/tree/main/opentelemetry-v2/custom-instrumentation/workflow-activity-propagation) +- [Workflow-to-Activity propagation sample](https://github.com/temporalio/samples-go/tree/main/opentelemetry-v2/workflow-activity-propagation) — custom spans propagated from a Workflow to an Activity. -- [Client-to-Update propagation sample](https://github.com/temporalio/samples-go/tree/main/opentelemetry-v2/custom-instrumentation/client-update-propagation) +- [Client-to-Update propagation sample](https://github.com/temporalio/samples-go/tree/main/opentelemetry-v2/client-update-propagation) — custom spans propagated from a Client into a Workflow Update. - [OpenTelemetry v2 Go package](https://pkg.go.dev/go.temporal.io/sdk/contrib/opentelemetry-v2) — the full plugin reference. From d7be3bc929e4a5db873cf2c09aaa193d6a3078cb Mon Sep 17 00:00:00 2001 From: Patrik Beqo Date: Wed, 19 Aug 2026 15:32:44 -0400 Subject: [PATCH 03/11] fix: task queue name --- docs/develop/go/integrations/opentelemetry-v2.mdx | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/develop/go/integrations/opentelemetry-v2.mdx b/docs/develop/go/integrations/opentelemetry-v2.mdx index 7f92eb70dc..c81e3a6f42 100644 --- a/docs/develop/go/integrations/opentelemetry-v2.mdx +++ b/docs/develop/go/integrations/opentelemetry-v2.mdx @@ -196,7 +196,7 @@ In another terminal, start the Workflow: ```bash temporal workflow execute \ - --task-queue opentelemetry-v2 \ + --task-queue automatic-instrumentation \ --type Workflow \ --input '"Temporal"' ``` @@ -278,7 +278,7 @@ In another terminal, start the Workflow: ```bash temporal workflow execute \ - --task-queue opentelemetry-v2 \ + --task-queue workflow-activity-propagation \ --type Workflow \ --input '"Temporal"' ``` From ea02cd9a25909c172ad33e0caa14de43f14afbfe Mon Sep 17 00:00:00 2001 From: Patrik Beqo Date: Wed, 19 Aug 2026 16:00:51 -0400 Subject: [PATCH 04/11] fix: address copilot comments --- docs/develop/go/best-practices/context-propagation.mdx | 2 +- docs/develop/go/integrations/opentelemetry-v2.mdx | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/docs/develop/go/best-practices/context-propagation.mdx b/docs/develop/go/best-practices/context-propagation.mdx index c74a6d701b..643cfffcfa 100644 --- a/docs/develop/go/best-practices/context-propagation.mdx +++ b/docs/develop/go/best-practices/context-propagation.mdx @@ -16,7 +16,7 @@ tags: description: How to propagate custom key-value data across Workflow, Activity, and Child Workflow boundaries using the Temporal Go SDK. --- -Context propagation lets you pass custom key-value data from a Client to Workflows, and from Workflows to Activities and Child Workflows, without threading it through every function signature. Common use cases include propagating tenant IDs, auth tokens, or other request-scoped metadata. +Context propagation lets you pass custom key-value data from a Client to Workflows, and from Workflows to Activities and Child Workflows, without threading it through every function signature. Common use cases include propagating tenant identifiers, auth tokens, or other request-scoped metadata. {/* TODO: Link to /encyclopedia/context-propagation once that page lands */} diff --git a/docs/develop/go/integrations/opentelemetry-v2.mdx b/docs/develop/go/integrations/opentelemetry-v2.mdx index c81e3a6f42..16329fdb8c 100644 --- a/docs/develop/go/integrations/opentelemetry-v2.mdx +++ b/docs/develop/go/integrations/opentelemetry-v2.mdx @@ -375,9 +375,10 @@ func Workflow(ctx workflow.Context) (string, error) { The Workflow waits for the Update to set its completion state and for `workflow.AllHandlersFinished` to report all Update handlers done before returning. -From the root of a `samples-go` checkout, start the Worker: +From the root of a `samples-go` checkout, start Jaeger and the Worker: ```bash +docker compose -f opentelemetry-v2/docker-compose.yaml up -d go run opentelemetry-v2/client-update-propagation/worker/main.go ``` From 3f2c0163be6339bc81cd451ec36f325bafc6fbeb Mon Sep 17 00:00:00 2001 From: Patrik Beqo Date: Thu, 20 Aug 2026 09:26:26 -0400 Subject: [PATCH 05/11] fix: title --- docs/develop/go/platform/observability.mdx | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/docs/develop/go/platform/observability.mdx b/docs/develop/go/platform/observability.mdx index 8d75cf6cea..9d07e7fb5b 100644 --- a/docs/develop/go/platform/observability.mdx +++ b/docs/develop/go/platform/observability.mdx @@ -43,8 +43,8 @@ For a complete list of metrics capable of being emitted, see the [SDK metrics re :::tip[Use the OpenTelemetry v2 integration] -New metrics setups should use the -[OpenTelemetry v2 integration](/develop/go/integrations/opentelemetry-v2#export-temporal-sdk-metrics) instead. It +For new metrics setups that use OpenTelemetry, use the +[OpenTelemetry v2 integration](/develop/go/integrations/opentelemetry-v2#export-temporal-sdk-metrics). It configures Temporal SDK metrics through the Go SDK Plugin API and is currently in [Pre-release](/evaluate/development-production-features/release-stages#pre-release). @@ -102,8 +102,8 @@ Tracing allows you to view the call graph of a Workflow along with its Activitie :::tip[Use the OpenTelemetry v2 integration] -New tracing setups should use the -[OpenTelemetry v2 integration](/develop/go/integrations/opentelemetry-v2) instead. It uses the Go SDK Plugin API to +For new tracing setups that use OpenTelemetry, use the +[OpenTelemetry v2 integration](/develop/go/integrations/opentelemetry-v2). It uses the Go SDK Plugin API to propagate OpenTelemetry trace context across Temporal calls and is currently in [Pre-release](/evaluate/development-production-features/release-stages#pre-release). From e4d3fa5623cc8ad9b346bd43a2dd8a23fd247698 Mon Sep 17 00:00:00 2001 From: Patrik Beqo Date: Thu, 20 Aug 2026 10:26:03 -0400 Subject: [PATCH 06/11] chore: sync OpenTelemetry v2 snippets --- docs/develop/go/integrations/opentelemetry-v2.mdx | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/docs/develop/go/integrations/opentelemetry-v2.mdx b/docs/develop/go/integrations/opentelemetry-v2.mdx index 16329fdb8c..11732a133e 100644 --- a/docs/develop/go/integrations/opentelemetry-v2.mdx +++ b/docs/develop/go/integrations/opentelemetry-v2.mdx @@ -88,6 +88,7 @@ func InitializeGlobalTracerProvider( otel.SetTracerProvider(provider) return provider, nil } + ``` @@ -146,6 +147,7 @@ func InitializeGlobalMeterProvider(serviceName string) (*sdkmetric.MeterProvider otel.SetMeterProvider(provider) return provider, nil } + ``` @@ -176,6 +178,7 @@ func StartPrometheusEndpoint() (*http.Server, error) { }() return server, nil } + ``` @@ -246,6 +249,7 @@ func Activity(ctx context.Context, name string) (string, error) { return fmt.Sprintf("Hello, %s!", name), nil } + ``` @@ -324,6 +328,7 @@ func sendUpdate( } return result, nil } + ``` @@ -369,6 +374,7 @@ func Workflow(ctx workflow.Context) (string, error) { } return result, nil } + ``` From c6496cf14bbb3d48a8a6b1f616585af3e75427f2 Mon Sep 17 00:00:00 2001 From: flippedcoder Date: Thu, 20 Aug 2026 13:26:00 -0500 Subject: [PATCH 07/11] chore(otel): updated the content for docs --- .../go/integrations/opentelemetry-v2.mdx | 47 ++++++++++--------- 1 file changed, 25 insertions(+), 22 deletions(-) diff --git a/docs/develop/go/integrations/opentelemetry-v2.mdx b/docs/develop/go/integrations/opentelemetry-v2.mdx index 11732a133e..190a107d96 100644 --- a/docs/develop/go/integrations/opentelemetry-v2.mdx +++ b/docs/develop/go/integrations/opentelemetry-v2.mdx @@ -21,15 +21,12 @@ opt in to Temporal SDK operation spans and SDK metrics. This guide demonstrates two approaches: -- **Automatic instrumentation:** The Worker plugin creates spans for Temporal SDK operations and reports Temporal SDK +- [Automatic instrumentation](#enable-automatic-instrumentation): The Worker plugin creates spans for Temporal SDK operations and reports Temporal SDK metrics. -- **Custom instrumentation:** The plugin propagates trace context, but you create the spans that describe your +- [Custom instrumentation](#add-custom-instrumentation): The plugin propagates trace context, but you create the spans that describe your application. -This guide assumes that you understand OpenTelemetry fundamentals. For OpenTelemetry concepts and exporter options, -refer to the [OpenTelemetry documentation](https://opentelemetry.io/docs/). - -Code snippets in this guide are taken from the +For OpenTelemetry concepts and exporter options, refer to the [OpenTelemetry documentation](https://opentelemetry.io/docs/). Code snippets in this guide are taken from the [OpenTelemetry v2 sample](https://github.com/temporalio/samples-go/tree/main/opentelemetry-v2). Refer to the sample for the complete, runnable code. @@ -58,7 +55,7 @@ go get github.com/prometheus/client_golang@latest ## Configure tracing -Create and install a replay-safe tracer provider in every process that creates the plugin or calls +Create and install a `ReplaySafeTracerProvider` in every process that creates the plugin or calls `temporalotel.Tracer`. The replay-safe provider gives Workflow spans consistent identities when Workflow code replays. @@ -95,13 +92,13 @@ func InitializeGlobalTracerProvider( The samples use distinct OpenTelemetry service names for each instrumented process. This makes process boundaries clear when one trace contains spans from both a Client and a Worker. -The OTLP exporter uses an insecure loopback connection for this local example. Configure transport security for your +The OTLP exporter uses an insecure loopback connection for this example. Configure transport security for your production telemetry backend. ## Enable automatic instrumentation Set `AddTemporalSpans` to `true` to have the plugin create spans for Temporal SDK operations performed by the Worker. -Provide `MetricsHandlerOptions` to install the plugin's Temporal SDK metrics handler. A non-nil metrics configuration +Provide `MetricsHandlerOptions` to install the plugin's Temporal SDK metrics handler. A non-`nil` metrics configuration enables the complete SDK metric set. @@ -183,7 +180,7 @@ func StartPrometheusEndpoint() (*http.Server, error) { Opening the listener before starting the serving goroutine makes an address conflict fail during Worker startup. The -endpoint at [http://127.0.0.1:9090/metrics](http://127.0.0.1:9090/metrics) returns metrics in Prometheus format for an +endpoint at `http://127.0.0.1:9090/metrics` returns metrics in Prometheus format for an external Prometheus server to scrape. ### Run the automatic instrumentation sample @@ -207,8 +204,8 @@ temporal workflow execute \ The trace begins with the Worker's automatic spans, since only the Worker installs the plugin in this sample. These spans describe Temporal SDK operations that the Worker performs, such as running the Workflow and its Activity. -Open the [Jaeger UI](http://127.0.0.1:16686), select `temporal-otel-v2-automatic-worker`, and inspect the Worker trace. Open -[http://127.0.0.1:9090/metrics](http://127.0.0.1:9090/metrics) to inspect the Temporal SDK metrics. +Open your Jaeger UI, select `temporal-otel-v2-automatic-worker`, and inspect the Worker trace. Open +`http://127.0.0.1:9090/metrics` to inspect the Temporal SDK metrics. ## Add custom instrumentation @@ -271,7 +268,7 @@ defer c.Close() ``` -From the root of a `samples-go` checkout, start Jaeger and the Worker: +From the root of the `samples-go` repo, start Jaeger and the Worker: ```bash docker compose -f opentelemetry-v2/docker-compose.yaml up -d @@ -287,7 +284,7 @@ temporal workflow execute \ --input '"Temporal"' ``` -Open the [Jaeger UI](http://127.0.0.1:16686), select +Open the Jaeger UI, select `temporal-otel-v2-custom-workflow-activity-propagation-worker`, and find the trace containing the `workflow-operation` and `activity-operation` spans created by the sample. @@ -332,7 +329,7 @@ func sendUpdate( ``` -The Update validator creates a replay-safe `validate-update` span, and the handler creates a `handle-update` span, +The Update validator creates a replay-safe `validate-update` span. The handler creates a `handle-update` span, changes the Workflow's completion state, and returns the result: @@ -381,7 +378,7 @@ func Workflow(ctx workflow.Context) (string, error) { The Workflow waits for the Update to set its completion state and for `workflow.AllHandlersFinished` to report all Update handlers done before returning. -From the root of a `samples-go` checkout, start Jaeger and the Worker: +From the root of the `samples-go` repo, start Jaeger and the Worker: ```bash docker compose -f opentelemetry-v2/docker-compose.yaml up -d @@ -394,7 +391,7 @@ In another terminal, run the starter: go run opentelemetry-v2/client-update-propagation/starter/main.go ``` -In Jaeger, select `temporal-otel-v2-custom-client-update-propagation-client` and open the trace containing `send-update`, +In the Jaeger UI, select `temporal-otel-v2-custom-client-update-propagation-client` and open the trace containing `send-update`, `validate-update`, and `handle-update`. The trace crosses into `temporal-otel-v2-custom-client-update-propagation-worker`, connected by the context the default plugins propagate. @@ -403,10 +400,16 @@ connected by the context the default plugins propagate. Install the tracer and meter providers as OpenTelemetry globals before constructing the plugin or a Workflow tracer, so they're available to instrument every Temporal call the process makes. -When the process exits, close the Temporal Client or stop the Worker first, then shut down the Prometheus-format -endpoint, meter provider, and tracer provider, each with its own fresh, bounded context rather than one that might -already be canceled. Shutting down the tracer provider flushes the batch span processor, so spans created earlier in -the process still reach the exporter before it exits. +Shut things down in this order when the process exits: + +1. Close the Temporal Client or stop the Worker. +2. Shut down the Prometheus-format endpoint. +3. Shut down the meter provider. +4. Shut down the tracer provider. + +Pass each shutdown call its own new context with a timeout. A context left over from the rest of the program might +already be canceled, which cuts the shutdown short. Shutting down the tracer provider flushes the batch span processor, +so spans created earlier still reach the exporter before the process exits. ## Propagate trace context and baggage @@ -414,7 +417,7 @@ The plugin automatically propagates OpenTelemetry trace context and baggage acro W3C Trace Context and Baggage propagator by default instead of the global OpenTelemetry propagator. Use `PluginOptions.TextMapPropagator` to provide a different propagator. -Temporal headers that carry baggage can be persisted in Workflow Event Histories. Do not put credentials, tokens, or +Temporal headers that carry baggage can be persisted in Event Histories. Do not put credentials, tokens, or other sensitive data in OpenTelemetry baggage. Set `PluginOptions.DisableBaggage` to `true` to turn off baggage propagation. From e1328a09907139a6eb8b9f62ae15ec039d0789c4 Mon Sep 17 00:00:00 2001 From: Patrik Beqo Date: Fri, 21 Aug 2026 10:17:54 -0400 Subject: [PATCH 08/11] feat: improve docs --- .../go/best-practices/context-propagation.mdx | 8 +- .../go/integrations/opentelemetry-v2.mdx | 450 +++++------------- docs/develop/go/platform/observability.mdx | 20 +- 3 files changed, 140 insertions(+), 338 deletions(-) diff --git a/docs/develop/go/best-practices/context-propagation.mdx b/docs/develop/go/best-practices/context-propagation.mdx index 643cfffcfa..56214658ef 100644 --- a/docs/develop/go/best-practices/context-propagation.mdx +++ b/docs/develop/go/best-practices/context-propagation.mdx @@ -18,13 +18,11 @@ description: How to propagate custom key-value data across Workflow, Activity, a Context propagation lets you pass custom key-value data from a Client to Workflows, and from Workflows to Activities and Child Workflows, without threading it through every function signature. Common use cases include propagating tenant identifiers, auth tokens, or other request-scoped metadata. -{/* TODO: Link to /encyclopedia/context-propagation once that page lands */} - :::tip -For OpenTelemetry tracing, use the [OpenTelemetry v2 integration](/develop/go/integrations/opentelemetry-v2). The plugin -handles OpenTelemetry trace-context propagation automatically, so you do not need a custom Temporal context propagator -for trace identifiers. +If you want to propagate tracing context, the Go SDK provides tracing integrations that handle propagation for you. +[Choose a tracing integration](/develop/go/platform/observability#tracing) before implementing a custom context +propagator. ::: diff --git a/docs/develop/go/integrations/opentelemetry-v2.mdx b/docs/develop/go/integrations/opentelemetry-v2.mdx index 190a107d96..262b68648f 100644 --- a/docs/develop/go/integrations/opentelemetry-v2.mdx +++ b/docs/develop/go/integrations/opentelemetry-v2.mdx @@ -3,7 +3,7 @@ id: opentelemetry-v2 title: OpenTelemetry v2 integration sidebar_label: OpenTelemetry v2 toc_max_heading_level: 3 -description: Configure custom tracing or automatic tracing and metrics for a Temporal Application with the Go SDK OpenTelemetry v2 plugin. +description: Configure trace propagation, automatic tracing, custom tracing, and metrics with the Go SDK OpenTelemetry v2 plugin. tags: - Go SDK - Temporal SDKs @@ -13,67 +13,47 @@ tags: import { ReleaseNoteHeader } from '@site/src/components'; -The OpenTelemetry v2 integration connects the Temporal Go SDK to OpenTelemetry through the -[Plugin API](/develop/plugins-guide). Use it to propagate application trace context across Temporal calls. You can also -opt in to Temporal SDK operation spans and SDK metrics. +Temporal's OpenTelemetry integration lets you understand the internal state +of Temporal applications across Clients, Workflows, Activities, and Nexus +Operations by instrumenting them with +[OpenTelemetry](https://opentelemetry.io/docs/what-is-opentelemetry/). - - -This guide demonstrates two approaches: - -- [Automatic instrumentation](#enable-automatic-instrumentation): The Worker plugin creates spans for Temporal SDK operations and reports Temporal SDK - metrics. -- [Custom instrumentation](#add-custom-instrumentation): The plugin propagates trace context, but you create the spans that describe your - application. +Temporal provides [durable execution](/temporal#durable-execution). OpenTelemetry +is the vendor-neutral framework for generating and exporting telemetry +to your backend. -For OpenTelemetry concepts and exporter options, refer to the [OpenTelemetry documentation](https://opentelemetry.io/docs/). Code snippets in this guide are taken from the -[OpenTelemetry v2 sample](https://github.com/temporalio/samples-go/tree/main/opentelemetry-v2). Refer to the sample for -the complete, runnable code. +The OpenTelemetry plugin is what connects the two. It propagates OpenTelemetry +context across Temporal boundaries. It can also create spans and emit metrics +for Temporal SDK operations. -## Prerequisites + -- Set up your local development environment by following - [Set up your local development environment](/develop/go/set-up-your-local-go). -- Leave the Temporal development server running to test the sample locally. -- Install Docker to run Jaeger for the local tracing examples. +All code snippets in this guide are taken from the +[OpenTelemetry v2 sample](https://github.com/temporalio/samples-go/tree/main/opentelemetry-v2). +Refer to the sample for complete code. -## Install the plugin +## Install -Install the OpenTelemetry v2 plugin: +Add the OpenTelemetry v2 integration to your Go module: ```bash go get go.temporal.io/sdk/contrib/opentelemetry-v2@latest ``` -The samples also use the OpenTelemetry OTLP gRPC and Prometheus exporters: +Also add the OpenTelemetry SDK packages and the exporter or metric reader your +backend requires. -```bash -go get go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc@latest -go get go.opentelemetry.io/otel/exporters/prometheus@latest -go get github.com/prometheus/client_golang@latest -``` - -## Configure tracing +## Set up the tracer provider -Create and install a `ReplaySafeTracerProvider` in every process that creates the plugin or calls -`temporalotel.Tracer`. The replay-safe provider gives Workflow spans consistent identities when Workflow code replays. +A [Tracer Provider](https://opentelemetry.io/docs/concepts/signals/traces/#tracer-provider) +is the factory for Tracers. Create Temporal's replay-safe Tracer Provider and +install it as the OpenTelemetry global before you create the plugin or call +`Tracer`: - + [opentelemetry-v2/setup.go](https://github.com/temporalio/samples-go/blob/main/opentelemetry-v2/setup.go) ```go -func InitializeGlobalTracerProvider( - ctx context.Context, - serviceName string, -) (*temporalotel.ReplaySafeTracerProvider, error) { - exporter, err := otlptracegrpc.New( - ctx, - otlptracegrpc.WithEndpoint("127.0.0.1:4317"), - otlptracegrpc.WithInsecure(), - ) - if err != nil { - return nil, err - } - +// ... provider := temporalotel.NewReplaySafeTracerProvider( // WithBatcher performs exporter I/O outside the Workflow goroutine. sdktrace.WithBatcher(exporter), @@ -83,146 +63,62 @@ func InitializeGlobalTracerProvider( )), ) otel.SetTracerProvider(provider) - return provider, nil -} - ``` -The samples use distinct OpenTelemetry service names for each instrumented process. This makes process boundaries clear -when one trace contains spans from both a Client and a Worker. +`NewReplaySafeTracerProvider` keeps span IDs stable across retries and replay when +instrumenting Workflows. A standard OpenTelemetry Tracer Provider is not safe for +creating spans in Workflows. -The OTLP exporter uses an insecure loopback connection for this example. Configure transport security for your -production telemetry backend. +Your application owns the Tracer Provider for the life of the process. Shut it +down before exit so remaining spans can flush through the +[trace exporter](https://opentelemetry.io/docs/concepts/signals/traces/#trace-exporters). -## Enable automatic instrumentation +## Add the plugin -Set `AddTemporalSpans` to `true` to have the plugin create spans for Temporal SDK operations performed by the Worker. -Provide `MetricsHandlerOptions` to install the plugin's Temporal SDK metrics handler. A non-`nil` metrics configuration -enables the complete SDK metric set. +Pass the plugin to your Temporal Client when you create it. Workers made from +that Client get the plugin automatically. - -[opentelemetry-v2/automatic-instrumentation/worker/main.go](https://github.com/temporalio/samples-go/blob/main/opentelemetry-v2/automatic-instrumentation/worker/main.go) + +[opentelemetry-v2/workflow-activity-propagation/worker/main.go](https://github.com/temporalio/samples-go/blob/main/opentelemetry-v2/workflow-activity-propagation/worker/main.go) ```go -plugin, err := temporalotel.NewPlugin(temporalotel.PluginOptions{ - TracerOptions: tracing.TracerOptions{ - AddTemporalSpans: true, - }, - MetricsHandlerOptions: &temporalotel.MetricsHandlerOptions{ - UseMonotonicCounters: true, - }, -}) +plugin, err := temporalotel.NewPlugin(temporalotel.PluginOptions{}) if err != nil { return fmt.Errorf("unable to create plugin: %w", err) } -``` - - -`UseMonotonicCounters` represents Temporal SDK counters as OpenTelemetry `Int64Counter` instruments instead of -`Int64UpDownCounter` instruments. - -### Export Temporal SDK metrics - -Create and install a meter provider backed by the OpenTelemetry Prometheus exporter: - - -[opentelemetry-v2/setup.go](https://github.com/temporalio/samples-go/blob/main/opentelemetry-v2/setup.go) -```go -func InitializeGlobalMeterProvider(serviceName string) (*sdkmetric.MeterProvider, error) { - exporter, err := otelprometheus.New() - if err != nil { - return nil, err - } - provider := sdkmetric.NewMeterProvider( - sdkmetric.WithReader(exporter), - sdkmetric.WithResource(resource.NewWithAttributes( - semconv.SchemaURL, - semconv.ServiceName(serviceName), - )), - ) - otel.SetMeterProvider(provider) - return provider, nil -} - -``` - - -Start a loopback-only HTTP endpoint that serves the Prometheus handler: - - -[opentelemetry-v2/setup.go](https://github.com/temporalio/samples-go/blob/main/opentelemetry-v2/setup.go) -```go -func StartPrometheusEndpoint() (*http.Server, error) { - mux := http.NewServeMux() - mux.Handle("/metrics", promhttp.Handler()) - - server := &http.Server{ - Addr: "127.0.0.1:9090", - Handler: mux, - ReadHeaderTimeout: 5 * time.Second, - } - listener, err := net.Listen("tcp", server.Addr) - if err != nil { - return nil, err - } - - log.Println("Prometheus metrics available at http://127.0.0.1:9090/metrics") - go func() { - if err := server.Serve(listener); err != nil && !errors.Is(err, http.ErrServerClosed) { - log.Println("Prometheus endpoint failed:", err) - } - }() - return server, nil +c, err := client.Dial(client.Options{Plugins: []client.Plugin{plugin}}) +if err != nil { + return fmt.Errorf("unable to create client: %w", err) } - +defer c.Close() ``` -Opening the listener before starting the serving goroutine makes an address conflict fail during Worker startup. The -endpoint at `http://127.0.0.1:9090/metrics` returns metrics in Prometheus format for an -external Prometheus server to scrape. +By default the plugin only performs +[context propagation](https://opentelemetry.io/docs/concepts/context-propagation/) +so [Span Context](https://opentelemetry.io/docs/concepts/signals/traces/#span-context) +can cross Temporal boundaries. -### Run the automatic instrumentation sample +## Add custom spans -From the root of a `samples-go` checkout, start Jaeger and the Worker: +### In Workflows -```bash -docker compose -f opentelemetry-v2/docker-compose.yaml up -d -go run opentelemetry-v2/automatic-instrumentation/worker/main.go -``` - -In another terminal, start the Workflow: - -```bash -temporal workflow execute \ - --task-queue automatic-instrumentation \ - --type Workflow \ - --input '"Temporal"' -``` +A [Tracer](https://opentelemetry.io/docs/concepts/signals/traces/#tracer) +creates spans. In Workflows, use `Tracer` instead of `otel.Tracer`. It keeps +span IDs and start times accurate across retries and replay. A standard +OpenTelemetry Tracer is not safe for creating spans in Workflows. -The trace begins with the Worker's automatic spans, since only the Worker installs the plugin in this sample. These -spans describe Temporal SDK operations that the Worker performs, such as running the Workflow and its Activity. +As in +[OpenTelemetry Go](https://opentelemetry.io/docs/languages/go/instrumentation/), +`Start` returns a context that contains the active span. Pass that +`workflow.Context` to downstream Temporal calls so later spans nest under it as +children: -Open your Jaeger UI, select `temporal-otel-v2-automatic-worker`, and inspect the Worker trace. Open -`http://127.0.0.1:9090/metrics` to inspect the Temporal SDK metrics. - -## Add custom instrumentation - -By default, the plugin propagates context: spans your application creates remain connected across Clients, Workflows, -Activities, and other Temporal calls. The custom samples focus on that span propagation. - -### Propagate from a Workflow to an Activity - -Use `temporalotel.Tracer` to create replay-safe spans in Workflow code. Use an ordinary OpenTelemetry tracer in Activity -code. Passing the span-derived Workflow context to `workflow.ExecuteActivity` connects the Activity span to the Workflow -span. - - + [opentelemetry-v2/workflow-activity-propagation/opentelemetry.go](https://github.com/temporalio/samples-go/blob/main/opentelemetry-v2/workflow-activity-propagation/opentelemetry.go) ```go -const instrumentationName = "github.com/temporalio/samples-go/opentelemetry-v2/workflow-activity-propagation" - +// ... func Workflow(ctx workflow.Context, name string) (string, error) { tracer := temporalotel.Tracer(instrumentationName) ctx, span := tracer.Start(ctx, "workflow-operation") @@ -239,199 +135,107 @@ func Workflow(ctx workflow.Context, name string) (string, error) { return result, nil } +``` + +### Outside Workflows + +In Clients, Activities, and other non-Workflow code, use an ordinary OpenTelemetry +[Tracer](https://opentelemetry.io/docs/concepts/signals/traces/#tracer): + + +[opentelemetry-v2/workflow-activity-propagation/opentelemetry.go](https://github.com/temporalio/samples-go/blob/main/opentelemetry-v2/workflow-activity-propagation/opentelemetry.go) +```go +// ... func Activity(ctx context.Context, name string) (string, error) { _, span := otel.Tracer(instrumentationName).Start(ctx, "activity-operation") defer span.End() return fmt.Sprintf("Hello, %s!", name), nil } - ``` -The Worker creates a default plugin so it can propagate the custom Workflow span to the Activity: +## Enable automatic instrumentation - -[opentelemetry-v2/workflow-activity-propagation/worker/main.go](https://github.com/temporalio/samples-go/blob/main/opentelemetry-v2/workflow-activity-propagation/worker/main.go) +Set options on `PluginOptions` to create spans and emit metrics for Temporal +SDK operations: + + +[opentelemetry-v2/automatic-instrumentation/worker/main.go](https://github.com/temporalio/samples-go/blob/main/opentelemetry-v2/automatic-instrumentation/worker/main.go) ```go -plugin, err := temporalotel.NewPlugin(temporalotel.PluginOptions{}) +plugin, err := temporalotel.NewPlugin(temporalotel.PluginOptions{ + TracerOptions: tracing.TracerOptions{ + AddTemporalSpans: true, + }, + MetricsHandlerOptions: &temporalotel.MetricsHandlerOptions{ + UseMonotonicCounters: true, + }, +}) if err != nil { return fmt.Errorf("unable to create plugin: %w", err) } - -c, err := client.Dial(client.Options{Plugins: []client.Plugin{plugin}}) -if err != nil { - return fmt.Errorf("unable to create client: %w", err) -} -defer c.Close() ``` -From the root of the `samples-go` repo, start Jaeger and the Worker: +### `AddTemporalSpans` -```bash -docker compose -f opentelemetry-v2/docker-compose.yaml up -d -go run opentelemetry-v2/workflow-activity-propagation/worker/main.go -``` - -In another terminal, start the Workflow: - -```bash -temporal workflow execute \ - --task-queue workflow-activity-propagation \ - --type Workflow \ - --input '"Temporal"' -``` - -Open the Jaeger UI, select -`temporal-otel-v2-custom-workflow-activity-propagation-worker`, and find the trace -containing the `workflow-operation` and `activity-operation` spans created by the sample. +Set `AddTemporalSpans` to `true` to create spans for Temporal SDK operations +across Clients, Workflows, Activities, and Nexus Operations. -### Propagate from a Client to an Update +### `MetricsHandlerOptions` -Install the plugin in both processes when a custom client span must cross a Temporal call. The client-side plugin injects -the span context into the Update headers. The Worker plugin extracts it into the Update handler's Workflow context. +Set `MetricsHandlerOptions` to a non-`nil` value to emit +[Temporal SDK metrics](/references/sdk-metrics) through OpenTelemetry. +`UseMonotonicCounters` controls whether counters are monotonic. -Create the replay-safe tracer provider before the plugin in both processes, and add the default plugin when you create -each Temporal Client. Keep the `send-update` span open until the Update reaches the completed stage and returns its -result: +By default the handler uses a Meter from the global +[Meter Provider](https://opentelemetry.io/docs/concepts/signals/metrics/#meter-provider). +Set `MetricsHandlerOptions.Meter` to use a specific Meter. - -[opentelemetry-v2/client-update-propagation/starter/main.go](https://github.com/temporalio/samples-go/blob/main/opentelemetry-v2/client-update-propagation/starter/main.go) -```go -func sendUpdate( - ctx context.Context, - c client.Client, - workflowID string, - name string, -) (string, error) { - ctx, span := otel.Tracer(instrumentationName).Start(ctx, "send-update") - defer span.End() - - handle, err := c.UpdateWorkflow(ctx, client.UpdateWorkflowOptions{ - WorkflowID: workflowID, - UpdateName: clientupdate.UpdateName, - WaitForStage: client.WorkflowUpdateStageCompleted, - Args: []interface{}{name}, - }) - if err != nil { - return "", fmt.Errorf("unable to send Workflow Update: %w", err) - } - - var result string - if err := handle.Get(ctx, &result); err != nil { - return "", fmt.Errorf("unable to get Workflow Update result: %w", err) - } - return result, nil -} - -``` - - -The Update validator creates a replay-safe `validate-update` span. The handler creates a `handle-update` span, -changes the Workflow's completion state, and returns the result: - - -[opentelemetry-v2/client-update-propagation/workflow.go](https://github.com/temporalio/samples-go/blob/main/opentelemetry-v2/client-update-propagation/workflow.go) -```go -func Workflow(ctx workflow.Context) (string, error) { - var result string - updateCompleted := false - - err := workflow.SetUpdateHandlerWithOptions( - ctx, - UpdateName, - func(ctx workflow.Context, name string) (string, error) { - _, span := temporalotel.Tracer(instrumentationName).Start(ctx, "handle-update") - defer span.End() - - result = fmt.Sprintf("Hello, %s!", name) - updateCompleted = true - return result, nil - }, - workflow.UpdateHandlerOptions{ - Validator: func(ctx workflow.Context, name string) error { - _, span := temporalotel.Tracer(instrumentationName).Start(ctx, "validate-update") - defer span.End() - - if name == "" { - return fmt.Errorf("name cannot be empty") - } - return nil - }, - }, - ) - if err != nil { - return "", fmt.Errorf("unable to register Update handler: %w", err) - } - - if err := workflow.Await(ctx, func() bool { return updateCompleted && workflow.AllHandlersFinished(ctx) }); err != nil { - return "", err - } - return result, nil -} - -``` - - -The Workflow waits for the Update to set its completion state and for `workflow.AllHandlersFinished` to report all -Update handlers done before returning. - -From the root of the `samples-go` repo, start Jaeger and the Worker: - -```bash -docker compose -f opentelemetry-v2/docker-compose.yaml up -d -go run opentelemetry-v2/client-update-propagation/worker/main.go -``` - -In another terminal, run the starter: - -```bash -go run opentelemetry-v2/client-update-propagation/starter/main.go -``` +## Configure context propagation -In the Jaeger UI, select `temporal-otel-v2-custom-client-update-propagation-client` and open the trace containing `send-update`, -`validate-update`, and `handle-update`. The trace crosses into `temporal-otel-v2-custom-client-update-propagation-worker`, -connected by the context the default plugins propagate. +[Context propagation](https://opentelemetry.io/docs/concepts/context-propagation/) +is how OpenTelemetry moves context across process boundaries: inject on the way +out, extract on the way in. -## Shut down telemetry providers +The plugin propagates +[Span Context](https://opentelemetry.io/docs/concepts/signals/traces/#span-context), +which keeps spans linked into one trace, and +[baggage](https://opentelemetry.io/docs/concepts/signals/baggage/): optional +key-value data that travels with the context. -Install the tracer and meter providers as OpenTelemetry globals before constructing the plugin or a Workflow tracer, so -they're available to instrument every Temporal call the process makes. +### `TextMapPropagator` -Shut things down in this order when the process exits: +The plugin injects and extracts both with a +[TextMapPropagator](https://opentelemetry.io/docs/specs/otel/context/api-propagators/#textmap-propagator). +By default that propagator supports +[W3C Trace Context](https://www.w3.org/TR/trace-context/) and +[W3C Baggage](https://www.w3.org/TR/baggage/). Set +`PluginOptions.TextMapPropagator` to override it. -1. Close the Temporal Client or stop the Worker. -2. Shut down the Prometheus-format endpoint. -3. Shut down the meter provider. -4. Shut down the tracer provider. +### `HeaderKey` -Pass each shutdown call its own new context with a timeout. A context left over from the rest of the program might -already be canceled, which cuts the shutdown short. Shutting down the tracer provider flushes the batch span processor, -so spans created earlier still reach the exporter before the process exits. +Propagated values are stored in the Temporal header under `_tracer-data`. Set +`TracerOptions.HeaderKey` to use a different key. -## Propagate trace context and baggage +### `DisableBaggage` -The plugin automatically propagates OpenTelemetry trace context and baggage across Temporal calls, using a composite -W3C Trace Context and Baggage propagator by default instead of the global OpenTelemetry propagator. Use -`PluginOptions.TextMapPropagator` to provide a different propagator. +Set `DisableBaggage` to `true` to stop propagating baggage. -Temporal headers that carry baggage can be persisted in Event Histories. Do not put credentials, tokens, or -other sensitive data in OpenTelemetry baggage. Set `PluginOptions.DisableBaggage` to `true` to turn off baggage -propagation. +### `AllowInvalidParentSpans` -For other application context, see [Context Propagation](/develop/go/best-practices/context-propagation). +Set `AllowInvalidParentSpans` to `true` to ignore errors when extracting +[Span Context](https://opentelemetry.io/docs/concepts/signals/traces/#span-context) +from Temporal headers. Use this when migrating between tracing libraries +while Workflows or Activities are still in progress. ## Resources -- [Automatic instrumentation sample](https://github.com/temporalio/samples-go/tree/main/opentelemetry-v2/automatic-instrumentation) - — spans the Worker creates for Temporal SDK operations, plus Temporal SDK metrics. -- [Workflow-to-Activity propagation sample](https://github.com/temporalio/samples-go/tree/main/opentelemetry-v2/workflow-activity-propagation) - — custom spans propagated from a Workflow to an Activity. -- [Client-to-Update propagation sample](https://github.com/temporalio/samples-go/tree/main/opentelemetry-v2/client-update-propagation) - — custom spans propagated from a Client into a Workflow Update. -- [OpenTelemetry v2 Go package](https://pkg.go.dev/go.temporal.io/sdk/contrib/opentelemetry-v2) — the full plugin - reference. -- [Go SDK observability guide](/develop/go/platform/observability) — where tracing and metrics fit among the SDK's - other observability tools. +- [OpenTelemetry v2 sample](https://github.com/temporalio/samples-go/tree/main/opentelemetry-v2) +- [OpenTelemetry v2 Go package](https://pkg.go.dev/go.temporal.io/sdk/contrib/opentelemetry-v2) +- [Traces](https://opentelemetry.io/docs/concepts/signals/traces/) +- [Metrics](https://opentelemetry.io/docs/concepts/signals/metrics/) +- [Baggage](https://opentelemetry.io/docs/concepts/signals/baggage/) +- [Context propagation](https://opentelemetry.io/docs/concepts/context-propagation/) +- [Go SDK observability guide](/develop/go/platform/observability) diff --git a/docs/develop/go/platform/observability.mdx b/docs/develop/go/platform/observability.mdx index 9d07e7fb5b..bb9756d41e 100644 --- a/docs/develop/go/platform/observability.mdx +++ b/docs/develop/go/platform/observability.mdx @@ -43,14 +43,13 @@ For a complete list of metrics capable of being emitted, see the [SDK metrics re :::tip[Use the OpenTelemetry v2 integration] -For new metrics setups that use OpenTelemetry, use the -[OpenTelemetry v2 integration](/develop/go/integrations/opentelemetry-v2#export-temporal-sdk-metrics). It -configures Temporal SDK metrics through the Go SDK Plugin API and is currently in -[Pre-release](/evaluate/development-production-features/release-stages#pre-release). +To instrument Temporal applications with OpenTelemetry, use the +[OpenTelemetry v2 integration](/develop/go/integrations/opentelemetry-v2) +([Pre-release](/evaluate/development-production-features/release-stages#pre-release)). +It propagates OpenTelemetry context across Temporal boundaries and +is replay-safe when instrumenting Workflows. ::: - -- For an overview of Prometheus and Grafana integration, refer to the [Monitoring](/self-hosted-guide/monitoring) guide. - For a list of metrics, see the [SDK metrics reference](/references/sdk-metrics). - For an end-to-end example that exposes metrics with the Go SDK, refer to the [samples-go](https://github.com/temporalio/samples-go/tree/main/metrics) repo. @@ -102,10 +101,11 @@ Tracing allows you to view the call graph of a Workflow along with its Activitie :::tip[Use the OpenTelemetry v2 integration] -For new tracing setups that use OpenTelemetry, use the -[OpenTelemetry v2 integration](/develop/go/integrations/opentelemetry-v2). It uses the Go SDK Plugin API to -propagate OpenTelemetry trace context across Temporal calls and is currently in -[Pre-release](/evaluate/development-production-features/release-stages#pre-release). +To instrument Temporal applications with OpenTelemetry, use the +[OpenTelemetry v2 integration](/develop/go/integrations/opentelemetry-v2) +([Pre-release](/evaluate/development-production-features/release-stages#pre-release)). +It propagates OpenTelemetry context across Temporal boundaries and +is replay-safe when instrumenting Workflows. ::: From 6b21825d47ad4e832a173c4b2eb71e5425c9b3bd Mon Sep 17 00:00:00 2001 From: Patrik Beqo Date: Fri, 21 Aug 2026 10:33:49 -0400 Subject: [PATCH 09/11] feat: address codex comments --- .../go/integrations/opentelemetry-v2.mdx | 17 +++++++++++------ 1 file changed, 11 insertions(+), 6 deletions(-) diff --git a/docs/develop/go/integrations/opentelemetry-v2.mdx b/docs/develop/go/integrations/opentelemetry-v2.mdx index 262b68648f..048c1540c7 100644 --- a/docs/develop/go/integrations/opentelemetry-v2.mdx +++ b/docs/develop/go/integrations/opentelemetry-v2.mdx @@ -74,6 +74,14 @@ Your application owns the Tracer Provider for the life of the process. Shut it down before exit so remaining spans can flush through the [trace exporter](https://opentelemetry.io/docs/concepts/signals/traces/#trace-exporters). +## Set up the meter provider + +A [Meter Provider](https://opentelemetry.io/docs/concepts/signals/metrics/#meter-provider) +is the factory for Meters. If you enable `MetricsHandlerOptions`, install a +configured Meter Provider with `otel.SetMeterProvider` before you create the +plugin, or pass a Meter via `MetricsHandlerOptions.Meter`. OpenTelemetry's +default global Meter Provider is a no-op. + ## Add the plugin Pass the plugin to your Temporal Client when you create it. Workers made from @@ -187,11 +195,6 @@ across Clients, Workflows, Activities, and Nexus Operations. Set `MetricsHandlerOptions` to a non-`nil` value to emit [Temporal SDK metrics](/references/sdk-metrics) through OpenTelemetry. -`UseMonotonicCounters` controls whether counters are monotonic. - -By default the handler uses a Meter from the global -[Meter Provider](https://opentelemetry.io/docs/concepts/signals/metrics/#meter-provider). -Set `MetricsHandlerOptions.Meter` to use a specific Meter. ## Configure context propagation @@ -203,7 +206,9 @@ The plugin propagates [Span Context](https://opentelemetry.io/docs/concepts/signals/traces/#span-context), which keeps spans linked into one trace, and [baggage](https://opentelemetry.io/docs/concepts/signals/baggage/): optional -key-value data that travels with the context. +key-value data that travels with the context. Do not put credentials, tokens, +or personal data in baggage. The plugin serializes baggage into Temporal +headers that can be persisted in Workflow Event History. ### `TextMapPropagator` From f402468a36d1301828863cf4d221409249364a6a Mon Sep 17 00:00:00 2001 From: Patrik Beqo Date: Fri, 21 Aug 2026 11:28:17 -0400 Subject: [PATCH 10/11] fix: remove retry mention --- docs/develop/go/integrations/opentelemetry-v2.mdx | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/develop/go/integrations/opentelemetry-v2.mdx b/docs/develop/go/integrations/opentelemetry-v2.mdx index 048c1540c7..bab7711fee 100644 --- a/docs/develop/go/integrations/opentelemetry-v2.mdx +++ b/docs/develop/go/integrations/opentelemetry-v2.mdx @@ -66,7 +66,7 @@ install it as the OpenTelemetry global before you create the plugin or call ``` -`NewReplaySafeTracerProvider` keeps span IDs stable across retries and replay when +`NewReplaySafeTracerProvider` keeps span IDs stable across replay when instrumenting Workflows. A standard OpenTelemetry Tracer Provider is not safe for creating spans in Workflows. @@ -114,7 +114,7 @@ can cross Temporal boundaries. A [Tracer](https://opentelemetry.io/docs/concepts/signals/traces/#tracer) creates spans. In Workflows, use `Tracer` instead of `otel.Tracer`. It keeps -span IDs and start times accurate across retries and replay. A standard +span IDs and start times accurate across replay. A standard OpenTelemetry Tracer is not safe for creating spans in Workflows. As in From 235c47785eacda9af25bed49321193faec48864a Mon Sep 17 00:00:00 2001 From: Patrik Beqo Date: Fri, 21 Aug 2026 14:59:11 -0400 Subject: [PATCH 11/11] fix: improvements --- .../go/integrations/opentelemetry-v2.mdx | 90 +++++++++++-------- 1 file changed, 53 insertions(+), 37 deletions(-) diff --git a/docs/develop/go/integrations/opentelemetry-v2.mdx b/docs/develop/go/integrations/opentelemetry-v2.mdx index bab7711fee..45e2dd3a33 100644 --- a/docs/develop/go/integrations/opentelemetry-v2.mdx +++ b/docs/develop/go/integrations/opentelemetry-v2.mdx @@ -18,13 +18,13 @@ of Temporal applications across Clients, Workflows, Activities, and Nexus Operations by instrumenting them with [OpenTelemetry](https://opentelemetry.io/docs/what-is-opentelemetry/). -Temporal provides [durable execution](/temporal#durable-execution). OpenTelemetry -is the vendor-neutral framework for generating and exporting telemetry -to your backend. - -The OpenTelemetry plugin is what connects the two. It propagates OpenTelemetry -context across Temporal boundaries. It can also create spans and emit metrics -for Temporal SDK operations. +OpenTelemetry instruments your applications to give you insight into your +deployed environments. Temporal Workflows complicate that picture because a +trace can span across different Workers over long stretches of time, which +can scatter a trace into disconnected fragments. The OpenTelemetry plugin +solves this by propagating OpenTelemetry context across those Temporal +boundaries, keeping a trace intact end to end. It can also generate spans and +emit metrics for Temporal SDK operations automatically. @@ -32,6 +32,16 @@ All code snippets in this guide are taken from the [OpenTelemetry v2 sample](https://github.com/temporalio/samples-go/tree/main/opentelemetry-v2). Refer to the sample for complete code. +## Prerequisites + +- This guide assumes you are already familiar with OpenTelemetry. If you aren't, refer to the + [OpenTelemetry documentation](https://opentelemetry.io/docs/) for more details. +- If you are new to Temporal, we recommend reading [Understanding Temporal](/evaluate/understanding-temporal) or taking the + [Temporal 101](https://learn.temporal.io/courses/temporal_101/) course. +- Ensure you have set up your local development environment by following the + [Set up your local development environment](/develop/go/set-up-your-local-go) guide. When you're done, leave the + Temporal Development Server running if you want to test your code locally. + ## Install Add the OpenTelemetry v2 integration to your Go module: @@ -46,9 +56,15 @@ backend requires. ## Set up the tracer provider A [Tracer Provider](https://opentelemetry.io/docs/concepts/signals/traces/#tracer-provider) -is the factory for Tracers. Create Temporal's replay-safe Tracer Provider and +is a factory for Tracers, and it configures the Tracers it creates, including +how they generate span IDs. A standard Tracer Provider assigns a new random +span ID each time a span is created, but Temporal Workflows replay, +re-executing the same code and recreating what should be the same span with a +different random ID each time. Temporal's replay-safe Tracer Provider avoids +this by generating span IDs from a deterministic source tied to the +Workflow, so the same span gets the same ID on every replay. Create it and install it as the OpenTelemetry global before you create the plugin or call -`Tracer`: +`Tracer`. [opentelemetry-v2/setup.go](https://github.com/temporalio/samples-go/blob/main/opentelemetry-v2/setup.go) @@ -66,10 +82,6 @@ install it as the OpenTelemetry global before you create the plugin or call ``` -`NewReplaySafeTracerProvider` keeps span IDs stable across replay when -instrumenting Workflows. A standard OpenTelemetry Tracer Provider is not safe for -creating spans in Workflows. - Your application owns the Tracer Provider for the life of the process. Shut it down before exit so remaining spans can flush through the [trace exporter](https://opentelemetry.io/docs/concepts/signals/traces/#trace-exporters). @@ -77,10 +89,11 @@ down before exit so remaining spans can flush through the ## Set up the meter provider A [Meter Provider](https://opentelemetry.io/docs/concepts/signals/metrics/#meter-provider) -is the factory for Meters. If you enable `MetricsHandlerOptions`, install a -configured Meter Provider with `otel.SetMeterProvider` before you create the -plugin, or pass a Meter via `MetricsHandlerOptions.Meter`. OpenTelemetry's -default global Meter Provider is a no-op. +is a factory for Meters. OpenTelemetry's default global Meter Provider is a +no-op, so if you enable `MetricsHandlerOptions`, you need to supply a +configured one yourself, either by installing it with `otel.SetMeterProvider` +before you create the plugin, or by passing a Meter directly through +`MetricsHandlerOptions.Meter`. ## Add the plugin @@ -113,15 +126,14 @@ can cross Temporal boundaries. ### In Workflows A [Tracer](https://opentelemetry.io/docs/concepts/signals/traces/#tracer) -creates spans. In Workflows, use `Tracer` instead of `otel.Tracer`. It keeps -span IDs and start times accurate across replay. A standard -OpenTelemetry Tracer is not safe for creating spans in Workflows. - -As in -[OpenTelemetry Go](https://opentelemetry.io/docs/languages/go/instrumentation/), -`Start` returns a context that contains the active span. Pass that -`workflow.Context` to downstream Temporal calls so later spans nest under it as -children: +creates spans that capture information about a given operation. A standard +Tracer stamps a span with the current time and emits it as soon as it +completes, but Temporal Workflows replay, re-executing the same code and +stamping what should be the same span with a new time and emitting a +duplicate span. Temporal's replay-safe `Tracer` avoids this by stamping a +span with `workflow.Now`, Temporal's replay-safe clock, and skipping a span +that already completed on a previous successful execution. Use it instead +of `otel.Tracer` in Workflows. [opentelemetry-v2/workflow-activity-propagation/opentelemetry.go](https://github.com/temporalio/samples-go/blob/main/opentelemetry-v2/workflow-activity-propagation/opentelemetry.go) @@ -146,6 +158,12 @@ func Workflow(ctx workflow.Context, name string) (string, error) { ``` +As in +[OpenTelemetry Go](https://opentelemetry.io/docs/languages/go/instrumentation/), +`Start` returns a context that contains the active span. Pass that +`workflow.Context` to downstream Temporal calls so later spans nest under it as +children. + ### Outside Workflows In Clients, Activities, and other non-Workflow code, use an ordinary OpenTelemetry @@ -166,9 +184,6 @@ func Activity(ctx context.Context, name string) (string, error) { ## Enable automatic instrumentation -Set options on `PluginOptions` to create spans and emit metrics for Temporal -SDK operations: - [opentelemetry-v2/automatic-instrumentation/worker/main.go](https://github.com/temporalio/samples-go/blob/main/opentelemetry-v2/automatic-instrumentation/worker/main.go) ```go @@ -199,16 +214,17 @@ Set `MetricsHandlerOptions` to a non-`nil` value to emit ## Configure context propagation [Context propagation](https://opentelemetry.io/docs/concepts/context-propagation/) -is how OpenTelemetry moves context across process boundaries: inject on the way -out, extract on the way in. - -The plugin propagates +is how OpenTelemetry moves context across process boundaries, injecting it +on the way out and extracting it on the way in. The plugin performs this +propagation for you across Temporal boundaries, carrying [Span Context](https://opentelemetry.io/docs/concepts/signals/traces/#span-context), which keeps spans linked into one trace, and -[baggage](https://opentelemetry.io/docs/concepts/signals/baggage/): optional -key-value data that travels with the context. Do not put credentials, tokens, -or personal data in baggage. The plugin serializes baggage into Temporal -headers that can be persisted in Workflow Event History. +[baggage](https://opentelemetry.io/docs/concepts/signals/baggage/), optional +key-value data that travels with the context. + +Do not put credentials, tokens, or personal data in baggage since the +plugin serializes it into Temporal headers that can be persisted in +Workflow Event History. ### `TextMapPropagator`