Add JavaScript source map symbolication

.github/workflows/build.yaml

@@ -580,8 +580,72 @@ jobs:
           kubectl patch cronjob/ex-dev-jobs-maintain-indexes -p '{"spec":{"suspend": false}}' --namespace $DEV_NAMESPACE
           kubectl patch cronjob/ex-dev-jobs-migration -p '{"spec":{"suspend": false}}' --namespace $DEV_NAMESPACE
 
-          kubectl wait --for=condition=available --timeout=300s deployment/ex-dev-app --namespace $DEV_NAMESPACE
-          kubectl wait --for=condition=available --timeout=300s deployment/ex-dev-api --namespace $DEV_NAMESPACE
+          wait_for_versioned_pod() {
+            component="$1"
+            replica_set=""
+            for attempt in {1..12}; do
+              replica_set=$(kubectl get replicasets --namespace "$DEV_NAMESPACE" --selector "component=${component}" --output json \
+                | jq --raw-output --arg image_tag ":${VERSION}" '
+                    [.items[]
+                      | select(any(.spec.template.spec.containers[]; .image | endswith($image_tag)))
+                      | { name: .metadata.name, created: .metadata.creationTimestamp }]
+                    | sort_by(.created)
+                    | last
+                    | .name // empty')
+              [[ -n "$replica_set" ]] && break
+              sleep 5
+            done
+            if [[ -z "$replica_set" ]]; then
+              echo "::error::No ReplicaSet for ${component} uses version ${VERSION}."
+              return 1
+            fi
+
+            pod_template_hash=$(kubectl get replicaset "$replica_set" --namespace "$DEV_NAMESPACE" \
+              --output jsonpath='{.metadata.labels.pod-template-hash}')
+            kubectl wait --for=create --timeout=60s pod \
+              --namespace "$DEV_NAMESPACE" \
+              --selector "component=${component},pod-template-hash=${pod_template_hash}"
+            if ! kubectl wait --for=condition=Ready --timeout=300s pod \
+                --namespace "$DEV_NAMESPACE" \
+                --selector "component=${component},pod-template-hash=${pod_template_hash}"; then
+              echo "::group::${component} rollout diagnostics"
+              kubectl get pods --namespace "$DEV_NAMESPACE" \
+                --selector "component=${component},pod-template-hash=${pod_template_hash}" \
+                --output wide
+              for pod in $(kubectl get pods --namespace "$DEV_NAMESPACE" \
+                  --selector "component=${component},pod-template-hash=${pod_template_hash}" \
+                  --output name); do
+                kubectl describe "$pod" --namespace "$DEV_NAMESPACE"
+                kubectl logs "$pod" --namespace "$DEV_NAMESPACE" --all-containers --tail=300 \
+                  | grep --extended-regexp 'Now listening on:|Application started|Application is shutting down' || true
+              done
+              echo "::endgroup::"
+              return 1
+            fi
+          }
+
+          wait_for_versioned_pod ex-dev-app
+          wait_for_versioned_pod ex-dev-api
+          wait_for_versioned_pod ex-dev-jobs-event-posts
+
+          for endpoint in dev-app.exceptionless.io dev-api.exceptionless.io dev-collector.exceptionless.io; do
+            deployed_version=""
+            for attempt in {1..12}; do
+              if deployed_version=$(curl --fail --silent --show-error --retry 2 --retry-all-errors --retry-delay 2 \
+                  --header "Cache-Control: no-cache" \
+                  "https://${endpoint}/api/v2/about?deployment=${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}-${attempt}" \
+                  | jq --raw-output '.informational_version'); then
+                if [[ "$deployed_version" == "${VERSION}+"* ]]; then
+                  break
+                fi
+              fi
+              sleep 5
+            done
+            if [[ "$deployed_version" != "${VERSION}+"* ]]; then
+              echo "::error::${endpoint} is serving ${deployed_version}; expected ${VERSION}."
+              exit 1
+            fi
+          done
 
           kubectl annotate namespace $DEV_NAMESPACE exceptionless.io/dev-started-at="$(date -u +%Y-%m-%dT%H:%M:%SZ)" --overwrite
           kubectl annotate namespace $DEV_NAMESPACE exceptionless.io/dev-auto-stop-days="1" --overwrite

Dockerfile

@@ -44,6 +44,8 @@ FROM mcr.microsoft.com/dotnet/aspnet:10.0.9 AS job
 WORKDIR /app
 COPY --from=job-publish /app/src/Exceptionless.Job/out ./
 
+ENV ASPNETCORE_URLS=http://+:8080
+
 EXPOSE 8080
 
 ENTRYPOINT [ "dotnet", "Exceptionless.Job.dll" ]
@@ -61,6 +63,8 @@ FROM mcr.microsoft.com/dotnet/aspnet:10.0.9 AS api
 WORKDIR /app
 COPY --from=api-publish /app/src/Exceptionless.Web/out ./
 
+ENV ASPNETCORE_URLS=http://+:8080
+
 EXPOSE 8080
 
 ENTRYPOINT [ "dotnet", "Exceptionless.Web.dll" ]

docs/docs/source-maps.md

@@ -0,0 +1,57 @@
+---
+title: "JavaScript Source Maps"
+---
+
+# JavaScript Source Maps
+
+Exceptionless uses source maps to turn minified JavaScript stack frames into the original file names, line and column numbers, and function names. Symbolication happens before an event is assigned to a stack, so readable function names also improve stack grouping.
+
+## Automatic discovery
+
+No domain allowlist or project setup is required for public source maps. When an error contains an absolute HTTPS JavaScript URL, Exceptionless checks the generated file's `SourceMap` or `X-SourceMap` response header and its `sourceMappingURL` comment. If neither is present, it also checks the conventional `<generated-file>.map` URL.
+
+Downloaded maps are validated and cached in project-scoped file storage. Automatically downloaded maps are revalidated after one hour so a stable generated-file URL cannot retain a map from an older deployment indefinitely. If refresh fails, Exceptionless leaves the generated frame unchanged instead of risking a misleading stack trace from the stale map.
+
+Exceptionless only makes anonymous HTTPS requests on the standard port to public network addresses. Redirects and every resolved address are revalidated. New automatic discoveries use plan-aware limits per client key, project, and organization; outbound requests are also limited per destination, IP address, and cluster. Free plans allow five new discoveries per client key and ten per project or organization in each 15-minute window by default. Paid plans have higher limits. Failed URLs are cached for 15 minutes, duplicate discoveries share the same in-flight work, and refreshes use a separate smaller outbound budget. If any limit is reached, Exceptionless leaves the generated frame unchanged without rejecting or delaying the event.
+
+Downloads also have time, redirect, size, and local and cluster-wide concurrency limits. Parsed maps use a bounded in-memory cache. Manual and deployment uploads do not consume automatic-discovery quotas. Each project can store up to 1,000 maps and 1 GiB by default; replacing a map for the same generated URL does not consume another slot. Self-hosted installations can tune these safeguards under the `SourceMaps` configuration section, including `MaximumArtifactsPerProject`, `MaximumStorageSizePerProject`, `AutoDownloadRateLimitPeriodMinutes`, `MaximumAutoDiscoveriesPerFreeClientKey`, `MaximumAutoDiscoveriesPerProject`, `MaximumAutoDownloadRequestsPerDestination`, `MaximumAutoRefreshRequestsGlobally`, `AutoDownloadRefreshIntervalMinutes`, `ParsedSourceMapCacheLifetimeMinutes`, and `MaximumParsedSourceMapCacheSize`.
+
+## Uploading a source map
+
+Upload a map when it is private or is not deployed next to the generated JavaScript:
+
+1. Open the project and select **Source Maps** under **Project Settings**.
+2. Enter the exact absolute URL that appears in the generated stack frame, including any path or query string used to identify the build.
+3. Select the corresponding source map and upload it.
+
+Uploading another map for the same generated file URL replaces the previous map. Uploaded and automatically discovered maps appear together on the Source Maps page and can be deleted there.
+
+### Uploading during deployment
+
+For build automation, create a project-scoped token that has only the `source-maps:write` scope. Set `EXCEPTIONLESS_SERVER_URL` to `https://be.exceptionless.io` for the hosted service or to the root URL of your self-hosted installation. Create the token once with a user-scoped token, then store the returned `id` as a protected CI/CD secret:
+
+```shell
+curl --fail-with-body --request POST \
+  "${EXCEPTIONLESS_SERVER_URL}/api/v2/projects/${EXCEPTIONLESS_PROJECT_ID}/tokens" \
+  --header "Authorization: Bearer ${EXCEPTIONLESS_USER_TOKEN}" \
+  --header "Content-Type: application/json" \
+  --data "{\"organization_id\":\"${EXCEPTIONLESS_ORGANIZATION_ID}\",\"project_id\":\"${EXCEPTIONLESS_PROJECT_ID}\",\"scopes\":[\"source-maps:write\"],\"notes\":\"CI source map uploads\"}"
+```
+
+Upload each map after the generated JavaScript has been deployed. The `generated_file_url` must be the exact absolute URL that will appear in stack frames:
+
+```shell
+curl --fail-with-body --request POST \
+  "${EXCEPTIONLESS_SERVER_URL}/api/v2/projects/${EXCEPTIONLESS_PROJECT_ID}/source-maps" \
+  --header "Authorization: Bearer ${EXCEPTIONLESS_SOURCE_MAP_TOKEN}" \
+  --form "generated_file_url=https://cdn.example.com/assets/app.a1b2c3.js" \
+  --form "file=@dist/assets/app.a1b2c3.js.map;type=application/json"
+```
+
+The upload token is accepted only for its assigned project and cannot read events, manage the project, list source maps, or use ordinary user APIs. Do not use a normal client API key for source map uploads because client keys are commonly embedded in distributed applications.
+
+Source maps must use the version 3 flat-map format. Indexed source maps with a `sections` property and authenticated automatic downloads are planned follow-up capabilities; private maps can be uploaded in the meantime.
+
+## Deployment guidance
+
+Generate source maps as part of the same build that produces the minified JavaScript. Content-hashed generated file names are preferred because the generated URL then identifies a specific build. You can publish the `.map` file for zero-configuration discovery or keep it private and upload it to Exceptionless during deployment.

src/Exceptionless.Core/Authorization/AuthorizationRoles.cs

@@ -12,12 +12,14 @@ public static class AuthorizationRoles
     public const string ProjectsReadPolicy = nameof(ProjectsReadPolicy);
     public const string StacksReadPolicy = nameof(StacksReadPolicy);
     public const string StacksWritePolicy = nameof(StacksWritePolicy);
+    public const string SourceMapsWritePolicy = nameof(SourceMapsWritePolicy);
     public const string EventsReadPolicy = nameof(EventsReadPolicy);
     public const string McpRead = "mcp:read";
     public const string ProjectsRead = "projects:read";
     public const string StacksRead = "stacks:read";
     public const string StacksWrite = "stacks:write";
+    public const string SourceMapsWrite = "source-maps:write";
     public const string EventsRead = "events:read";
     public const string OfflineAccess = "offline_access";
-    public static readonly ISet<string> AllScopes = new HashSet<string>([Client, User, GlobalAdmin]);
+    public static readonly ISet<string> AllScopes = new HashSet<string>([Client, User, GlobalAdmin, SourceMapsWrite]);
 }

src/Exceptionless.Core/Bootstrapper.cs

@@ -25,6 +25,7 @@
 using Exceptionless.Core.Seed;
 using Exceptionless.Core.Serialization;
 using Exceptionless.Core.Services;
+using Exceptionless.Core.Services.SourceMaps;
 using Exceptionless.Core.Utility;
 using Exceptionless.Core.Validation;
 using Foundatio.Caching;
@@ -193,6 +194,16 @@ public static void RegisterServices(IServiceCollection services, AppOptions appO
                 AllowAutoRedirect = false,
                 ConnectCallback = ConnectToPublicAddressAsync
             });
+        services.AddSingleton<SourceMapRequestThrottle>();
+        services.AddHttpClient(SourceMapService.HttpClientName)
+            .ConfigurePrimaryHttpMessageHandler(serviceProvider => CreateSourceMapHttpMessageHandler(
+                serviceProvider.GetRequiredService<SourceMapRequestThrottle>(),
+                DecompressionMethods.All));
+        services.AddHttpClient(SourceMapService.GeneratedFileHttpClientName)
+            .ConfigurePrimaryHttpMessageHandler(serviceProvider => CreateSourceMapHttpMessageHandler(
+                serviceProvider.GetRequiredService<SourceMapRequestThrottle>(),
+                DecompressionMethods.None));
+        services.AddSingleton<SourceMapService>();
         services.AddSingleton<OAuthService>();
         services.AddSingleton<UsageService>();
         services.AddSingleton<SlackService>();
@@ -223,9 +234,21 @@ private static async ValueTask<Stream> ConnectToPublicAddressAsync(SocketsHttpCo
             }
         }
 
-        throw new HttpRequestException($"OAuth client metadata host '{context.DnsEndPoint.Host}' did not resolve to a reachable public address.", lastException);
+        throw new HttpRequestException($"Host '{context.DnsEndPoint.Host}' did not resolve to a reachable public address.", lastException);
     }
 
+    internal static SocketsHttpHandler CreateSourceMapHttpMessageHandler(SourceMapRequestThrottle throttle, DecompressionMethods automaticDecompression)
+        => new()
+        {
+            ActivityHeadersPropagator = null,
+            AllowAutoRedirect = false,
+            AutomaticDecompression = automaticDecompression,
+            ConnectCallback = throttle.ConnectToPublicAddressAsync,
+            PreAuthenticate = false,
+            UseCookies = false,
+            UseProxy = false
+        };
+
     public static void LogConfiguration(IServiceProvider serviceProvider, AppOptions appOptions, ILogger logger)
     {
         if (!logger.IsEnabled(LogLevel.Warning))

src/Exceptionless.Core/Configuration/AppOptions.cs

@@ -80,6 +80,7 @@ public class AppOptions
     public StripeOptions StripeOptions { get; internal set; } = null!;
     public AuthOptions AuthOptions { get; internal set; } = null!;
     public OAuthServerOptions OAuthServerOptions { get; internal set; } = null!;
+    public SourceMapOptions SourceMapOptions { get; internal set; } = null!;
 
     public static AppOptions ReadFromConfiguration(IConfiguration config)
     {
@@ -133,6 +134,7 @@ public static AppOptions ReadFromConfiguration(IConfiguration config)
         options.StripeOptions = StripeOptions.ReadFromConfiguration(config);
         options.AuthOptions = AuthOptions.ReadFromConfiguration(config);
         options.OAuthServerOptions = OAuthServerOptions.ReadFromConfiguration(config);
+        options.SourceMapOptions = SourceMapOptions.ReadFromConfiguration(config);
 
         return options;
     }