Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -80,6 +80,7 @@ static IDurableTaskWorkerBuilder UseExternalizedPayloadsCore(IDurableTaskWorkerB
}

opt.Capabilities.Add(P.WorkerCapability.LargePayloads);
opt.ExternalPayloadStoreUri = store.ExternalPayloadStoreBaseUri?.ToString();
});

return builder;
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -67,6 +67,15 @@ public BlobPayloadStore(LargePayloadStorageOptions options)
this.containerClient = serviceClient.GetBlobContainerClient(options.ContainerName);
}

/// <inheritdoc/>
public override Uri? ExternalPayloadStoreBaseUri => new BlobUriBuilder(this.containerClient.Uri)
{
BlobContainerName = string.Empty,
BlobName = string.Empty,
Sas = null,
Query = string.Empty,
}.ToUri();

/// <inheritdoc/>
public override async Task<string> UploadAsync(string payLoad, CancellationToken cancellationToken)
{
Expand Down
6 changes: 6 additions & 0 deletions src/Extensions/AzureBlobPayloads/PayloadStore/PayloadStore.cs
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,12 @@ namespace Microsoft.DurableTask;
/// </summary>
public abstract class PayloadStore
{
/// <summary>
/// Gets the non-secret base URI (scheme+host[+account path for emulator]) of the external payload store, or
/// <c>null</c> if not applicable. Never contains a secret/SAS/key.
/// </summary>
public virtual Uri? ExternalPayloadStoreBaseUri => null;

/// <summary>
/// Uploads a payload and returns an opaque reference token that can be embedded in orchestration messages.
/// </summary>
Expand Down
4 changes: 4 additions & 0 deletions src/Grpc/orchestrator_service.proto
Original file line number Diff line number Diff line change
Expand Up @@ -832,6 +832,10 @@ message GetWorkItemsRequest {

repeated WorkerCapability capabilities = 10;
WorkItemFilters workItemFilters = 11;

// Non-secret base URI of the external payload store (e.g. "https://<account>.blob.core.windows.net/") where this
// worker externalizes large payloads. Empty when no external payload store is configured. Never contains a secret.
string external_payload_store_uri = 12 [json_name = "externalPayloadStoreUri"];
}

enum WorkerCapability {
Expand Down
1 change: 1 addition & 0 deletions src/Worker/Grpc/GrpcDurableTaskWorker.Processor.cs
Original file line number Diff line number Diff line change
Expand Up @@ -333,6 +333,7 @@ async ValueTask<OrchestrationRuntimeState> BuildRuntimeStateAsync(
workerOptions.Concurrency.MaximumConcurrentEntityWorkItems,
Capabilities = { this.worker.grpcOptions.Capabilities },
WorkItemFilters = this.worker.workItemFilters?.ToGrpcWorkItemFilters(),
ExternalPayloadStoreUri = this.worker.grpcOptions.ExternalPayloadStoreUri ?? string.Empty,
},
cancellationToken: cancellation);
}
Expand Down
6 changes: 6 additions & 0 deletions src/Worker/Grpc/GrpcDurableTaskWorkerOptions.cs
Original file line number Diff line number Diff line change
Expand Up @@ -44,6 +44,12 @@ public sealed class GrpcDurableTaskWorkerOptions : DurableTaskWorkerOptions
/// </summary>
public HashSet<P.WorkerCapability> Capabilities { get; } = new() { P.WorkerCapability.HistoryStreaming };

/// <summary>
/// Gets or sets the non-secret external payload store base URI reported to the backend on the work-item
/// handshake. Empty/null when not using externalized payloads.
/// </summary>
public string? ExternalPayloadStoreUri { get; set; }

/// <summary>
/// Gets or sets the maximum size of all actions in a complete orchestration work item chunk.
/// The default value is 3.9MB. We leave some headroom to account for request size overhead.
Expand Down
111 changes: 111 additions & 0 deletions test/Worker/Grpc.Tests/ExternalPayloadStoreUriTests.cs
Original file line number Diff line number Diff line change
@@ -0,0 +1,111 @@
// Copyright (c) Microsoft Corporation.
// Licensed under the MIT License.

using Azure.Core;
using Microsoft.Extensions.DependencyInjection;

namespace Microsoft.DurableTask.Worker.Grpc.Tests;

/// <summary>
/// Tests for the non-secret external payload store base URI that the worker reports on the work-item handshake.
/// </summary>
public class ExternalPayloadStoreUriTests
{
// Well-known Azurite development storage account key. Used here only to prove it is never leaked into the base URI.
const string DevelopmentAccountKey =
"Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==";

[Fact]
public void ExternalPayloadStoreBaseUri_EmulatorConnectionString_ReturnsAccountBaseUriWithoutContainerOrSas()
{
// Arrange
LargePayloadStorageOptions options = new("UseDevelopmentStorage=true")
{
ContainerName = "my-payloads",
};
BlobPayloadStore store = new(options);

// Act
Uri? baseUri = store.ExternalPayloadStoreBaseUri;

// Assert
baseUri.Should().NotBeNull();
baseUri!.AbsoluteUri.TrimEnd('/').Should().Be("http://127.0.0.1:10000/devstoreaccount1");
baseUri.Query.Should().BeEmpty();
baseUri.AbsoluteUri.Should().NotContain("my-payloads");
}

[Fact]
public void ExternalPayloadStoreBaseUri_ConnectionStringWithAccountKey_ReturnsAccountBaseUriAndNeverLeaksKey()
{
// Arrange
LargePayloadStorageOptions options = new(
$"DefaultEndpointsProtocol=https;AccountName=myaccount;AccountKey={DevelopmentAccountKey};EndpointSuffix=core.windows.net")
{
ContainerName = "my-payloads",
};
BlobPayloadStore store = new(options);

// Act
Uri? baseUri = store.ExternalPayloadStoreBaseUri;

// Assert
baseUri.Should().Be(new Uri("https://myaccount.blob.core.windows.net/"));
baseUri!.Query.Should().BeEmpty();
baseUri.AbsoluteUri.Should().NotContain(DevelopmentAccountKey);
baseUri.AbsoluteUri.Should().NotContain("my-payloads");
}

[Fact]
public void ExternalPayloadStoreBaseUri_AccountUriAndCredential_ReturnsAccountBaseUriWithoutContainerOrSas()
{
// Arrange
LargePayloadStorageOptions options = new(
new Uri("https://myaccount.blob.core.windows.net"),
new FakeTokenCredential())
{
ContainerName = "my-payloads",
};
BlobPayloadStore store = new(options);

// Act
Uri? baseUri = store.ExternalPayloadStoreBaseUri;

// Assert
baseUri.Should().Be(new Uri("https://myaccount.blob.core.windows.net/"));
baseUri!.Query.Should().BeEmpty();
baseUri.AbsoluteUri.Should().NotContain("my-payloads");
}

[Fact]
public void UseExternalizedPayloads_SetsExternalPayloadStoreUriFromStore()
{
// Arrange
ServiceCollection services = new();
DefaultDurableTaskWorkerBuilder builder = new(null, services);
builder.UseGrpc(GrpcChannel.ForAddress("http://localhost:9001"));
builder.UseExternalizedPayloads(opt =>
{
opt.ContainerName = "my-payloads";
opt.ConnectionString = "UseDevelopmentStorage=true";
});

// Act
IServiceProvider provider = services.BuildServiceProvider();
GrpcDurableTaskWorkerOptions options = provider.GetOptions<GrpcDurableTaskWorkerOptions>();

// Assert
options.ExternalPayloadStoreUri.Should().NotBeNullOrEmpty();
options.ExternalPayloadStoreUri!.TrimEnd('/').Should().Be("http://127.0.0.1:10000/devstoreaccount1");
}

sealed class FakeTokenCredential : TokenCredential
{
public override AccessToken GetToken(TokenRequestContext requestContext, CancellationToken cancellationToken)
=> new("fake-token", DateTimeOffset.MaxValue);

public override ValueTask<AccessToken> GetTokenAsync(
TokenRequestContext requestContext, CancellationToken cancellationToken)
=> new(this.GetToken(requestContext, cancellationToken));
}
}
Loading