From 17f6ddbe503710af8dff9c39c2bc517348df972e Mon Sep 17 00:00:00 2001 From: Bhagirath Mehta Date: Mon, 17 Aug 2026 13:56:05 -0500 Subject: [PATCH 1/4] Limit model blob downloads to three hours Prevent stalled Azure blob transfers from running indefinitely while preserving explicit callback cancellation. Document the current inference cancellation limits and remove an overbroad C# cancellation claim. Files changed: - README.md - sdk_v2/cpp/src/download/blob_downloader.cc - sdk_v2/cs/README.md Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> Copilot-Session: 16cc8b6f-2bc4-4469-bf59-f21439569349 --- README.md | 5 +++++ sdk_v2/cpp/src/download/blob_downloader.cc | 9 +++++++-- sdk_v2/cs/README.md | 2 +- 3 files changed, 13 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index b86e183cd..50187be8e 100644 --- a/README.md +++ b/README.md @@ -195,6 +195,11 @@ Please report issues or suggest improvements in the [GitHub Issues](https://gith ## ❔ Frequently asked questions +### Can inference be cancelled? + +Streaming inference supports cooperative cancellation. Non-streaming inference cannot currently be cancelled after +processing starts and runs to completion. + ### Is Foundry Local a web server and CLI tool? No. Foundry Local is an **end-to-end local AI solution** that your application ships with. It handles model acquisition, hardware acceleration, and inference inside your app process through the SDK. The optional web server and CLI are available for development workflows, but the core product is the local AI runtime and SDK that you integrate directly into your application. diff --git a/sdk_v2/cpp/src/download/blob_downloader.cc b/sdk_v2/cpp/src/download/blob_downloader.cc index 9254a8b7f..4bb964afc 100644 --- a/sdk_v2/cpp/src/download/blob_downloader.cc +++ b/sdk_v2/cpp/src/download/blob_downloader.cc @@ -181,7 +181,8 @@ void AzureBlobDownloader::DownloadBlob(const std::string& sas_uri, // Single shared Azure context for the whole blob; calling Cancel() on it // propagates into every in-flight chunk read. - Azure::Core::Context azure_ctx; + auto azure_ctx = Azure::Core::Context{}.WithDeadline( + Azure::DateTime(std::chrono::system_clock::now() + std::chrono::hours{3})); // Internal cancel flag flipped by the orchestrator on first chunk failure // or by external cancellation; checked by workers between iterations. std::atomic internal_cancel{false}; @@ -409,7 +410,11 @@ void AzureBlobDownloader::DownloadBlob(const std::string& sas_uri, // All chunks done — sidecar is no longer needed. BlobDownloadState::DeleteState(local_path, logger_); } catch (const Azure::Core::OperationCancelledException&) { - FL_THROW(FOUNDRY_LOCAL_ERROR_OPERATION_CANCELLED, "download cancelled"); + if (cancelled && cancelled->load(std::memory_order_relaxed)) { + FL_THROW(FOUNDRY_LOCAL_ERROR_OPERATION_CANCELLED, "download cancelled"); + } + + FL_THROW(FOUNDRY_LOCAL_ERROR_NETWORK, "model download timed out after 3 hours"); } catch (const Azure::Core::RequestFailedException& e) { FL_THROW(FOUNDRY_LOCAL_ERROR_NETWORK, std::string("failed to download blob '") + blob_name + "': " + e.what()); diff --git a/sdk_v2/cs/README.md b/sdk_v2/cs/README.md index 253c6a8a0..b555e3e20 100644 --- a/sdk_v2/cs/README.md +++ b/sdk_v2/cs/README.md @@ -12,7 +12,7 @@ The Foundry Local C# SDK provides a .NET interface for running AI models locally - **Model variants** — select specific hardware/quantization variants per model alias - **Optional web service** — start an OpenAI-compatible REST endpoint (`/v1/chat_completions`, `/v1/models`) - **WinML acceleration** — built-in Windows hardware acceleration with automatic EP download -- **Full async/await** — every operation supports `CancellationToken` and async patterns +- **Async APIs** — idiomatic async/await support across the SDK - **IDisposable** — deterministic cleanup of native resources ## Installation From 57ba6ba44f4b5f697eebab6e19972008e3617e8f Mon Sep 17 00:00:00 2001 From: Bhagirath Mehta Date: Mon, 17 Aug 2026 13:59:27 -0500 Subject: [PATCH 2/4] Keep cancellation note on one line Avoid an unnecessary source-line break in the short cancellation limitation. Files changed: - README.md Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> Copilot-Session: 16cc8b6f-2bc4-4469-bf59-f21439569349 --- README.md | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/README.md b/README.md index 50187be8e..506a4d20d 100644 --- a/README.md +++ b/README.md @@ -197,8 +197,7 @@ Please report issues or suggest improvements in the [GitHub Issues](https://gith ### Can inference be cancelled? -Streaming inference supports cooperative cancellation. Non-streaming inference cannot currently be cancelled after -processing starts and runs to completion. +Streaming inference supports cooperative cancellation. Non-streaming inference cannot currently be cancelled after processing starts and runs to completion. ### Is Foundry Local a web server and CLI tool? From ffb79f814552487ce5a50f7ed50bd11e7f35a4bb Mon Sep 17 00:00:00 2001 From: Bhagirath Mehta Date: Mon, 17 Aug 2026 14:15:54 -0500 Subject: [PATCH 3/4] Address Copilot timeout review Correct the cancellation documentation to reflect existing Request.Cancel support and the remaining cooperative/no-deadline limitations. Add focused regression coverage for timeout versus explicit-cancellation error classification without expanding production timeout plumbing. Files changed: - README.md - sdk_v2/cpp/test/internal_api/download_test.cc Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> Copilot-Session: 16cc8b6f-2bc4-4469-bf59-f21439569349 --- README.md | 2 +- sdk_v2/cpp/test/internal_api/download_test.cc | 42 ++++++++++++++++++- 2 files changed, 42 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 506a4d20d..3aba62b03 100644 --- a/README.md +++ b/README.md @@ -197,7 +197,7 @@ Please report issues or suggest improvements in the [GitHub Issues](https://gith ### Can inference be cancelled? -Streaming inference supports cooperative cancellation. Non-streaming inference cannot currently be cancelled after processing starts and runs to completion. +Inference cancellation is cooperative. Low-level requests can be cancelled with `Request.Cancel()`, while streaming APIs provide language-specific cancellation mechanisms. Cancellation may not take effect until the current generation step completes, and inference requests do not currently have a built-in timeout. ### Is Foundry Local a web server and CLI tool? diff --git a/sdk_v2/cpp/test/internal_api/download_test.cc b/sdk_v2/cpp/test/internal_api/download_test.cc index d8a5f9b5c..c11a3ba2c 100644 --- a/sdk_v2/cpp/test/internal_api/download_test.cc +++ b/sdk_v2/cpp/test/internal_api/download_test.cc @@ -21,6 +21,7 @@ #include "test_helpers.h" #include "util/path_safety.h" #include "util/region_fallback.h" +#include #include #include #include @@ -1542,6 +1543,7 @@ namespace { class FakeChunkAzureDownloader : public AzureBlobDownloader { public: int64_t blob_size = 0; + std::function get_blob_size_hook; /// Per-call hook. Receives the chunk offset and size plus a `sink` callback /// that forwards bytes to the file writer. Allowed to: @@ -1566,7 +1568,13 @@ class FakeChunkAzureDownloader : public AzureBlobDownloader { FakeChunkAzureDownloader() : AzureBlobDownloader(fl::test::NullLog()) {} protected: - int64_t GetBlobSize(ChunkContext& /*ctx*/) override { return blob_size; } + int64_t GetBlobSize(ChunkContext& /*ctx*/) override { + if (get_blob_size_hook) { + get_blob_size_hook(); + } + + return blob_size; + } void DownloadChunkStreaming(ChunkContext& ctx, int64_t offset, int64_t size, std::vector& scratch, @@ -1598,6 +1606,38 @@ class FakeChunkAzureDownloader : public AzureBlobDownloader { } // namespace +TEST(AzureBlobDownloaderTimeoutTest, DeadlineCancellationWithoutExternalCancellationBecomesNetworkError) { + auto tmpdir = TempPath::CreateTempDir(); + FakeChunkAzureDownloader d; + d.get_blob_size_hook = []() { + throw Azure::Core::OperationCancelledException("deadline exceeded"); + }; + + try { + d.DownloadBlob("", "blob", (tmpdir.path() / "blob.bin").string(), 1); + FAIL() << "expected fl::Exception"; + } catch (const fl::Exception& e) { + EXPECT_EQ(e.code(), FOUNDRY_LOCAL_ERROR_NETWORK); + EXPECT_NE(std::string(e.what()).find("model download timed out after 3 hours"), std::string::npos); + } +} + +TEST(AzureBlobDownloaderTimeoutTest, DeadlineCancellationWithExternalCancellationRemainsCancelled) { + auto tmpdir = TempPath::CreateTempDir(); + FakeChunkAzureDownloader d; + d.get_blob_size_hook = []() { + throw Azure::Core::OperationCancelledException("deadline exceeded"); + }; + std::atomic cancelled{true}; + + try { + d.DownloadBlob("", "blob", (tmpdir.path() / "blob.bin").string(), 1, nullptr, &cancelled); + FAIL() << "expected fl::Exception"; + } catch (const fl::Exception& e) { + EXPECT_EQ(e.code(), FOUNDRY_LOCAL_ERROR_OPERATION_CANCELLED); + } +} + TEST(AzureBlobDownloaderResumeTest, SkipsChunksAlreadyMarkedCompleteInSidecar) { auto tmpdir = TempPath::CreateTempDir(); auto local = tmpdir.path() / "blob.bin"; From 631a5840b4ca0a2db6e215b7a821b21047984f9a Mon Sep 17 00:00:00 2001 From: Bhagirath Mehta Date: Mon, 17 Aug 2026 14:18:40 -0500 Subject: [PATCH 4/4] Remove timeout classification tests Keep the release change limited to the production deadline and concise documentation rather than adding test-only timeout machinery. Files changed: - sdk_v2/cpp/test/internal_api/download_test.cc Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> Copilot-Session: 16cc8b6f-2bc4-4469-bf59-f21439569349 --- sdk_v2/cpp/test/internal_api/download_test.cc | 42 +------------------ 1 file changed, 1 insertion(+), 41 deletions(-) diff --git a/sdk_v2/cpp/test/internal_api/download_test.cc b/sdk_v2/cpp/test/internal_api/download_test.cc index c11a3ba2c..d8a5f9b5c 100644 --- a/sdk_v2/cpp/test/internal_api/download_test.cc +++ b/sdk_v2/cpp/test/internal_api/download_test.cc @@ -21,7 +21,6 @@ #include "test_helpers.h" #include "util/path_safety.h" #include "util/region_fallback.h" -#include #include #include #include @@ -1543,7 +1542,6 @@ namespace { class FakeChunkAzureDownloader : public AzureBlobDownloader { public: int64_t blob_size = 0; - std::function get_blob_size_hook; /// Per-call hook. Receives the chunk offset and size plus a `sink` callback /// that forwards bytes to the file writer. Allowed to: @@ -1568,13 +1566,7 @@ class FakeChunkAzureDownloader : public AzureBlobDownloader { FakeChunkAzureDownloader() : AzureBlobDownloader(fl::test::NullLog()) {} protected: - int64_t GetBlobSize(ChunkContext& /*ctx*/) override { - if (get_blob_size_hook) { - get_blob_size_hook(); - } - - return blob_size; - } + int64_t GetBlobSize(ChunkContext& /*ctx*/) override { return blob_size; } void DownloadChunkStreaming(ChunkContext& ctx, int64_t offset, int64_t size, std::vector& scratch, @@ -1606,38 +1598,6 @@ class FakeChunkAzureDownloader : public AzureBlobDownloader { } // namespace -TEST(AzureBlobDownloaderTimeoutTest, DeadlineCancellationWithoutExternalCancellationBecomesNetworkError) { - auto tmpdir = TempPath::CreateTempDir(); - FakeChunkAzureDownloader d; - d.get_blob_size_hook = []() { - throw Azure::Core::OperationCancelledException("deadline exceeded"); - }; - - try { - d.DownloadBlob("", "blob", (tmpdir.path() / "blob.bin").string(), 1); - FAIL() << "expected fl::Exception"; - } catch (const fl::Exception& e) { - EXPECT_EQ(e.code(), FOUNDRY_LOCAL_ERROR_NETWORK); - EXPECT_NE(std::string(e.what()).find("model download timed out after 3 hours"), std::string::npos); - } -} - -TEST(AzureBlobDownloaderTimeoutTest, DeadlineCancellationWithExternalCancellationRemainsCancelled) { - auto tmpdir = TempPath::CreateTempDir(); - FakeChunkAzureDownloader d; - d.get_blob_size_hook = []() { - throw Azure::Core::OperationCancelledException("deadline exceeded"); - }; - std::atomic cancelled{true}; - - try { - d.DownloadBlob("", "blob", (tmpdir.path() / "blob.bin").string(), 1, nullptr, &cancelled); - FAIL() << "expected fl::Exception"; - } catch (const fl::Exception& e) { - EXPECT_EQ(e.code(), FOUNDRY_LOCAL_ERROR_OPERATION_CANCELLED); - } -} - TEST(AzureBlobDownloaderResumeTest, SkipsChunksAlreadyMarkedCompleteInSidecar) { auto tmpdir = TempPath::CreateTempDir(); auto local = tmpdir.path() / "blob.bin";