From b6a9f33df7037016b6ca6f6f61b28ea13ac66d57 Mon Sep 17 00:00:00 2001 From: Rohan Dsouza Date: Tue, 21 Jul 2026 23:11:46 +0530 Subject: [PATCH 1/2] fix: migrate create_table's upload path to the presigned upload flow runtimedb PR #952 removed the legacy POST /v1/files endpoint in favor of the presigned direct-to-storage flow (POST /v1/uploads -> PUT -> finalize). hotdata-ibis's create_table/upload_file called the now-dead endpoint via the SDK's generated UploadsApi.upload_file, breaking every write against any runtimedb deployment past that commit (confirmed on production). Switches to hotdata.uploads.UploadsApi (the SDK's own hand-written orchestration of the presigned flow: session create, storage PUT, finalize, with retries/multipart handling) instead of hand-rolling the flow here. No SDK version bump needed -- hotdata.uploads is already present across the whole >=0.7,<0.9 pin range. upload_file's return value now carries upload_id (FinalizeUploadResponse) instead of id (the old UploadResponse). Test fixtures updated to mock the three-stage presigned flow instead of the single POST /v1/files call. Verified: full offline suite (101 tests) against the updated flow, and live against production (api.hotdata.dev, where /v1/files is confirmed 404) -- examples 01, 03, 04, 05, and the vector-search example all pass end to end, including the create_database -> create_table -> query -> drop_database write/read cycle. --- CHANGELOG.md | 7 ++++ src/ibis_hotdata/backend.py | 5 +-- src/ibis_hotdata/http.py | 26 +++++++++++--- tests/conftest.py | 67 +++++++++++++++++++++++++++++++++++ tests/test_hotdata_backend.py | 63 +++----------------------------- tests/test_hotdata_http.py | 37 +++++++------------ 6 files changed, 116 insertions(+), 89 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 59dd82e..6792c55 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,13 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +### Fixed + +- `create_table` (and thus `upload_file`) now uses the `hotdata` SDK's presigned + direct-to-storage upload flow (`hotdata.uploads.UploadsApi`) instead of the + removed `POST /v1/files` endpoint (`runtimedb` #952). `upload_file`'s return + value now carries `upload_id` (was `id`) to match the new + `FinalizeUploadResponse` shape. ## [0.3.2] - 2026-07-20 diff --git a/src/ibis_hotdata/backend.py b/src/ibis_hotdata/backend.py index 1e48dad..08c49e8 100644 --- a/src/ibis_hotdata/backend.py +++ b/src/ibis_hotdata/backend.py @@ -557,7 +557,8 @@ def to_pyarrow_batches( ) def upload_file(self, data: bytes, *, content_type: str | None = None) -> dict[str, Any]: - """POST ``/v1/files``; returns the upload record (use ``id`` with managed table loads).""" + """Direct-to-storage presigned upload; returns the finalized upload record + (use ``upload_id`` with managed table loads).""" try: return self._http.upload_file(data, content_type=content_type) except HotdataAPIError as exc: @@ -695,7 +696,7 @@ def create_table( connection_id, schema_name, name, - upload_id=upload["id"], + upload_id=upload["upload_id"], ) except HotdataAPIError as exc: raise _ibis_err_from_hotdata(exc) from exc diff --git a/src/ibis_hotdata/http.py b/src/ibis_hotdata/http.py index 1638e7e..b4c544f 100644 --- a/src/ibis_hotdata/http.py +++ b/src/ibis_hotdata/http.py @@ -19,7 +19,6 @@ QueryApi, QueryRunsApi, ResultsApi, - UploadsApi, ) from hotdata.api.databases_api import DatabasesApi from hotdata.exceptions import ApiException @@ -29,6 +28,7 @@ from hotdata.models.database_default_schema_decl import DatabaseDefaultSchemaDecl from hotdata.models.database_default_table_decl import DatabaseDefaultTableDecl from hotdata.models.load_managed_table_request import LoadManagedTableRequest +from hotdata.uploads import UploadError, UploadsApi T = TypeVar("T") @@ -64,6 +64,13 @@ def _from_api_exception(exc: ApiException) -> HotdataAPIError: return HotdataAPIError(msg.strip(), status_code=exc.status, body=exc.body) +def _from_upload_error(exc: UploadError) -> HotdataAPIError: + """Map the presigned-upload flow's ``UploadError`` (session/storage/finalize + failures) onto our own error type, same shape as ``_from_api_exception``. + """ + return HotdataAPIError(f"Hotdata upload error: {exc}", status_code=getattr(exc, "status", None)) + + def _ipc_stream_bytes_to_table(data: bytes) -> pa.Table: with pa_ipc.open_stream(io.BytesIO(data)) as reader: return reader.read_all() @@ -185,10 +192,19 @@ def execute_query( raise HotdataAPIError("Unexpected query response type") def upload_file(self, data: bytes, *, content_type: str | None = None) -> dict[str, Any]: - kwargs: dict[str, Any] = {} - if content_type is not None: - kwargs["_content_type"] = content_type - resp = self._safe_call(self._uploads.upload_file, data, **kwargs) + """Direct-to-storage presigned upload: create session, ``PUT``, finalize. + + Returns the finalized upload record (``upload_id`` is what managed-table + loads need) -- see ``hotdata.uploads.UploadsApi.upload_file``. + """ + try: + resp = self._uploads.upload_file( + data, content_type=content_type, request_timeout=self._timeout + ) + except UploadError as exc: + raise _from_upload_error(exc) from exc + except ApiException as exc: + raise _from_api_exception(exc) from exc return resp.model_dump(by_alias=True, mode="json") def list_databases(self) -> dict[str, Any]: diff --git a/tests/conftest.py b/tests/conftest.py index bd5951e..73069ed 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -1,9 +1,13 @@ from __future__ import annotations +import json +from collections.abc import Callable + import pytest pytest.importorskip("pytest_httpserver") from pytest_httpserver import HTTPServer +from werkzeug.wrappers import Request, Response @pytest.fixture(autouse=True) @@ -26,3 +30,66 @@ def srv(httpserver: HTTPServer) -> str: """Base URL without trailing slash (matches Hotdata client normalization).""" return httpserver.url_for("/").rstrip("/") + + +def mock_presigned_upload_flow( + httpserver: HTTPServer, + *, + upload_id: str = "upl_1", + finalize_token: str = "tok_1", + on_create_session: Callable[[Request], Response] | None = None, + on_storage_put: Callable[[Request], Response] | None = None, + on_finalize: Callable[[Request], Response] | None = None, +) -> None: + """Mock ``hotdata.uploads.UploadsApi.upload_file``'s presigned flow: a single- + ``PUT`` upload session (``POST /v1/uploads``), the storage ``PUT`` itself, and + finalize (``POST /v1/uploads/{upload_id}/finalize``). Each stage accepts an + override handler for tests that need to assert on that stage's request. + """ + storage_path = f"/mock-storage/{upload_id}" + + def default_create_session(req: Request) -> Response: + return Response( + json.dumps( + { + "mode": "single", + "url": httpserver.url_for(storage_path), + "headers": {}, + "upload_id": upload_id, + "finalize_token": finalize_token, + } + ), + status=201, + content_type="application/json", + ) + + httpserver.expect_oneshot_request("/v1/uploads", method="POST").respond_with_handler( + on_create_session or default_create_session + ) + + def default_storage_put(req: Request) -> Response: + return Response(b"", status=200) + + httpserver.expect_oneshot_request(storage_path, method="PUT").respond_with_handler( + on_storage_put or default_storage_put + ) + + def default_finalize(req: Request) -> Response: + assert req.headers.get("X-Upload-Finalize-Token") == finalize_token + return Response( + json.dumps( + { + "upload_id": upload_id, + "status": "ready", + "size_bytes": len(req.get_data()), + "created_at": "2026-01-01T00:00:00Z", + "content_type": "application/parquet", + } + ), + status=200, + content_type="application/json", + ) + + httpserver.expect_oneshot_request( + f"/v1/uploads/{upload_id}/finalize", method="POST" + ).respond_with_handler(on_finalize or default_finalize) diff --git a/tests/test_hotdata_backend.py b/tests/test_hotdata_backend.py index 4f2fd88..4f8c31f 100644 --- a/tests/test_hotdata_backend.py +++ b/tests/test_hotdata_backend.py @@ -14,6 +14,7 @@ from werkzeug.wrappers import Request, Response pytest.importorskip("pytest_httpserver") +from conftest import mock_presigned_upload_flow from pytest_httpserver import HTTPServer # Managed database identifiers for mocked Hotdata (SQL shape ``sales.public.orders``). @@ -106,25 +107,7 @@ def mock_managed_create_table_flow( managed_database_detail_response() ) - def default_upload(req: Request) -> Response: - assert req.headers["Content-Type"] == "application/parquet" - return Response( - json.dumps( - { - "id": "upl_1", - "status": "ready", - "size_bytes": len(req.get_data()), - "created_at": "2026-01-01T00:00:00Z", - "content_type": "application/parquet", - } - ), - status=201, - content_type="application/json", - ) - - httpserver.expect_request("/v1/files", method="POST").respond_with_handler( - on_upload or default_upload - ) + mock_presigned_upload_flow(httpserver, upload_id="upl_1", on_storage_put=on_upload) def on_load(req: Request) -> Response: body = req.get_json() @@ -378,19 +361,7 @@ def test_create_table_from_pandas_uploads_managed_table(httpserver: HTTPServer, def on_upload(req: Request) -> Response: uploaded["table"] = pq.read_table(io.BytesIO(req.get_data())) - return Response( - json.dumps( - { - "id": "upl_1", - "status": "ready", - "size_bytes": len(req.get_data()), - "created_at": "2026-01-01T00:00:00Z", - "content_type": "application/parquet", - } - ), - status=201, - content_type="application/json", - ) + return Response(b"", status=200) mock_managed_create_table_flow( httpserver, @@ -421,19 +392,7 @@ def test_create_table_from_pyarrow_uploads_managed_table(httpserver: HTTPServer, def on_upload(req: Request) -> Response: uploaded["table"] = pq.read_table(io.BytesIO(req.get_data())) - return Response( - json.dumps( - { - "id": "upl_1", - "status": "ready", - "size_bytes": len(req.get_data()), - "created_at": "2026-01-01T00:00:00Z", - "content_type": "application/parquet", - } - ), - status=201, - content_type="application/json", - ) + return Response(b"", status=200) mock_managed_create_table_flow( httpserver, @@ -461,19 +420,7 @@ def test_create_table_schema_only_uploads_empty_parquet(httpserver: HTTPServer, def on_upload(req: Request) -> Response: uploaded["table"] = pq.read_table(io.BytesIO(req.get_data())) - return Response( - json.dumps( - { - "id": "upl_1", - "status": "ready", - "size_bytes": len(req.get_data()), - "created_at": "2026-01-01T00:00:00Z", - "content_type": "application/parquet", - } - ), - status=201, - content_type="application/json", - ) + return Response(b"", status=200) mock_managed_create_table_flow( httpserver, diff --git a/tests/test_hotdata_http.py b/tests/test_hotdata_http.py index dc056ab..c00b253 100644 --- a/tests/test_hotdata_http.py +++ b/tests/test_hotdata_http.py @@ -6,6 +6,7 @@ import pyarrow as pa import pyarrow.ipc as ipc import pytest +from conftest import mock_presigned_upload_flow from pytest_httpserver import HTTPServer from werkzeug.wrappers import Request, Response @@ -229,19 +230,7 @@ def test_list_connections_raises_on_http_error(httpserver: HTTPServer): def test_upload_file_then_load_managed_table(httpserver: HTTPServer): - httpserver.expect_oneshot_request( - "/v1/files", - method="POST", - ).respond_with_json( - { - "id": "upl_1", - "status": "ready", - "size_bytes": 3, - "created_at": "2026-01-01T00:00:00Z", - "content_type": "application/parquet", - }, - status=201, - ) + mock_presigned_upload_flow(httpserver, upload_id="upl_1") def on_load(req: Request) -> Response: body = req.get_json() @@ -267,12 +256,12 @@ def on_load(req: Request) -> Response: verify_ssl=False, ) up = client.upload_file(b"parquet", content_type="application/parquet") - assert up["id"] == "upl_1" + assert up["upload_id"] == "upl_1" loaded = client.load_managed_table( "conn_sales", "public", "demo_tbl", - upload_id=up["id"], + upload_id=up["upload_id"], ) assert loaded["table_name"] == "demo_tbl" client.close() @@ -338,23 +327,23 @@ def test_delete_managed_table_and_database(httpserver: HTTPServer): def test_upload_file_accepts_content_type(httpserver: HTTPServer): - def on_upload(req: Request) -> Response: - assert req.headers["Content-Type"] == "application/parquet" + def on_create_session(req: Request) -> Response: + assert req.get_json()["content_type"] == "application/parquet" return Response( json.dumps( { - "id": "upl_1", - "status": "ready", - "size_bytes": len(req.get_data()), - "created_at": "2026-01-01T00:00:00Z", - "content_type": "application/parquet", + "mode": "single", + "url": httpserver.url_for("/mock-storage/upl_1"), + "headers": {}, + "upload_id": "upl_1", + "finalize_token": "tok_1", } ), status=201, content_type="application/json", ) - httpserver.expect_oneshot_request("/v1/files", method="POST").respond_with_handler(on_upload) + mock_presigned_upload_flow(httpserver, upload_id="upl_1", on_create_session=on_create_session) client = HotdataClient( api_url=httpserver.url_for("/").rstrip("/"), @@ -363,5 +352,5 @@ def on_upload(req: Request) -> Response: verify_ssl=False, ) out = client.upload_file(b"parquet", content_type="application/parquet") - assert out["id"] == "upl_1" + assert out["upload_id"] == "upl_1" client.close() From 06a49605f834348cbfe38c5ab1c31b759e3a0e7d Mon Sep 17 00:00:00 2001 From: Rohan Dsouza Date: Wed, 22 Jul 2026 10:48:24 +0530 Subject: [PATCH 2/2] docs: restore blank line between CHANGELOG sections The conflict resolution merging #40 dropped the blank line between the Fixed and Added sections under Unreleased. --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 127bd16..63843db 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -14,6 +14,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 removed `POST /v1/files` endpoint (`runtimedb` #952). `upload_file`'s return value now carries `upload_id` (was `id`) to match the new `FinalizeUploadResponse` shape. + ### Added - `ibis_hotdata.vector`: `cosine_distance`, `l2_distance`, `negative_dot_product`