feat(tangle-cli): add python_pipeline authoring DSL

packages/tangle-cli/src/tangle_cli/component_from_func.py

@@ -181,11 +181,18 @@ def all_outputs(self) -> list[ParamInfo]:
 
 
 def _ensure_cloud_pipelines_shim() -> None:
-    """Register import-time shims used while introspecting authoring files.
-
-    This allows loading Python files that use `from cloud_pipelines import components`
-    and/or TD authoring decorators without requiring those authoring packages.
-    The TD authoring constructs are stripped from generated runtime code later.
+    """Register the import-time ``cloud_pipelines`` shim used while introspecting
+    authoring files.
+
+    This lets us load Python files that use ``from cloud_pipelines import
+    components`` without requiring that authoring package to be installed; the
+    authoring constructs are stripped from the generated runtime code later.
+
+    OSS deliberately does NOT fabricate a shim for any *downstream* authoring
+    surface (e.g. a module a downstream package exposes to re-export the
+    authoring objects under its own import path). A downstream package that
+    wants its own authoring path recognised both makes that module importable
+    itself and registers it via :func:`register_authoring_import_module`.
     """
     if "cloud_pipelines" not in sys.modules:
         components_mod = types.ModuleType("cloud_pipelines.components")
@@ -198,41 +205,6 @@ def _ensure_cloud_pipelines_shim() -> None:
         sys.modules["cloud_pipelines"] = cloud_pipelines_mod
         sys.modules["cloud_pipelines.components"] = components_mod
 
-    _ensure_tangle_deploy_authoring_shim()
-
-
-def _identity_decorator(*args, **kwargs):
-    def decorate(func):
-        return func
-
-    return decorate
-
-
-class _AuthoringGeneric:
-    def __class_getitem__(cls, item):
-        return cls
-
-    def __init__(self, *args, **kwargs):
-        pass
-
-
-def _ensure_tangle_deploy_authoring_shim() -> None:
-    """Register a tiny shim for TD pipeline authoring imports if absent."""
-    if "tangle_deploy.python_pipeline" in sys.modules:
-        return
-
-    tangle_deploy_mod = sys.modules.get("tangle_deploy") or types.ModuleType("tangle_deploy")
-    python_pipeline_mod = types.ModuleType("tangle_deploy.python_pipeline")
-    for name in ("task", "pipeline", "subpipeline", "registered"):
-        setattr(python_pipeline_mod, name, _identity_decorator)
-    for name in ("In", "Out", "Outputs", "TaskEnv"):
-        setattr(python_pipeline_mod, name, _AuthoringGeneric)
-    setattr(python_pipeline_mod, "ref", lambda *args, **kwargs: None)
-
-    setattr(tangle_deploy_mod, "python_pipeline", python_pipeline_mod)
-    sys.modules.setdefault("tangle_deploy", tangle_deploy_mod)
-    sys.modules["tangle_deploy.python_pipeline"] = python_pipeline_mod
-
 
 def load_python_module(file_path: Path, extra_sys_path: list[Path] | None = None) -> Any:
     """Dynamically import a Python module from a file path.
@@ -757,20 +729,47 @@ def _is_main_str(n: ast.expr) -> bool:
 # too, exactly like @task.
 _AUTHORING_DECORATOR_NAMES = frozenset({"task", "pipeline", "subpipeline", "registered"})
 
-# The python-pipeline authoring module. ONLY imports of this module (and its
-# submodules) are authoring-only and stripped from the baked source. We
-# deliberately do NOT strip other ``tangle_deploy.*`` packages (e.g.
-# ``tangle_deploy.utils``): those may be legitimate runtime helpers used inside a
-# ``@task`` body, and dropping them would raise ``NameError`` in the operation
-# container.
-_AUTHORING_IMPORT_MODULE = "tangle_deploy.python_pipeline"
+# The python-pipeline authoring modules. ONLY imports of these modules (and
+# their submodules) are authoring-only and stripped from the baked source. We
+# deliberately do NOT strip other packages that merely share a top-level name
+# (e.g. a downstream ``*.utils``): those may be legitimate runtime helpers used
+# inside a ``@task`` body, and dropping them would raise ``NameError`` in the
+# operation container.
+#
+# OSS recognises exactly one authoring surface out of the box: the canonical
+# ``tangle_cli.python_pipeline`` path. A downstream package that re-exports the
+# authoring objects under its own module path — so authors may write ``from
+# <downstream>.python_pipeline import task`` — registers that path via
+# :func:`register_authoring_import_module`; codegen then strips either import
+# the same way. OSS never hardcodes a downstream module name (the dependency
+# points inward), mirroring the resolver/reader registries in the hydrator.
+_AUTHORING_IMPORT_MODULES: list[str] = ["tangle_cli.python_pipeline"]
+
+
+def register_authoring_import_module(module: str) -> None:
+    """Register *module* as an additional python-pipeline authoring surface.
+
+    A downstream package that re-exports the ``tangle_cli.python_pipeline``
+    authoring objects under its own module path calls this (typically at import
+    time) so codegen strips ``from <module> import ...`` / ``import <module>``
+    lines — and their submodules — from baked runtime source exactly like the
+    canonical OSS surface. Idempotent: registering an already-known module is a
+    no-op, so repeated import-time registration is safe.
+    """
+    if module not in _AUTHORING_IMPORT_MODULES:
+        _AUTHORING_IMPORT_MODULES.append(module)
+
+
+def authoring_import_modules() -> tuple[str, ...]:
+    """Return the python-pipeline authoring modules recognised by codegen."""
+    return tuple(_AUTHORING_IMPORT_MODULES)
 
 # The authoring-only ``TaskEnv`` class name. A module-level ``X = TaskEnv(...)``
 # (or ``X = <alias>.TaskEnv(...)``) declaration is authoring-only by contract and
 # is stripped from the baked source by ``_strip_authoring_constructs``.
 # Matched by trailing NAME only (like the authoring decorators), because in
-# python-pipeline authoring files ``TaskEnv`` always
-# resolves to ``tangle_deploy.python_pipeline.TaskEnv``.
+# python-pipeline authoring files ``TaskEnv`` always resolves to the
+# python-pipeline authoring surface's ``TaskEnv``.
 _AUTHORING_ENV_CLASS_NAME = "TaskEnv"
 
 
@@ -793,7 +792,7 @@ def _decorator_called_name(node: ast.expr) -> str | None:
 
     Handles ``@name`` / ``@name(...)`` and ``@mod.name`` / ``@mod.name(...)``
     forms, returning the trailing attribute/name (e.g. ``task`` for both
-    ``@task(...)`` and ``@tangle_deploy.python_pipeline.task(...)``). Returns
+    ``@task(...)`` and ``@tangle_cli.python_pipeline.task(...)``). Returns
     ``None`` for shapes we do not recognise so callers leave them untouched.
 
     Limitation (v1, intentional): matching is by trailing NAME only, not by
@@ -812,33 +811,36 @@ def _decorator_called_name(node: ast.expr) -> str | None:
     return None
 
 
+def _is_authoring_module(name: str) -> bool:
+    """Return True if *name* is an authoring module or a submodule of one."""
+    return any(name == mod or name.startswith(mod + ".") for mod in _AUTHORING_IMPORT_MODULES)
+
+
 def _is_authoring_import(node: ast.stmt) -> bool:
     """Return True if *node* imports the python-pipeline authoring surface.
 
-    Matches ONLY the ``tangle_deploy.python_pipeline`` module (and its
-    submodules):
+    Matches ONLY the registered authoring modules (and their submodules) — the
+    canonical ``tangle_cli.python_pipeline`` plus any registered via
+    :func:`register_authoring_import_module`:
 
-    - ``from tangle_deploy.python_pipeline import ...`` (including the aliased
-      ``from tangle_deploy.python_pipeline import ref as operation_by_ref`` form
-      and submodules like ``from tangle_deploy.python_pipeline.x import y``);
-    - ``import tangle_deploy.python_pipeline`` / ``import
-      tangle_deploy.python_pipeline as tp``.
+    - ``from tangle_cli.python_pipeline import ...`` (including the aliased
+      ``from tangle_cli.python_pipeline import ref as operation_by_ref`` form
+      and submodules like ``from tangle_cli.python_pipeline.x import y``);
+    - ``import tangle_cli.python_pipeline`` / ``import
+      tangle_cli.python_pipeline as tp``;
+    - the equivalents for any registered downstream authoring path.
 
-    It does NOT match other ``tangle_deploy.*`` packages (e.g.
-    ``from tangle_deploy.utils import X``) — those can be genuine runtime helpers
+    It does NOT match other packages that merely share a top-level name (e.g. a
+    downstream ``*.utils`` module) — those can be genuine runtime helpers
     referenced inside a ``@task`` body and must survive into the baked program.
     Relative imports (``from . import x``) are never authoring imports.
     """
     if isinstance(node, ast.ImportFrom):
         if node.level:  # relative import — not the authoring package
             return False
-        module = node.module or ""
-        return module == _AUTHORING_IMPORT_MODULE or module.startswith(_AUTHORING_IMPORT_MODULE + ".")
+        return _is_authoring_module(node.module or "")
     if isinstance(node, ast.Import):
-        return any(
-            alias.name == _AUTHORING_IMPORT_MODULE or alias.name.startswith(_AUTHORING_IMPORT_MODULE + ".")
-            for alias in node.names
-        )
+        return any(_is_authoring_module(alias.name) for alias in node.names)
     return False
 
 
@@ -971,7 +973,7 @@ def _strip_authoring_constructs(source_code: str) -> str:
     - re-running an ``@task`` / ``@pipeline`` / ``@subpipeline`` decorator
       replaces the function with a ``CallableRef`` recorder, which raises at
       call time because there is no active ``@pipeline`` trace context;
-    - on a thin image the ``from tangle_deploy.python_pipeline import ...``
+    - on a thin image the ``from tangle_cli.python_pipeline import ...``
       import itself can fail with ``ImportError``.
 
     This removes them via surgical AST line-range deletion (mirroring
@@ -986,9 +988,9 @@ def _strip_authoring_constructs(source_code: str) -> str:
 
     Scope of the strip (intentional v1 boundaries):
 
-    - imports: only ``tangle_deploy.python_pipeline`` (and submodules) are
-      dropped — see ``_is_authoring_import``. Other ``tangle_deploy.*`` runtime
-      helpers are preserved.
+    - imports: only the registered authoring modules (and submodules) are
+      dropped — see ``_is_authoring_import``. Other runtime helpers that merely
+      share a top-level name are preserved.
     - decorators: matched by trailing NAME (``task`` / ``pipeline`` /
       ``subpipeline``), not by import resolution — see ``_decorator_called_name``
       for the limitation. Unrelated decorators (``@functools.cache``,
@@ -1686,7 +1688,8 @@ def generate_component_yaml(
         path_annotation_mode: ``"oss"`` always records source/YAML paths relative
             to their common ancestor. ``"td_legacy"`` only uses that relative
             common-root behavior inside a git checkout; outside git it records
-            ``file_path.name`` / ``output_path.name`` like legacy tangle-deploy.
+            ``file_path.name`` / ``output_path.name`` to preserve the legacy
+            downstream driver's historical basename-only snapshots.
 
     Returns:
         True on success, False on failure.
@@ -1764,8 +1767,9 @@ def generate_component_yaml(
         # Use the common ancestor of source and output so both paths are clean
         # forward references (no ".."). This lets later local maintenance
         # commands find the source even when YAML is generated into a separate
-        # output directory. TD legacy compatibility keeps basename-only paths
-        # outside a git checkout to preserve historical snapshots.
+        # output directory. Legacy (``td_legacy``) compatibility keeps
+        # basename-only paths outside a git checkout to preserve historical
+        # snapshots.
         resolved_source = file_path.resolve()
         resolved_output = output_path.resolve()
         common_dir = Path(os.path.commonpath([resolved_source, resolved_output]))

packages/tangle-cli/src/tangle_cli/python_pipeline/__init__.py

@@ -0,0 +1,43 @@
+"""Python-first authoring surface for Tangle pipelines.
+
+End users write::
+
+    from tangle_cli.python_pipeline import pipeline, task, registered, ref, raw, subpipeline, TaskEnv, In, Out
+
+``cfg`` is NOT a top-level export — it is a parameter the framework
+injects into the user's pipeline function at trace time. Importing the
+:class:`tangle_cli.python_pipeline.cfg.Cfg` class is reserved for the
+compile driver.
+
+``import tangle_cli.python_pipeline`` is kept light: it does not
+eagerly import the heavy ``tangle_cli.component_generator`` codegen
+module or the tracer machinery.
+
+Module map: authoring entry points live in :mod:`.pipeline`,
+:mod:`.task`, :mod:`.subpipeline`, :mod:`.registered`, :mod:`.ref` and
+:mod:`.raw`; the trace-time IR is built in :mod:`.trace` / :mod:`.graph`
+and lowered to the dehydrated dict shape by :mod:`.emit`.
+"""
+from __future__ import annotations
+
+from .pipeline import pipeline
+from .raw import raw
+from .ref import ref
+from .registered import registered
+from .subpipeline import subpipeline
+from .task import task
+from .task_env import TaskEnv
+from .types import In, Out, Outputs
+
+__all__ = [
+    "pipeline",
+    "task",
+    "registered",
+    "ref",
+    "raw",
+    "subpipeline",
+    "TaskEnv",
+    "In",
+    "Out",
+    "Outputs",
+]