python-package.yml

---
name: CI/CD
on:
  push:
    branches: [master]
  pull_request:
    branches: [master]
  # Run the checks on merge-queue candidates too, so the merge queue enabled
  # on `master` can validate each candidate as a `merge_group` event -- the
  # required `check` job must run there or queued PRs stall.
  merge_group:
  release:
    types: [created]
    branches: [master]
env:
  FORCE_COLOR: '1'  # Make tools pretty.
  PIP_DISABLE_PIP_VERSION_CHECK: '1'
  PIP_NO_PYTHON_VERSION_WARNING: '1'
  PYTHON_LATEST: '3.12'
# One run per pull request at a time: a new push supersedes the run of the
# commit it replaces, so the ~45 job-minutes of a full matrix are not spent
# validating a commit nobody will ever merge.
#
# Cancellation is deliberately limited to `pull_request`.  Every other event
# gets its own group and always runs to completion: a cancelled `merge_group`
# candidate would stall the merge queue, a cancelled `release` would skip
# publishing, and `push` runs on master are what seed the caches everything
# else restores from.
concurrency:
  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
  cancel-in-progress: ${{ github.event_name == 'pull_request' }}
jobs:
  lint:
    name: Check linting
    runs-on: ubuntu-latest
    steps:
      - name: Checkout project
        uses: actions/checkout@v4
        with:
          fetch-depth: 0
      - name: Set up Python
        uses: actions/setup-python@v5
        with:
          python-version: ${{ env.PYTHON_LATEST }}
          cache: pip
          # Hash every requirements file the workflow installs from, not just
          # the default `**/requirements.txt`.  The pins this job actually
          # installs live in test.txt, which pulls requirements.txt and several
          # extras/ files in via `-r`; with the default pattern the key never
          # moved when those changed, so the entry could neither be refreshed
          # (a hit never re-saves) nor serve the packages that were added.
          # Every job here uses the same list so they all share one entry per
          # interpreter instead of each warming a partial one.
          cache-dependency-path: |
            requirements/*.txt
            requirements/extras/*.txt
      - run: |
          python -m pip install build
          python -m pip install -r requirements/test.txt
        name: Install core libraries for build and install
      - name: Run linting checks
        run: scripts/check
  test-pytest:
    name: 'Python ${{ matrix.python-version }}/Cython: ${{ matrix.use-cython }}/Driver: ${{ matrix.kafka-driver }}'
    runs-on: ubuntu-latest
    timeout-minutes: 10  # Maybe we should remove this someday but the PyPy tests are acting strange
    strategy:
      # Complete all jobs even if one fails, allows us to see
      # for example if a test fails only when Cython is enabled
      fail-fast: false
      matrix:
        python-version: ['3.10', '3.11', '3.12', '3.13', '3.14']
        use-cython: ['true', 'false']
        experimental: [false]
        # aiokafka is the default driver and a core dependency, so it runs the
        # full grid.  The confluent driver lives behind the optional
        # `ckafka` extra; give it a dedicated leg (below) so its tests --
        # tests/unit/transport/drivers/test_confluent.py, otherwise skipped for
        # a missing confluent_kafka -- actually run.
        kafka-driver: ['aiokafka']
        include:
          # Python 3.15.  `allow-prereleases` below resolves this to whatever
          # pre-release the runner image publishes -- 3.15.0rc1 at the time of
          # writing, and 3.15.0 final once it ships, with no change needed
          # here.  Same shape as a stable row: Cython on and off, plus a
          # confluent leg.
          #
          # Advisory (`experimental: true` -> `continue-on-error`) for two
          # reasons.  The pre-release one is the same as everywhere else: rc2
          # and final are still to come.  The specific one is that faust does
          # not import at all on 3.15 until mode-streaming ships the fix for
          # `typing._eval_type`, whose `type_params` argument became a required
          # positional in 3.15 -- every faust Record model resolves its
          # annotations through `mode.utils.objects.annotations`, so collection
          # dies before the first test runs.  The fix is on mode's master; this
          # leg goes green once a mode-streaming release carries it (bump the
          # floor in requirements/requirements.txt then, and move these entries
          # into the matrix above to make 3.15 required).
          - python-version: '3.15'
            use-cython: 'true'
            experimental: true
            kafka-driver: 'aiokafka'
          - python-version: '3.15'
            use-cython: 'false'
            experimental: true
            kafka-driver: 'aiokafka'
          - python-version: '3.15'
            use-cython: 'false'
            experimental: true
            kafka-driver: 'confluent'
          # confluent driver: broker-less unit tests over a pure-Python
          # wrapper, so one leg per Python version (Cython off) is enough.
          - python-version: '3.10'
            use-cython: 'false'
            experimental: false
            kafka-driver: 'confluent'
          - python-version: '3.11'
            use-cython: 'false'
            experimental: false
            kafka-driver: 'confluent'
          - python-version: '3.12'
            use-cython: 'false'
            experimental: false
            kafka-driver: 'confluent'
          - python-version: '3.13'
            use-cython: 'false'
            experimental: false
            kafka-driver: 'confluent'
          - python-version: '3.14'
            use-cython: 'false'
            experimental: false
            kafka-driver: 'confluent'
    env:
      USE_CYTHON: ${{ matrix.use-cython }}
    continue-on-error: ${{ matrix.experimental }}
    steps:
      - uses: actions/checkout@v4
        with:
          fetch-depth: 0
      - uses: actions/setup-python@v5
        with:
          python-version: ${{ matrix.python-version }}
          # Required by the 3.15 rows: a bare `3.15` matches stable releases
          # only and fails with "Version 3.15 was not found in the local
          # cache".  Safe for every other row -- it widens `3.X` to
          # `~3.X.0-0`, and a pre-release only wins when no stable release
          # satisfies the spec, so 3.10-3.14 still resolve to their newest
          # stable patch.
          allow-prereleases: true
          cache: pip
          # See the lint job: test.txt alone leaves the key unchanged when the
          # transitively-included pins move, and the confluent legs below
          # install extras/ckafka.txt on top of it.
          cache-dependency-path: |
            requirements/*.txt
            requirements/extras/*.txt
      - name: Install dependencies
        run: |
          pip install -r requirements/test.txt
          pip install .
          if [ "${{ matrix.kafka-driver }}" = "confluent" ]; then
            pip install -r requirements/extras/ckafka.txt
          fi
      - name: Build the Cython extensions in place
        # `pip install .` above compiles the extensions into site-packages,
        # where the tests never see them: pytest runs from the repository
        # root, so `import faust` resolves to the source tree, and every
        # accelerated import sits behind `try: ... except ImportError`.  The
        # fallback engaged silently, so these legs differed from the
        # `use-cython: false` ones only in whether the build step succeeded --
        # the compiled code itself was never executed by a single test.
        #
        # Building in place puts the .so files next to the .pyx files, which
        # is what the source-tree import actually picks up.
        if: matrix.use-cython == 'true'
        run: USE_CYTHON=1 python setup.py build_ext --inplace
      - name: Run tests
        # FAUST_REQUIRE_CYTHON turns a silent fallback into a failure, so this
        # leg cannot quietly go back to testing pure Python if the build stops
        # producing importable extensions.  See
        # tests/unit/test_cython_parity.py.
        env:
          FAUST_REQUIRE_CYTHON: ${{ matrix.use-cython == 'true' && '1' || '' }}
        run: |
          if [ "${{ matrix.kafka-driver }}" = "confluent" ]; then
            # Dedicated confluent leg: run just the confluent driver's unit
            # tests (the aiokafka legs already cover the rest of the suite).
            pytest tests/unit/transport/drivers/test_confluent.py
          else
            scripts/tests
          fi
      - name: Enforce coverage
        uses: codecov/codecov-action@v5
        with:
          token: ${{ secrets.CODECOV_TOKEN }}
  test-freethreading:
    name: 'Python ${{ matrix.python-version }} (free-threaded)'
    runs-on: ubuntu-latest
    timeout-minutes: 15
    strategy:
      fail-fast: false
      matrix:
        # The two interpreters `[tool.cibuildwheel]` publishes free-threaded
        # wheels for.  Keep the two lists in step: a version we ship a wheel
        # for is a version this job has to cover.
        python-version: ['3.13t', '3.14t']
    steps:
      - uses: actions/checkout@v4
        with:
          fetch-depth: 0
      - uses: actions/setup-python@v5
        with:
          python-version: ${{ matrix.python-version }}
          cache: pip
          cache-dependency-path: |
            requirements/*.txt
            requirements/extras/*.txt
      - name: Install dependencies
        # Not requirements/test.txt: parts of it cannot be built on a
        # free-threaded interpreter at all (twine -> cffi, and hypothesis'
        # PyO3 extension on 3.13t).  freethreading.txt is that list minus the
        # ones that fail, and documents each omission.
        run: |
          pip install -r requirements/freethreading.txt
          pip install 'Cython>=3.1' setuptools setuptools_scm
          # Editable, unlike the other jobs' `pip install .`.  pytest runs from
          # the repo root, so `import faust` resolves to the source tree either
          # way -- but the suite also needs the distribution *metadata* to
          # exist, because `faust/__init__.py` does
          # `version("faust-streaming")` at import time.  An editable install
          # registers that metadata against the tree the tests actually import,
          # instead of a second copy in site-packages that nothing loads.
          USE_CYTHON=1 pip install -e . --no-build-isolation
      - name: Build the Cython extensions in place
        # The extensions have to sit next to the .pyx files or they are never
        # imported: `faust/streams.py` and friends pull their accelerated
        # implementation in behind `try: ... except ImportError`, so a missing
        # .so silently falls back to pure Python and the job would test
        # something other than what it thinks.  This is also what lets
        # tests/unit/test_free_threading.py import the extensions rather than
        # skipping.
        run: USE_CYTHON=1 python setup.py build_ext --inplace
      - name: Verify the extensions did not silently re-enable the GIL
        # Fails loudly if an extension is missing `freethreading_compatible`,
        # rather than leaving it to a RuntimeWarning nobody reads.  Runs
        # before the suite so the cause is obvious when it breaks.
        run: python -m pytest tests/unit/test_free_threading.py -v --no-cov
      - name: Run tests
        # PYTHON_GIL=0 keeps the GIL off for the whole run even if some
        # *dependency* re-enables it (aiokafka's _crecords does, today), so
        # the suite really is exercised without a GIL rather than quietly
        # falling back to one.
        env:
          PYTHON_GIL: '0'
          # As in the main matrix: fail rather than silently fall back to pure
          # Python if the extensions stop being importable from the tree.
          FAUST_REQUIRE_CYTHON: '1'
        run: python -m pytest tests/unit tests/functional -q --no-cov
  test-pypy:
    name: 'Python pypy3.11/Cython: false'
    runs-on: ubuntu-latest
    # PyPy runs the pure-Python paths and is markedly slower than CPython, and
    # this leg now runs the formerly-skipped tests too; give it headroom over
    # the old 10-minute cap, which already clipped the run at ~96%.
    timeout-minutes: 15
    continue-on-error: true
    env:
      USE_CYTHON: 'false'
    steps:
      - uses: actions/checkout@v4
        with:
          fetch-depth: 0
      - uses: actions/setup-python@v5
        with:
          python-version: pypy3.11
          cache: pip
          cache-dependency-path: |
            requirements/*.txt
            requirements/extras/*.txt
      - name: Install dependencies
        id: install
        continue-on-error: true
        run: |
          pip install -r requirements/test.txt
          pip install .
      - name: Run tests
        if: steps.install.outcome == 'success'
        # `scripts/tests` also runs bandit after pytest, and pytest coverage on
        # PyPy is expensive enough to push this advisory leg into the timeout.
        # Run pytest directly without coverage, and stop at the first failure so
        # a red job reports the real test error instead of a timeout.
        run: >
          python -m pytest tests/unit tests/functional tests/integration
          tests/meticulous tests/regression -x --no-cov
  test-integration:
    name: 'Integration (Kafka ${{ matrix.kafka-version }})'
    runs-on: ubuntu-latest
    timeout-minutes: 10
    # Advisory for now: broker tests can be flaky while we stabilise them, so
    # a red here must not block the required checks / the merge queue.  This
    # job is intentionally NOT in the `check` job's `needs`.
    continue-on-error: true
    strategy:
      fail-fast: false
      matrix:
        # Test against both a 3.x broker and Kafka 4.x (KRaft-only, released
        # 2025).  The `KAFKA_CONTROLLER_QUORUM_VOTERS` static-voter config below
        # works for both.
        kafka-version: ['3.8.1', '4.0.0']
    steps:
      - uses: actions/checkout@v4
        with:
          fetch-depth: 0
      - uses: actions/setup-python@v5
        with:
          python-version: '3.12'
          cache: pip
          # This job also installs extras/ckafka.txt, which the old
          # test.txt-only key did not cover.
          cache-dependency-path: |
            requirements/*.txt
            requirements/extras/*.txt
      # Run Kafka as a plain container (not a GH `services:` container) so its
      # very chatty broker log stays inside the container -- retrievable on
      # demand via `docker logs` -- and the job log shows the pytest output.
      - name: Start Kafka (KRaft, single node)
        run: |
          docker run -d --name kafka -p 9092:9092 \
            -e KAFKA_NODE_ID=1 \
            -e KAFKA_PROCESS_ROLES=broker,controller \
            -e KAFKA_LISTENERS=PLAINTEXT://0.0.0.0:9092,CONTROLLER://0.0.0.0:9093 \
            -e KAFKA_ADVERTISED_LISTENERS=PLAINTEXT://localhost:9092 \
            -e KAFKA_CONTROLLER_LISTENER_NAMES=CONTROLLER \
            -e KAFKA_LISTENER_SECURITY_PROTOCOL_MAP=CONTROLLER:PLAINTEXT,PLAINTEXT:PLAINTEXT \
            -e KAFKA_CONTROLLER_QUORUM_VOTERS=1@localhost:9093 \
            -e KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR=1 \
            -e KAFKA_TRANSACTION_STATE_LOG_REPLICATION_FACTOR=1 \
            -e KAFKA_TRANSACTION_STATE_LOG_MIN_ISR=1 \
            -e KAFKA_GROUP_INITIAL_REBALANCE_DELAY_MS=0 \
            -e KAFKA_AUTO_CREATE_TOPICS_ENABLE=true \
            apache/kafka:${{ matrix.kafka-version }}
      - name: Install dependencies
        run: |
          pip install -r requirements/test.txt
          pip install -r requirements/extras/ckafka.txt
          pip install .
      - name: Wait for Kafka to be ready
        run: |
          python - <<'PY'
          import socket, time, sys
          deadline = time.monotonic() + 120
          while time.monotonic() < deadline:
              try:
                  with socket.create_connection(("localhost", 9092), 2):
                      print("Kafka port is open")
                      break
              except OSError:
                  time.sleep(2)
          else:
              sys.exit("Kafka did not become reachable in time")
          PY
      - name: Run live-broker integration tests
        env:
          FAUST_TEST_BROKER: 'kafka://localhost:9092'
        run: pytest tests/integration/broker -v -ra --tb=short --no-cov
      # Only the tail of the broker log on failure, so it augments rather than
      # buries the pytest output in the job log.
      - name: Kafka broker logs (on failure)
        if: failure()
        run: docker logs --tail 40 kafka
  test-redis-integration:
    name: 'Redis cache integration'
    runs-on: ubuntu-latest
    timeout-minutes: 10
    # Advisory for now, like the Kafka integration job: intentionally NOT in
    # the `check` job's `needs`, so a red here does not block the merge queue.
    continue-on-error: true
    services:
      redis:
        image: redis:7
        ports:
          - 6379:6379
        options: >-
          --health-cmd "redis-cli ping"
          --health-interval 5s
          --health-timeout 3s
          --health-retries 5
    steps:
      - uses: actions/checkout@v4
        with:
          fetch-depth: 0
      - uses: actions/setup-python@v5
        with:
          python-version: '3.12'
          cache: pip
          cache-dependency-path: |
            requirements/*.txt
            requirements/extras/*.txt
      - name: Install dependencies
        run: |
          pip install -r requirements/test.txt
          pip install .
      # Runs the redis cache backend against the real server started above --
      # the unit tests only ever mock the client.
      - name: Run redis cache integration tests
        env:
          FAUST_TEST_REDIS: 'redis://localhost:6379'
        run: pytest tests/integration/cache -v -ra --tb=short --no-cov
  check:  # This job does nothing and is only used for the branch protection
    name: ✅ Ensure the required checks passing
    if: always()
    # test-freethreading gates too: `[tool.cibuildwheel]` publishes cp313t and
    # cp314t wheels, and a wheel we ship should not be able to go out on a red
    # run.  (The integration jobs stay out of this list -- they are advisory.)
    needs: [lint, test-pytest, test-freethreading]
    runs-on: ubuntu-latest
    steps:
      - name: Decide whether the needed jobs succeeded or failed
        uses: re-actors/alls-green@release/v1
        with:
          jobs: ${{ toJSON(needs) }}
  build_wheels:
    name: 📦 Build wheels on ${{ matrix.os }}
    runs-on: ${{ matrix.os }}
    needs: check
    if: github.event_name == 'release' && github.event.action == 'created'
    strategy:
      matrix:
        # Modern, supported runner images (ubuntu-20.04 was retired in 2025).
        # macos-15-intel -> x86_64 wheels, macos-14 -> arm64 wheels,
        # windows-2022 -> AMD64 wheels. Build config lives in
        # pyproject.toml's [tool.cibuildwheel] (cp3*, auto64, Cython).
        os: [ubuntu-latest, macos-15-intel, macos-14, windows-2022]
    steps:
      - uses: actions/checkout@v4
        with:
          fetch-depth: 0
      - name: Build wheels
        # cibuildwheel 2.21.3 predates CPython 3.14, so `build = "cp3*"`
        # stopped at cp313 and no 3.14 wheels were published (issue #715).
        # 4.x builds 3.14 by default and still supports cp310-cp313.
        #
        # 4.2.0 is the first pin that knows about CPython 3.15, and it builds
        # cp315 by default -- no `CIBW_ENABLE: cpython-prerelease` needed,
        # because cibuildwheel gates a Python behind that group only until its
        # first release candidate (the ABI is frozen at rc1, so a wheel built
        # against 3.15.0rc1 is forward-compatible with 3.15.0 final).  The
        # `cp31?t-*` skip in pyproject.toml's [tool.cibuildwheel] keeps this
        # to cp315, without cp315t, matching the other versions.
        uses: pypa/cibuildwheel@v4.2.0
      - uses: actions/upload-artifact@v4
        with:
          name: cibw-wheels-${{ matrix.os }}
          path: ./wheelhouse/*.whl
          if-no-files-found: error
  build_sdist:
    name: 📦 Build the source distribution
    runs-on: ubuntu-latest
    needs: check
    if: github.event_name == 'release' && github.event.action == 'created'
    steps:
      - uses: actions/checkout@v4
        name: Checkout source repository
        with:
          fetch-depth: 0
      - uses: actions/setup-python@v5
        with:
          # Pin the interpreter (the step previously took whatever the runner
          # image shipped) so the pip cache below has a stable key.
          python-version: ${{ env.PYTHON_LATEST }}
          cache: pip
          cache-dependency-path: |
            requirements/*.txt
            requirements/extras/*.txt
      - name: Build sdist
        # Use `python -m build`, not the deprecated `setup.py sdist`, so the
        # sdist is named with the normalized project name
        # (`faust_streaming-*.tar.gz`).  PyPI enforces PEP 625 and rejects the
        # legacy hyphenated `faust-streaming-*.tar.gz`.
        run: >
          pip3 install build pkgconfig cython --upgrade &&
          python3 -m build --sdist --outdir dist
      - uses: actions/upload-artifact@v4
        name: Upload build artifacts
        with:
          name: cibw-sdist
          path: dist/*.tar.gz
          if-no-files-found: error
  publish:
    name: 📦 Publish to PyPI
    runs-on: ubuntu-latest
    needs: [build_wheels, build_sdist]
    permissions:
      id-token: write
    environment: pypi
    if: github.event_name == 'release' && github.event.action == 'created'
    steps:
      - name: Download all build artifacts (sdist + every wheel)
        uses: actions/download-artifact@v4
        with:
          pattern: cibw-*
          merge-multiple: true
          path: dist
      - name: Publish package to PyPI
        uses: pypa/gh-action-pypi-publish@release/v1
        with:
          password: ${{ secrets.PYPI_API_TOKEN }}
          skip-existing: true