fix(ci): unbreak Linux agents (no Docker Hub, emulated tests on 24.04-min)

dockertests/build-apps.ps1

@@ -125,30 +125,39 @@ foreach ($appDir in $appDirs) {
         Write-Host "[INFO] Source: $functionAppPath" -ForegroundColor Gray
         Write-Host "[INFO] Output: $packagePath" -ForegroundColor Gray
 
-        # Create squashfs package using Docker
-        # Use Ubuntu image and install squashfs-tools on the fly
-        # Mount both source and destination directories
-        $dockerArgs = @(
-            "run"
-            "--rm"
-            "-v"
-            "${functionAppPath}:/source"
-            "-v"
-            "${packageDir}:/output"
-            "ubuntu:22.04"
-            "bash"
-            "-c"
-            "apt-get update -qq && apt-get install -y -qq squashfs-tools > /dev/null 2>&1 && mksquashfs /source /output/$packageFileName -noappend -comp gzip"
-        )
-
-        Write-Host "[INFO] Running Docker container to create squashfs package..." -ForegroundColor DarkGray
-
-        $dockerOutput = & docker @dockerArgs 2>&1
-        $dockerExitCode = $LASTEXITCODE
-
-        if ($dockerExitCode -ne 0) {
+        # Prefer mksquashfs on the host. The CI agents cannot reach Docker Hub, and running the
+        # tool directly also avoids an apt-get from inside the container.
+        $mksquashfs = Get-Command mksquashfs -ErrorAction SilentlyContinue
+
+        if ($mksquashfs) {
+            Write-Host "[INFO] Creating squashfs package with mksquashfs..." -ForegroundColor DarkGray
+            $packageOutput = & mksquashfs $functionAppPath $packagePath -noappend -comp gzip 2>&1
+            $packageExitCode = $LASTEXITCODE
+        }
+        else {
+            # Fallback for machines without squashfs-tools (e.g. Windows dev boxes). Uses the MCR
+            # mirror because Docker Hub is not reachable from CI.
+            Write-Host "[INFO] mksquashfs not found; falling back to Docker..." -ForegroundColor DarkGray
+            $dockerArgs = @(
+                "run"
+                "--rm"
+                "-v"
+                "${functionAppPath}:/source"
+                "-v"
+                "${packageDir}:/output"
+                "mcr.microsoft.com/mirror/docker/library/ubuntu:22.04"
+                "bash"
+                "-c"
+                "apt-get update -qq && apt-get install -y -qq squashfs-tools > /dev/null 2>&1 && mksquashfs /source /output/$packageFileName -noappend -comp gzip"
+            )
+
+            $packageOutput = & docker @dockerArgs 2>&1
+            $packageExitCode = $LASTEXITCODE
+        }
+
+        if ($packageExitCode -ne 0) {
             Write-Host "[ERROR] Failed to create squashfs package for $appName" -ForegroundColor Red
-            Write-Host $dockerOutput -ForegroundColor DarkGray
+            Write-Host $packageOutput -ForegroundColor DarkGray
             $failCount++
             continue
         }

eng/ci/templates/jobs/run-docker-tests-linux.yml

@@ -9,22 +9,44 @@ jobs:
 
     pool:
       name: ${{ parameters.poolName }}
-      image: 1es-ubuntu-22.04
+      image: 1es-ubuntu-24.04-min
       os: linux
 
+    variables:
+      - template: ../java-versions.yml
+
     strategy:
       maxParallel: 4
+      # JDK vendors mirror production and the emulated jobs: Temurin for 8, Microsoft OpenJDK
+      # from 11 up. The previous PreInstalled lookup resolved to the agent image's Temurin for
+      # every version, so 11+ silently diverged from what we ship.
       matrix:
         Java-8:
           javaVersion: '8'
+          JDK_DOWNLOAD_LINK: 'https://github.com/adoptium/temurin8-binaries/releases/download/jdk8u$(JDK8_LINUX_VERSION)-b$(JDK8_LINUX_BUILD)/OpenJDK8U-jdk_x64_linux_hotspot_8u$(JDK8_LINUX_VERSION)b$(JDK8_LINUX_BUILD).tar.gz'
+          JAVA_VERSION: 'OpenJDK8U-jdk_x64_linux_hotspot_8u$(JDK8_LINUX_VERSION)b$(JDK8_LINUX_BUILD)'
         Java-11:
           javaVersion: '11'
+          JDK_DOWNLOAD_LINK: 'https://aka.ms/download-jdk/microsoft-jdk-$(JDK11_LINUX_VERSION)-linux-x64.tar.gz'
+          JAVA_VERSION: 'microsoft-jdk-$(JDK11_LINUX_VERSION)-linux-x64'
         Java-17:
           javaVersion: '17'
+          JDK_DOWNLOAD_LINK: 'https://aka.ms/download-jdk/microsoft-jdk-$(JDK17_LINUX_VERSION)-linux-x64.tar.gz'
+          JAVA_VERSION: 'microsoft-jdk-$(JDK17_LINUX_VERSION)-linux-x64'
         Java-21:
           javaVersion: '21'
+          JDK_DOWNLOAD_LINK: 'https://aka.ms/download-jdk/microsoft-jdk-$(JDK21_LINUX_VERSION)-linux-x64.tar.gz'
+          JAVA_VERSION: 'microsoft-jdk-$(JDK21_LINUX_VERSION)-linux-x64'
 
     steps:
+      # The -min image ships no Maven. squashfs-tools backs the packaging in build-apps.ps1, and
+      # python3-venv provides the venv module the pip steps below rely on.
+      - bash: |
+          set -e
+          sudo apt-get update
+          sudo apt-get install -y maven squashfs-tools python3-venv
+        displayName: 'Install build tools'
+
       # Maven resolves plugins and extensions before a pom's repositories are honored. Install the
       # mirror before MavenAuthenticate@0, which adds credentials to the same settings file.
       - pwsh: |
@@ -36,17 +58,6 @@ jobs:
         displayName: 'Authenticate Maven to CFS'
         inputs:
           artifactsFeeds: upstream-public
-      - bash: |
-          echo "=== disk BEFORE cleanup ==="
-          df -h /
-          echo "=== freeing space (toolsets this job does not use) ==="
-          sudo rm -rf /usr/local/lib/android /opt/ghc /usr/local/.ghcup /usr/share/swift \
-                      /opt/hostedtoolcache/CodeQL /opt/hostedtoolcache/PyPy 2>/dev/null || true
-          sudo apt-get clean || true
-          echo "=== disk AFTER cleanup ==="
-          df -h /
-        displayName: 'Free up disk space'
-
       - bash: |
           mvn --version
         displayName: 'Check Maven is installed'
@@ -55,18 +66,31 @@ jobs:
           java -version
         displayName: 'Check default java version'
 
-      - task: JavaToolInstaller@0
+      - pwsh: | # Download JDK for later installation
+          Invoke-WebRequest $(JDK_DOWNLOAD_LINK) -OutFile "$(JAVA_VERSION).tar.gz"
+          $current = get-location | select -ExpandProperty Path
+          Write-Host "##vso[task.setvariable variable=downloadPath;]$current"
+        displayName: 'Download jdk for Linux'
+
+      - task: JavaToolInstaller@0 # Install the JDK downloaded above; -min has no preinstalled JDKs
         displayName: 'Install Java $(javaVersion)'
         inputs:
           versionSpec: '$(javaVersion)'
           jdkArchitectureOption: 'x64'
-          jdkSourceOption: 'PreInstalled'
+          jdkSourceOption: LocalDirectory
+          jdkFile: "$(downloadPath)/$(JAVA_VERSION).tar.gz"
+          jdkDestinationDirectory: "$(downloadPath)/externals"
+          cleanDestinationDirectory: true
 
-      - task: UsePythonVersion@0
-        displayName: 'Use Python 3.11'
-        inputs:
-          versionSpec: '3.11'
-          addToPath: true
+      # -min carries no Python in the tool cache, so UsePythonVersion cannot resolve one. The
+      # system python3 satisfies the test kit (requires-python >= 3.8); install into a venv so
+      # pip is not refused by PEP 668 on the externally managed system interpreter.
+      - bash: |
+          set -e
+          python3 -m venv "$(Agent.TempDirectory)/venv"
+          echo "##vso[task.prependpath]$(Agent.TempDirectory)/venv/bin"
+          python3 --version
+        displayName: 'Create Python virtual environment'
 
       - task: PipAuthenticate@1
         displayName: 'Authenticate pip to CFS'

eng/ci/templates/jobs/run-emulated-tests-linux.yml

@@ -9,7 +9,7 @@ jobs:
 
     pool:
       name: ${{ parameters.poolName }}
-      image: 1es-ubuntu-22.04
+      image: 1es-ubuntu-24.04-min
       os: linux
 
     variables:
@@ -45,6 +45,16 @@ jobs:
           JAVA_VERSION_SPEC: '25'
 
     steps:
+      # The -min image ships no Maven and no Node, so install them up front.
+      - bash: |
+          set -e
+          sudo apt-get update
+          sudo apt-get install -y maven
+        displayName: 'Install Maven'
+      - task: NodeTool@0
+        inputs:
+          versionSpec: '20.x'
+        displayName: 'Install Node.js'
       # Maven resolves plugins and extensions before a pom's repositories are honored. Install the
       # mirror before MavenAuthenticate@0, which adds credentials to the same settings file.
       - pwsh: |
@@ -57,31 +67,11 @@ jobs:
         inputs:
           artifactsFeeds: upstream-public
       - bash: |
-          echo "=== disk BEFORE cleanup ==="
           df -h /
-          echo "=== top consumers on / (baseline) ==="
-          sudo du -xh --max-depth=2 / 2>/dev/null | sort -rh | head -20 || true
-          echo "=== freeing space (toolsets this job does not use) ==="
-          sudo rm -rf /usr/local/lib/android /opt/ghc /usr/local/.ghcup /usr/share/swift \
-                      /opt/hostedtoolcache/CodeQL /opt/hostedtoolcache/PyPy 2>/dev/null || true
-          docker image prune -af || true
-          sudo apt-get clean || true
-          echo "=== disk AFTER cleanup ==="
-          df -h /
-        displayName: 'Free up disk space'
-      - task: NuGetToolInstaller@1
-        inputs:
-          checkLatest: true
-        displayName: 'Install NuGet Tool'
+        displayName: 'Report disk space'
       - pwsh: |
           Get-Command mvn
         displayName: 'Check Maven is installed'
-      - task: JavaToolInstaller@0 # This step is necessary as Linux image has Java 11 as default
-        inputs:
-          versionSpec: '8'
-          jdkArchitectureOption: 'x64'
-          jdkSourceOption: 'PreInstalled'
-        displayName: 'Setup Java for Linux'
       - pwsh: |
           java -version
         displayName: 'Check default java version'
@@ -124,7 +114,9 @@ jobs:
             Write-Host "git tag not found. Setting package suffix to '$buildNumber'"
           }
           Write-Host "##vso[task.setvariable variable=buildNumber;isOutput=true;]$buildNumber"
-          .\package-pipeline.ps1 -buildNumber $buildNumber
+          # -skipNuget: these tests consume the worker from target/, never the .nupkg, and the
+          # nuget CLI needs mono, which the -min image does not ship.
+          .\package-pipeline.ps1 -buildNumber $buildNumber -skipNuget
         displayName: 'Executing build script'
       - pwsh: |
           ./eng/scripts/Export-JavaCacerts.ps1 -OutputPath "./emulatedtests/confluent_cloud_cacert.pem"

eng/ci/templates/jobs/run-emulated-tests-windows.yml

@@ -56,10 +56,6 @@ jobs:
         displayName: 'Authenticate Maven to CFS'
         inputs:
           artifactsFeeds: upstream-public
-      - task: NuGetToolInstaller@1
-        inputs:
-          checkLatest: true
-        displayName: 'Install NuGet Tool'
       - pwsh: |
           Get-Command mvn
         displayName: 'Check Maven is installed'
@@ -105,7 +101,8 @@ jobs:
             Write-Host "git tag not found. Setting package suffix to '$buildNumber'"
           }
           Write-Host "##vso[task.setvariable variable=buildNumber;isOutput=true;]$buildNumber"
-          .\package-pipeline.ps1 -buildNumber $buildNumber
+          # -skipNuget: these tests consume the worker from target/, never the .nupkg.
+          .\package-pipeline.ps1 -buildNumber $buildNumber -skipNuget
         displayName: 'Executing build script'
       - pwsh: |
           ./eng/scripts/Export-JavaCacerts.ps1 -OutputPath "./emulatedtests/confluent_cloud_cacert.pem"