From 3319ee3cdcbab697d09e62ed858f4b3d63058d2b Mon Sep 17 00:00:00 2001 From: AzureFunctionsJava Date: Thu, 13 Aug 2026 16:14:50 -0500 Subject: [PATCH 1/5] ci: try 1es-ubuntu-24.04-min for Linux jobs instead of disk cleanup EXPERIMENT (do not merge as-is). The 1es-ubuntu-22.04 image boots ~92% full on /, which forced the 'Free up disk space' rm -rf/prune steps added in #883. Try the leaner 1es-ubuntu-24.04-min image instead and drop the cleanup, keeping a df/du report so we can see the new baseline headroom. --- eng/ci/templates/jobs/run-docker-tests-linux.yml | 14 +++++--------- .../templates/jobs/run-emulated-tests-linux.yml | 15 ++++----------- 2 files changed, 9 insertions(+), 20 deletions(-) diff --git a/eng/ci/templates/jobs/run-docker-tests-linux.yml b/eng/ci/templates/jobs/run-docker-tests-linux.yml index 44a5bd8e..23b1bc43 100644 --- a/eng/ci/templates/jobs/run-docker-tests-linux.yml +++ b/eng/ci/templates/jobs/run-docker-tests-linux.yml @@ -9,7 +9,7 @@ jobs: pool: name: ${{ parameters.poolName }} - image: 1es-ubuntu-22.04 + image: 1es-ubuntu-24.04-min os: linux strategy: @@ -37,15 +37,11 @@ jobs: inputs: artifactsFeeds: upstream-public - bash: | - echo "=== disk BEFORE cleanup ===" + echo "=== disk space (no cleanup: measuring 1es-ubuntu-24.04-min baseline) ===" df -h / - echo "=== freeing space (toolsets this job does not use) ===" - sudo rm -rf /usr/local/lib/android /opt/ghc /usr/local/.ghcup /usr/share/swift \ - /opt/hostedtoolcache/CodeQL /opt/hostedtoolcache/PyPy 2>/dev/null || true - sudo apt-get clean || true - echo "=== disk AFTER cleanup ===" - df -h / - displayName: 'Free up disk space' + echo "=== top consumers on / ===" + sudo du -xh --max-depth=2 / 2>/dev/null | sort -rh | head -20 || true + displayName: 'Report disk space' - bash: | mvn --version diff --git a/eng/ci/templates/jobs/run-emulated-tests-linux.yml b/eng/ci/templates/jobs/run-emulated-tests-linux.yml index edb97da5..e2e4b83f 100644 --- a/eng/ci/templates/jobs/run-emulated-tests-linux.yml +++ b/eng/ci/templates/jobs/run-emulated-tests-linux.yml @@ -9,7 +9,7 @@ jobs: pool: name: ${{ parameters.poolName }} - image: 1es-ubuntu-22.04 + image: 1es-ubuntu-24.04-min os: linux variables: @@ -57,18 +57,11 @@ jobs: inputs: artifactsFeeds: upstream-public - bash: | - echo "=== disk BEFORE cleanup ===" + echo "=== disk space (no cleanup: measuring 1es-ubuntu-24.04-min baseline) ===" df -h / - echo "=== top consumers on / (baseline) ===" + echo "=== top consumers on / ===" sudo du -xh --max-depth=2 / 2>/dev/null | sort -rh | head -20 || true - echo "=== freeing space (toolsets this job does not use) ===" - sudo rm -rf /usr/local/lib/android /opt/ghc /usr/local/.ghcup /usr/share/swift \ - /opt/hostedtoolcache/CodeQL /opt/hostedtoolcache/PyPy 2>/dev/null || true - docker image prune -af || true - sudo apt-get clean || true - echo "=== disk AFTER cleanup ===" - df -h / - displayName: 'Free up disk space' + displayName: 'Report disk space' - task: NuGetToolInstaller@1 inputs: checkLatest: true From 7a2817bed268381dbe7abf52b6ca13839cf770e6 Mon Sep 17 00:00:00 2001 From: AzureFunctionsJava Date: Fri, 14 Aug 2026 15:26:20 -0500 Subject: [PATCH 2/5] ci: probe toolchain on 1es-ubuntu-24.04-min First run died at 'Check Maven is installed' (exit 127) so only the mvn gap is known. Replace the disk-only report with a probe that reports every tool these jobs need (mvn, java/javac, docker + daemon, python3, node/npm, dotnet) plus the tool cache and JDK dirs, so one run reveals the full gap. Always exits 0. --- .../templates/jobs/run-docker-tests-linux.yml | 18 ++++++++++++++---- .../jobs/run-emulated-tests-linux.yml | 18 ++++++++++++++---- 2 files changed, 28 insertions(+), 8 deletions(-) diff --git a/eng/ci/templates/jobs/run-docker-tests-linux.yml b/eng/ci/templates/jobs/run-docker-tests-linux.yml index 23b1bc43..9eb8fef8 100644 --- a/eng/ci/templates/jobs/run-docker-tests-linux.yml +++ b/eng/ci/templates/jobs/run-docker-tests-linux.yml @@ -37,11 +37,21 @@ jobs: inputs: artifactsFeeds: upstream-public - bash: | - echo "=== disk space (no cleanup: measuring 1es-ubuntu-24.04-min baseline) ===" + set +e + echo "=== disk ===" df -h / - echo "=== top consumers on / ===" - sudo du -xh --max-depth=2 / 2>/dev/null | sort -rh | head -20 || true - displayName: 'Report disk space' + echo "=== toolchain ===" + for t in mvn java javac docker python3 node npm dotnet; do + printf '%-10s %s\n' "$t" "$(command -v $t 2>/dev/null || echo MISSING)" + done + echo "=== /opt/hostedtoolcache ===" + ls -1 /opt/hostedtoolcache 2>/dev/null || echo "(absent)" + echo "=== JDKs (/usr/lib/jvm) ===" + ls -1 /usr/lib/jvm 2>/dev/null || echo "(none)" + echo "=== docker daemon ===" + docker info >/dev/null 2>&1 && echo "available" || echo "NOT available" + exit 0 + displayName: 'Probe: disk + toolchain' - bash: | mvn --version diff --git a/eng/ci/templates/jobs/run-emulated-tests-linux.yml b/eng/ci/templates/jobs/run-emulated-tests-linux.yml index e2e4b83f..cddd86f3 100644 --- a/eng/ci/templates/jobs/run-emulated-tests-linux.yml +++ b/eng/ci/templates/jobs/run-emulated-tests-linux.yml @@ -57,11 +57,21 @@ jobs: inputs: artifactsFeeds: upstream-public - bash: | - echo "=== disk space (no cleanup: measuring 1es-ubuntu-24.04-min baseline) ===" + set +e + echo "=== disk ===" df -h / - echo "=== top consumers on / ===" - sudo du -xh --max-depth=2 / 2>/dev/null | sort -rh | head -20 || true - displayName: 'Report disk space' + echo "=== toolchain ===" + for t in mvn java javac docker python3 node npm dotnet; do + printf '%-10s %s\n' "$t" "$(command -v $t 2>/dev/null || echo MISSING)" + done + echo "=== /opt/hostedtoolcache ===" + ls -1 /opt/hostedtoolcache 2>/dev/null || echo "(absent)" + echo "=== JDKs (/usr/lib/jvm) ===" + ls -1 /usr/lib/jvm 2>/dev/null || echo "(none)" + echo "=== docker daemon ===" + docker info >/dev/null 2>&1 && echo "available" || echo "NOT available" + exit 0 + displayName: 'Probe: disk + toolchain' - task: NuGetToolInstaller@1 inputs: checkLatest: true From d239f64fb296c130068bf27c8e3fcc59fcaf1bcb Mon Sep 17 00:00:00 2001 From: AzureFunctionsJava Date: Tue, 18 Aug 2026 17:03:09 -0500 Subject: [PATCH 3/5] ci: try plain 1es-ubuntu-24.04 instead of -min The -min image has the headroom (62G free vs 6.5G on 22.04) but ships no mvn/java/node/npm and an empty /usr/lib/jvm, so JavaToolInstaller PreInstalled has nothing to find. Try the non-min 24.04 image, which should carry the full toolchain and may still be leaner than 22.04. Probe step reports the result. --- eng/ci/templates/jobs/run-docker-tests-linux.yml | 2 +- eng/ci/templates/jobs/run-emulated-tests-linux.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/eng/ci/templates/jobs/run-docker-tests-linux.yml b/eng/ci/templates/jobs/run-docker-tests-linux.yml index 9eb8fef8..21571706 100644 --- a/eng/ci/templates/jobs/run-docker-tests-linux.yml +++ b/eng/ci/templates/jobs/run-docker-tests-linux.yml @@ -9,7 +9,7 @@ jobs: pool: name: ${{ parameters.poolName }} - image: 1es-ubuntu-24.04-min + image: 1es-ubuntu-24.04 os: linux strategy: diff --git a/eng/ci/templates/jobs/run-emulated-tests-linux.yml b/eng/ci/templates/jobs/run-emulated-tests-linux.yml index cddd86f3..4aaa5083 100644 --- a/eng/ci/templates/jobs/run-emulated-tests-linux.yml +++ b/eng/ci/templates/jobs/run-emulated-tests-linux.yml @@ -9,7 +9,7 @@ jobs: pool: name: ${{ parameters.poolName }} - image: 1es-ubuntu-24.04-min + image: 1es-ubuntu-24.04 os: linux variables: From 701fc1c69c2a6766072954e2d74a297db5d49699 Mon Sep 17 00:00:00 2001 From: AzureFunctionsJava Date: Tue, 18 Aug 2026 17:30:09 -0500 Subject: [PATCH 4/5] ci: move emulated Linux tests to 1es-ubuntu-24.04-min Plain 1es-ubuntu-24.04 does not exist in the pool, so -min is the only leaner option. Scope it to the emulated job, which is where the disk exhaustion actually killed agents. -min starts at 15% used (62G free) vs 22.04's 92% (6.5G), so the rm -rf/prune cleanup is no longer needed here. -min ships no Maven and no Node, so install both explicitly. The PreInstalled JDK step is dropped: /usr/lib/jvm is empty on -min, and it only set a default before the matrix JDK is installed from JDK_DOWNLOAD_LINK a few steps later. Docker integration tests stay on 1es-ubuntu-22.04 with the existing cleanup: that job resolves JDKs 8/11/17/21 via PreInstalled and uses UsePythonVersion, which -min cannot satisfy without a much larger change. --- .../templates/jobs/run-docker-tests-linux.yml | 24 +++++--------- .../jobs/run-emulated-tests-linux.yml | 33 +++++++------------ 2 files changed, 21 insertions(+), 36 deletions(-) diff --git a/eng/ci/templates/jobs/run-docker-tests-linux.yml b/eng/ci/templates/jobs/run-docker-tests-linux.yml index 21571706..44a5bd8e 100644 --- a/eng/ci/templates/jobs/run-docker-tests-linux.yml +++ b/eng/ci/templates/jobs/run-docker-tests-linux.yml @@ -9,7 +9,7 @@ jobs: pool: name: ${{ parameters.poolName }} - image: 1es-ubuntu-24.04 + image: 1es-ubuntu-22.04 os: linux strategy: @@ -37,21 +37,15 @@ jobs: inputs: artifactsFeeds: upstream-public - bash: | - set +e - echo "=== disk ===" + echo "=== disk BEFORE cleanup ===" df -h / - echo "=== toolchain ===" - for t in mvn java javac docker python3 node npm dotnet; do - printf '%-10s %s\n' "$t" "$(command -v $t 2>/dev/null || echo MISSING)" - done - echo "=== /opt/hostedtoolcache ===" - ls -1 /opt/hostedtoolcache 2>/dev/null || echo "(absent)" - echo "=== JDKs (/usr/lib/jvm) ===" - ls -1 /usr/lib/jvm 2>/dev/null || echo "(none)" - echo "=== docker daemon ===" - docker info >/dev/null 2>&1 && echo "available" || echo "NOT available" - exit 0 - displayName: 'Probe: disk + toolchain' + echo "=== freeing space (toolsets this job does not use) ===" + sudo rm -rf /usr/local/lib/android /opt/ghc /usr/local/.ghcup /usr/share/swift \ + /opt/hostedtoolcache/CodeQL /opt/hostedtoolcache/PyPy 2>/dev/null || true + sudo apt-get clean || true + echo "=== disk AFTER cleanup ===" + df -h / + displayName: 'Free up disk space' - bash: | mvn --version diff --git a/eng/ci/templates/jobs/run-emulated-tests-linux.yml b/eng/ci/templates/jobs/run-emulated-tests-linux.yml index 4aaa5083..cb98f976 100644 --- a/eng/ci/templates/jobs/run-emulated-tests-linux.yml +++ b/eng/ci/templates/jobs/run-emulated-tests-linux.yml @@ -9,7 +9,7 @@ jobs: pool: name: ${{ parameters.poolName }} - image: 1es-ubuntu-24.04 + image: 1es-ubuntu-24.04-min os: linux variables: @@ -45,6 +45,16 @@ jobs: JAVA_VERSION_SPEC: '25' steps: + # The -min image ships no Maven and no Node, so install them up front. + - bash: | + set -e + sudo apt-get update + sudo apt-get install -y maven + displayName: 'Install Maven' + - task: NodeTool@0 + inputs: + versionSpec: '20.x' + displayName: 'Install Node.js' # Maven resolves plugins and extensions before a pom's repositories are honored. Install the # mirror before MavenAuthenticate@0, which adds credentials to the same settings file. - pwsh: | @@ -57,21 +67,8 @@ jobs: inputs: artifactsFeeds: upstream-public - bash: | - set +e - echo "=== disk ===" df -h / - echo "=== toolchain ===" - for t in mvn java javac docker python3 node npm dotnet; do - printf '%-10s %s\n' "$t" "$(command -v $t 2>/dev/null || echo MISSING)" - done - echo "=== /opt/hostedtoolcache ===" - ls -1 /opt/hostedtoolcache 2>/dev/null || echo "(absent)" - echo "=== JDKs (/usr/lib/jvm) ===" - ls -1 /usr/lib/jvm 2>/dev/null || echo "(none)" - echo "=== docker daemon ===" - docker info >/dev/null 2>&1 && echo "available" || echo "NOT available" - exit 0 - displayName: 'Probe: disk + toolchain' + displayName: 'Report disk space' - task: NuGetToolInstaller@1 inputs: checkLatest: true @@ -79,12 +76,6 @@ jobs: - pwsh: | Get-Command mvn displayName: 'Check Maven is installed' - - task: JavaToolInstaller@0 # This step is necessary as Linux image has Java 11 as default - inputs: - versionSpec: '8' - jdkArchitectureOption: 'x64' - jdkSourceOption: 'PreInstalled' - displayName: 'Setup Java for Linux' - pwsh: | java -version displayName: 'Check default java version' From d47726907fd9382323820aebe0ca90d41de40d4e Mon Sep 17 00:00:00 2001 From: AzureFunctionsJava Date: Tue, 18 Aug 2026 17:54:27 -0500 Subject: [PATCH 5/5] ci: skip NuGet packaging in emulated Linux tests (-min has no mono) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The build script failed with '/opt/hostedtoolcache/NuGet/7.9.0/x64/nuget: 2: mono: not found' — the nuget CLI is a .NET Framework exe needing mono, which 1es-ubuntu-24.04-min does not ship. The emulated tests never consume the .nupkg: setup-tests-pipeline.ps1 copies the worker from target/ and only build-artifacts.yml publishes packages. So pass -skipNuget (as the docker test job already does) instead of installing mono, and drop the now-unused NuGetToolInstaller step. --- eng/ci/templates/jobs/run-emulated-tests-linux.yml | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/eng/ci/templates/jobs/run-emulated-tests-linux.yml b/eng/ci/templates/jobs/run-emulated-tests-linux.yml index cb98f976..6991e142 100644 --- a/eng/ci/templates/jobs/run-emulated-tests-linux.yml +++ b/eng/ci/templates/jobs/run-emulated-tests-linux.yml @@ -69,10 +69,6 @@ jobs: - bash: | df -h / displayName: 'Report disk space' - - task: NuGetToolInstaller@1 - inputs: - checkLatest: true - displayName: 'Install NuGet Tool' - pwsh: | Get-Command mvn displayName: 'Check Maven is installed' @@ -118,7 +114,9 @@ jobs: Write-Host "git tag not found. Setting package suffix to '$buildNumber'" } Write-Host "##vso[task.setvariable variable=buildNumber;isOutput=true;]$buildNumber" - .\package-pipeline.ps1 -buildNumber $buildNumber + # -skipNuget: these tests consume the worker from target/, never the .nupkg, and the + # nuget CLI needs mono, which the -min image does not ship. + .\package-pipeline.ps1 -buildNumber $buildNumber -skipNuget displayName: 'Executing build script' - pwsh: | ./eng/scripts/Export-JavaCacerts.ps1 -OutputPath "./emulatedtests/confluent_cloud_cacert.pem"