From 430c6952f11608f7830056d0b2401c400f7d2f35 Mon Sep 17 00:00:00 2001 From: AzureFunctionsJava Date: Wed, 19 Aug 2026 07:59:52 -0500 Subject: [PATCH 1/6] fix(ci): package test apps without pulling from Docker Hub Docker integration tests have failed on dev every night since 2026-08-15 at 'Build and package test apps': docker: Get https://registry-1.docker.io/v2/: context deadline exceeded. The agents can no longer reach Docker Hub, and build-apps.ps1 pulled ubuntu:22.04 purely to run mksquashfs. No code change caused this; dev has not moved since 08-12. Run mksquashfs on the agent instead, and install squashfs-tools in the job. This also drops the apt-get that ran inside the container. The Docker path is kept as a fallback for machines without squashfs-tools (e.g. Windows dev boxes) and now points at the MCR mirror rather than Docker Hub. Everything else in dockertests already used MCR (azurite, mesh); ubuntu:22.04 was the only Docker Hub dependency. --- dockertests/build-apps.ps1 | 55 +++++++++++-------- .../templates/jobs/run-docker-tests-linux.yml | 9 +++ 2 files changed, 41 insertions(+), 23 deletions(-) diff --git a/dockertests/build-apps.ps1 b/dockertests/build-apps.ps1 index 89e1e1a5..ec30f365 100644 --- a/dockertests/build-apps.ps1 +++ b/dockertests/build-apps.ps1 @@ -125,30 +125,39 @@ foreach ($appDir in $appDirs) { Write-Host "[INFO] Source: $functionAppPath" -ForegroundColor Gray Write-Host "[INFO] Output: $packagePath" -ForegroundColor Gray - # Create squashfs package using Docker - # Use Ubuntu image and install squashfs-tools on the fly - # Mount both source and destination directories - $dockerArgs = @( - "run" - "--rm" - "-v" - "${functionAppPath}:/source" - "-v" - "${packageDir}:/output" - "ubuntu:22.04" - "bash" - "-c" - "apt-get update -qq && apt-get install -y -qq squashfs-tools > /dev/null 2>&1 && mksquashfs /source /output/$packageFileName -noappend -comp gzip" - ) - - Write-Host "[INFO] Running Docker container to create squashfs package..." -ForegroundColor DarkGray - - $dockerOutput = & docker @dockerArgs 2>&1 - $dockerExitCode = $LASTEXITCODE - - if ($dockerExitCode -ne 0) { + # Prefer mksquashfs on the host. The CI agents cannot reach Docker Hub, and running the + # tool directly also avoids an apt-get from inside the container. + $mksquashfs = Get-Command mksquashfs -ErrorAction SilentlyContinue + + if ($mksquashfs) { + Write-Host "[INFO] Creating squashfs package with mksquashfs..." -ForegroundColor DarkGray + $packageOutput = & mksquashfs $functionAppPath $packagePath -noappend -comp gzip 2>&1 + $packageExitCode = $LASTEXITCODE + } + else { + # Fallback for machines without squashfs-tools (e.g. Windows dev boxes). Uses the MCR + # mirror because Docker Hub is not reachable from CI. + Write-Host "[INFO] mksquashfs not found; falling back to Docker..." -ForegroundColor DarkGray + $dockerArgs = @( + "run" + "--rm" + "-v" + "${functionAppPath}:/source" + "-v" + "${packageDir}:/output" + "mcr.microsoft.com/mirror/docker/library/ubuntu:22.04" + "bash" + "-c" + "apt-get update -qq && apt-get install -y -qq squashfs-tools > /dev/null 2>&1 && mksquashfs /source /output/$packageFileName -noappend -comp gzip" + ) + + $packageOutput = & docker @dockerArgs 2>&1 + $packageExitCode = $LASTEXITCODE + } + + if ($packageExitCode -ne 0) { Write-Host "[ERROR] Failed to create squashfs package for $appName" -ForegroundColor Red - Write-Host $dockerOutput -ForegroundColor DarkGray + Write-Host $packageOutput -ForegroundColor DarkGray $failCount++ continue } diff --git a/eng/ci/templates/jobs/run-docker-tests-linux.yml b/eng/ci/templates/jobs/run-docker-tests-linux.yml index 44a5bd8e..28e2eb50 100644 --- a/eng/ci/templates/jobs/run-docker-tests-linux.yml +++ b/eng/ci/templates/jobs/run-docker-tests-linux.yml @@ -47,6 +47,15 @@ jobs: df -h / displayName: 'Free up disk space' + # build-apps.ps1 packages the test apps with mksquashfs. Provide it on the agent so the + # script does not fall back to pulling an image from Docker Hub, which CI cannot reach. + - bash: | + set -e + sudo apt-get update + sudo apt-get install -y squashfs-tools + mksquashfs -version | head -1 + displayName: 'Install squashfs-tools' + - bash: | mvn --version displayName: 'Check Maven is installed' From 1e1ce472c48f8a691eb0858226664a98d136cb7f Mon Sep 17 00:00:00 2001 From: AzureFunctionsJava Date: Wed, 19 Aug 2026 13:08:40 -0500 Subject: [PATCH 2/6] ci: move emulated Linux tests to 1es-ubuntu-24.04-min The 22.04 image boots ~92% full on / (6.5G free), which is why dotnet test kept filling the disk and killing agents (exit 134, reported as a cancelled job). The workaround was to rm -rf Android/Haskell/Swift/CodeQL at job start. 1es-ubuntu-24.04-min starts at 15% used (62G free), so that cleanup is no longer needed here. -min ships no Maven and no Node, so install both. The PreInstalled JDK step is dropped (/usr/lib/jvm is empty on -min, and the matrix JDK is downloaded a few steps later anyway), and packaging runs with -skipNuget because the nuget CLI needs mono: these tests read the worker from target/, never the .nupkg. Docker integration tests stay on 22.04: that job resolves JDKs 8/11/17/21 via PreInstalled and uses UsePythonVersion, which -min cannot satisfy without a much larger change. --- .../jobs/run-emulated-tests-linux.yml | 38 ++++++++----------- 1 file changed, 15 insertions(+), 23 deletions(-) diff --git a/eng/ci/templates/jobs/run-emulated-tests-linux.yml b/eng/ci/templates/jobs/run-emulated-tests-linux.yml index edb97da5..6991e142 100644 --- a/eng/ci/templates/jobs/run-emulated-tests-linux.yml +++ b/eng/ci/templates/jobs/run-emulated-tests-linux.yml @@ -9,7 +9,7 @@ jobs: pool: name: ${{ parameters.poolName }} - image: 1es-ubuntu-22.04 + image: 1es-ubuntu-24.04-min os: linux variables: @@ -45,6 +45,16 @@ jobs: JAVA_VERSION_SPEC: '25' steps: + # The -min image ships no Maven and no Node, so install them up front. + - bash: | + set -e + sudo apt-get update + sudo apt-get install -y maven + displayName: 'Install Maven' + - task: NodeTool@0 + inputs: + versionSpec: '20.x' + displayName: 'Install Node.js' # Maven resolves plugins and extensions before a pom's repositories are honored. Install the # mirror before MavenAuthenticate@0, which adds credentials to the same settings file. - pwsh: | @@ -57,31 +67,11 @@ jobs: inputs: artifactsFeeds: upstream-public - bash: | - echo "=== disk BEFORE cleanup ===" df -h / - echo "=== top consumers on / (baseline) ===" - sudo du -xh --max-depth=2 / 2>/dev/null | sort -rh | head -20 || true - echo "=== freeing space (toolsets this job does not use) ===" - sudo rm -rf /usr/local/lib/android /opt/ghc /usr/local/.ghcup /usr/share/swift \ - /opt/hostedtoolcache/CodeQL /opt/hostedtoolcache/PyPy 2>/dev/null || true - docker image prune -af || true - sudo apt-get clean || true - echo "=== disk AFTER cleanup ===" - df -h / - displayName: 'Free up disk space' - - task: NuGetToolInstaller@1 - inputs: - checkLatest: true - displayName: 'Install NuGet Tool' + displayName: 'Report disk space' - pwsh: | Get-Command mvn displayName: 'Check Maven is installed' - - task: JavaToolInstaller@0 # This step is necessary as Linux image has Java 11 as default - inputs: - versionSpec: '8' - jdkArchitectureOption: 'x64' - jdkSourceOption: 'PreInstalled' - displayName: 'Setup Java for Linux' - pwsh: | java -version displayName: 'Check default java version' @@ -124,7 +114,9 @@ jobs: Write-Host "git tag not found. Setting package suffix to '$buildNumber'" } Write-Host "##vso[task.setvariable variable=buildNumber;isOutput=true;]$buildNumber" - .\package-pipeline.ps1 -buildNumber $buildNumber + # -skipNuget: these tests consume the worker from target/, never the .nupkg, and the + # nuget CLI needs mono, which the -min image does not ship. + .\package-pipeline.ps1 -buildNumber $buildNumber -skipNuget displayName: 'Executing build script' - pwsh: | ./eng/scripts/Export-JavaCacerts.ps1 -OutputPath "./emulatedtests/confluent_cloud_cacert.pem" From 611513f67dabd05d8372828eb1bc78ecc2413c24 Mon Sep 17 00:00:00 2001 From: AzureFunctionsJava Date: Wed, 19 Aug 2026 17:43:09 -0500 Subject: [PATCH 3/6] chore: remove installMavenPluginLocally.ps1 (clones a personal fork) The script cloned https://github.com/ahmedmuhsin/azure-maven-plugins (a personal fork) at branch sdk-types and ran mvn clean install, publishing the result into the local ~/.m2 repository that subsequent worker builds resolve from. That is untrusted build code landing in the build environment (CWE-829). Nothing references it: no pipeline, script, or doc mentions installMavenPluginLocally, so it is not reachable from CI and removing it changes no build behaviour. The sibling installAdditionsLocally.ps1 shows the intended convention, cloning the official Azure/azure-functions-java-additions repo. --- installMavenPluginLocally.ps1 | 20 -------------------- 1 file changed, 20 deletions(-) delete mode 100644 installMavenPluginLocally.ps1 diff --git a/installMavenPluginLocally.ps1 b/installMavenPluginLocally.ps1 deleted file mode 100644 index e221640d..00000000 --- a/installMavenPluginLocally.ps1 +++ /dev/null @@ -1,20 +0,0 @@ -# Variables for second repository -$repoUrl2 = 'https://github.com/ahmedmuhsin/azure-maven-plugins.git' -$branchName2 = 'sdk-types' -$repoName2 = 'azure-maven-plugins' - -# Clone the second repository -git clone $repoUrl2 - -# Change directory to the cloned repository -Set-Location $repoName2 - -# Checkout the desired branch -git checkout $branchName2 - -# Run Maven command to build/install, skipping tests and javadoc -if ($IsWindows) { - & "mvn" "clean" "install" "-DskipTests" "-Dmaven.javadoc.skip=true" -} else { - bash -c "mvn clean install -DskipTests -Dmaven.javadoc.skip=true" -} From cace7cf75ad4a97c495765bcc76cda31618c7f6e Mon Sep 17 00:00:00 2001 From: AzureFunctionsJava Date: Thu, 20 Aug 2026 11:56:04 -0500 Subject: [PATCH 4/6] ci: skip NuGet packaging in emulated Windows tests too Same reasoning as the Linux job: setup-tests-pipeline.ps1 copies the worker from target/, so the .nupkg these tests build is never read. Only build-artifacts.yml publishes packages. Drops the NuGet tool installer that existed to serve the pack step; 'Authenticate NuGet to CFS' and the dotnet test restore do not depend on it, as the Linux job already demonstrates. --- eng/ci/templates/jobs/run-emulated-tests-windows.yml | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/eng/ci/templates/jobs/run-emulated-tests-windows.yml b/eng/ci/templates/jobs/run-emulated-tests-windows.yml index 4c8490d6..00f52087 100644 --- a/eng/ci/templates/jobs/run-emulated-tests-windows.yml +++ b/eng/ci/templates/jobs/run-emulated-tests-windows.yml @@ -56,10 +56,6 @@ jobs: displayName: 'Authenticate Maven to CFS' inputs: artifactsFeeds: upstream-public - - task: NuGetToolInstaller@1 - inputs: - checkLatest: true - displayName: 'Install NuGet Tool' - pwsh: | Get-Command mvn displayName: 'Check Maven is installed' @@ -105,7 +101,8 @@ jobs: Write-Host "git tag not found. Setting package suffix to '$buildNumber'" } Write-Host "##vso[task.setvariable variable=buildNumber;isOutput=true;]$buildNumber" - .\package-pipeline.ps1 -buildNumber $buildNumber + # -skipNuget: these tests consume the worker from target/, never the .nupkg. + .\package-pipeline.ps1 -buildNumber $buildNumber -skipNuget displayName: 'Executing build script' - pwsh: | ./eng/scripts/Export-JavaCacerts.ps1 -OutputPath "./emulatedtests/confluent_cloud_cacert.pem" From d24c6564ecbf75f390595b931237fc1a6a403575 Mon Sep 17 00:00:00 2001 From: AzureFunctionsJava Date: Thu, 20 Aug 2026 12:08:01 -0500 Subject: [PATCH 5/6] ci: move docker integration tests to 1es-ubuntu-24.04-min Removes the last disk-cleanup step. On the docker legs that step cost 75-119s each (~100s avg, roughly 12% of a 10-16 min leg, ~6.7 min of agent time per build) purely to make room on an image that ships 67G of toolsets these tests never use. -min starts at 15% used, so nothing needs deleting. Adapting to the leaner image: install Maven, squashfs-tools and python3-venv up front; download the matrix JDK and install it with JavaToolInstaller LocalDirectory, since /usr/lib/jvm is empty on -min and PreInstalled resolves nothing; and replace UsePythonVersion (no Python in the -min tool cache) with the system python3, which satisfies the test kit's requires-python >= 3.8. pip installs into a venv because the system interpreter on 24.04 is externally managed (PEP 668). The docker daemon is present on -min, so the tests themselves are unaffected. --- .../templates/jobs/run-docker-tests-linux.yml | 68 +++++++++++-------- 1 file changed, 40 insertions(+), 28 deletions(-) diff --git a/eng/ci/templates/jobs/run-docker-tests-linux.yml b/eng/ci/templates/jobs/run-docker-tests-linux.yml index 28e2eb50..220e3615 100644 --- a/eng/ci/templates/jobs/run-docker-tests-linux.yml +++ b/eng/ci/templates/jobs/run-docker-tests-linux.yml @@ -9,22 +9,41 @@ jobs: pool: name: ${{ parameters.poolName }} - image: 1es-ubuntu-22.04 + image: 1es-ubuntu-24.04-min os: linux + variables: + - template: ../java-versions.yml + strategy: maxParallel: 4 matrix: Java-8: javaVersion: '8' + JDK_DOWNLOAD_LINK: 'https://github.com/adoptium/temurin8-binaries/releases/download/jdk8u$(JDK8_LINUX_VERSION)-b$(JDK8_LINUX_BUILD)/OpenJDK8U-jdk_x64_linux_hotspot_8u$(JDK8_LINUX_VERSION)b$(JDK8_LINUX_BUILD).tar.gz' + JAVA_VERSION: 'OpenJDK8U-jdk_x64_linux_hotspot_8u$(JDK8_LINUX_VERSION)b$(JDK8_LINUX_BUILD)' Java-11: javaVersion: '11' + JDK_DOWNLOAD_LINK: 'https://aka.ms/download-jdk/microsoft-jdk-$(JDK11_LINUX_VERSION)-linux-x64.tar.gz' + JAVA_VERSION: 'microsoft-jdk-$(JDK11_LINUX_VERSION)-linux-x64' Java-17: javaVersion: '17' + JDK_DOWNLOAD_LINK: 'https://aka.ms/download-jdk/microsoft-jdk-$(JDK17_LINUX_VERSION)-linux-x64.tar.gz' + JAVA_VERSION: 'microsoft-jdk-$(JDK17_LINUX_VERSION)-linux-x64' Java-21: javaVersion: '21' + JDK_DOWNLOAD_LINK: 'https://aka.ms/download-jdk/microsoft-jdk-$(JDK21_LINUX_VERSION)-linux-x64.tar.gz' + JAVA_VERSION: 'microsoft-jdk-$(JDK21_LINUX_VERSION)-linux-x64' steps: + # The -min image ships no Maven. squashfs-tools backs the packaging in build-apps.ps1, and + # python3-venv provides the venv module the pip steps below rely on. + - bash: | + set -e + sudo apt-get update + sudo apt-get install -y maven squashfs-tools python3-venv + displayName: 'Install build tools' + # Maven resolves plugins and extensions before a pom's repositories are honored. Install the # mirror before MavenAuthenticate@0, which adds credentials to the same settings file. - pwsh: | @@ -36,26 +55,6 @@ jobs: displayName: 'Authenticate Maven to CFS' inputs: artifactsFeeds: upstream-public - - bash: | - echo "=== disk BEFORE cleanup ===" - df -h / - echo "=== freeing space (toolsets this job does not use) ===" - sudo rm -rf /usr/local/lib/android /opt/ghc /usr/local/.ghcup /usr/share/swift \ - /opt/hostedtoolcache/CodeQL /opt/hostedtoolcache/PyPy 2>/dev/null || true - sudo apt-get clean || true - echo "=== disk AFTER cleanup ===" - df -h / - displayName: 'Free up disk space' - - # build-apps.ps1 packages the test apps with mksquashfs. Provide it on the agent so the - # script does not fall back to pulling an image from Docker Hub, which CI cannot reach. - - bash: | - set -e - sudo apt-get update - sudo apt-get install -y squashfs-tools - mksquashfs -version | head -1 - displayName: 'Install squashfs-tools' - - bash: | mvn --version displayName: 'Check Maven is installed' @@ -64,18 +63,31 @@ jobs: java -version displayName: 'Check default java version' - - task: JavaToolInstaller@0 + - pwsh: | # Download JDK for later installation + Invoke-WebRequest $(JDK_DOWNLOAD_LINK) -OutFile "$(JAVA_VERSION).tar.gz" + $current = get-location | select -ExpandProperty Path + Write-Host "##vso[task.setvariable variable=downloadPath;]$current" + displayName: 'Download jdk for Linux' + + - task: JavaToolInstaller@0 # Install the JDK downloaded above; -min has no preinstalled JDKs displayName: 'Install Java $(javaVersion)' inputs: versionSpec: '$(javaVersion)' jdkArchitectureOption: 'x64' - jdkSourceOption: 'PreInstalled' + jdkSourceOption: LocalDirectory + jdkFile: "$(downloadPath)/$(JAVA_VERSION).tar.gz" + jdkDestinationDirectory: "$(downloadPath)/externals" + cleanDestinationDirectory: true - - task: UsePythonVersion@0 - displayName: 'Use Python 3.11' - inputs: - versionSpec: '3.11' - addToPath: true + # -min carries no Python in the tool cache, so UsePythonVersion cannot resolve one. The + # system python3 satisfies the test kit (requires-python >= 3.8); install into a venv so + # pip is not refused by PEP 668 on the externally managed system interpreter. + - bash: | + set -e + python3 -m venv "$(Agent.TempDirectory)/venv" + echo "##vso[task.prependpath]$(Agent.TempDirectory)/venv/bin" + python3 --version + displayName: 'Create Python virtual environment' - task: PipAuthenticate@1 displayName: 'Authenticate pip to CFS' From d94100f0278ba6b5b8e7f7b735f72a3cb678ded8 Mon Sep 17 00:00:00 2001 From: AzureFunctionsJava Date: Thu, 20 Aug 2026 12:29:42 -0500 Subject: [PATCH 6/6] ci: note why the docker JDK vendors differ by version Temurin for 8, Microsoft OpenJDK from 11 up, matching production and the emulated jobs. Worth recording because the previous PreInstalled lookup read JAVA_HOME__X64 from the agent image, which supplied Temurin for every version, so the 11+ legs had quietly diverged from what we ship. --- eng/ci/templates/jobs/run-docker-tests-linux.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/eng/ci/templates/jobs/run-docker-tests-linux.yml b/eng/ci/templates/jobs/run-docker-tests-linux.yml index 220e3615..3055a240 100644 --- a/eng/ci/templates/jobs/run-docker-tests-linux.yml +++ b/eng/ci/templates/jobs/run-docker-tests-linux.yml @@ -17,6 +17,9 @@ jobs: strategy: maxParallel: 4 + # JDK vendors mirror production and the emulated jobs: Temurin for 8, Microsoft OpenJDK + # from 11 up. The previous PreInstalled lookup resolved to the agent image's Temurin for + # every version, so 11+ silently diverged from what we ship. matrix: Java-8: javaVersion: '8'