diff --git a/dockertests/build-apps.ps1 b/dockertests/build-apps.ps1 index 89e1e1a5..ec30f365 100644 --- a/dockertests/build-apps.ps1 +++ b/dockertests/build-apps.ps1 @@ -125,30 +125,39 @@ foreach ($appDir in $appDirs) { Write-Host "[INFO] Source: $functionAppPath" -ForegroundColor Gray Write-Host "[INFO] Output: $packagePath" -ForegroundColor Gray - # Create squashfs package using Docker - # Use Ubuntu image and install squashfs-tools on the fly - # Mount both source and destination directories - $dockerArgs = @( - "run" - "--rm" - "-v" - "${functionAppPath}:/source" - "-v" - "${packageDir}:/output" - "ubuntu:22.04" - "bash" - "-c" - "apt-get update -qq && apt-get install -y -qq squashfs-tools > /dev/null 2>&1 && mksquashfs /source /output/$packageFileName -noappend -comp gzip" - ) - - Write-Host "[INFO] Running Docker container to create squashfs package..." -ForegroundColor DarkGray - - $dockerOutput = & docker @dockerArgs 2>&1 - $dockerExitCode = $LASTEXITCODE - - if ($dockerExitCode -ne 0) { + # Prefer mksquashfs on the host. The CI agents cannot reach Docker Hub, and running the + # tool directly also avoids an apt-get from inside the container. + $mksquashfs = Get-Command mksquashfs -ErrorAction SilentlyContinue + + if ($mksquashfs) { + Write-Host "[INFO] Creating squashfs package with mksquashfs..." -ForegroundColor DarkGray + $packageOutput = & mksquashfs $functionAppPath $packagePath -noappend -comp gzip 2>&1 + $packageExitCode = $LASTEXITCODE + } + else { + # Fallback for machines without squashfs-tools (e.g. Windows dev boxes). Uses the MCR + # mirror because Docker Hub is not reachable from CI. + Write-Host "[INFO] mksquashfs not found; falling back to Docker..." -ForegroundColor DarkGray + $dockerArgs = @( + "run" + "--rm" + "-v" + "${functionAppPath}:/source" + "-v" + "${packageDir}:/output" + "mcr.microsoft.com/mirror/docker/library/ubuntu:22.04" + "bash" + "-c" + "apt-get update -qq && apt-get install -y -qq squashfs-tools > /dev/null 2>&1 && mksquashfs /source /output/$packageFileName -noappend -comp gzip" + ) + + $packageOutput = & docker @dockerArgs 2>&1 + $packageExitCode = $LASTEXITCODE + } + + if ($packageExitCode -ne 0) { Write-Host "[ERROR] Failed to create squashfs package for $appName" -ForegroundColor Red - Write-Host $dockerOutput -ForegroundColor DarkGray + Write-Host $packageOutput -ForegroundColor DarkGray $failCount++ continue } diff --git a/eng/ci/templates/jobs/run-docker-tests-linux.yml b/eng/ci/templates/jobs/run-docker-tests-linux.yml index 44a5bd8e..3055a240 100644 --- a/eng/ci/templates/jobs/run-docker-tests-linux.yml +++ b/eng/ci/templates/jobs/run-docker-tests-linux.yml @@ -9,22 +9,44 @@ jobs: pool: name: ${{ parameters.poolName }} - image: 1es-ubuntu-22.04 + image: 1es-ubuntu-24.04-min os: linux + variables: + - template: ../java-versions.yml + strategy: maxParallel: 4 + # JDK vendors mirror production and the emulated jobs: Temurin for 8, Microsoft OpenJDK + # from 11 up. The previous PreInstalled lookup resolved to the agent image's Temurin for + # every version, so 11+ silently diverged from what we ship. matrix: Java-8: javaVersion: '8' + JDK_DOWNLOAD_LINK: 'https://github.com/adoptium/temurin8-binaries/releases/download/jdk8u$(JDK8_LINUX_VERSION)-b$(JDK8_LINUX_BUILD)/OpenJDK8U-jdk_x64_linux_hotspot_8u$(JDK8_LINUX_VERSION)b$(JDK8_LINUX_BUILD).tar.gz' + JAVA_VERSION: 'OpenJDK8U-jdk_x64_linux_hotspot_8u$(JDK8_LINUX_VERSION)b$(JDK8_LINUX_BUILD)' Java-11: javaVersion: '11' + JDK_DOWNLOAD_LINK: 'https://aka.ms/download-jdk/microsoft-jdk-$(JDK11_LINUX_VERSION)-linux-x64.tar.gz' + JAVA_VERSION: 'microsoft-jdk-$(JDK11_LINUX_VERSION)-linux-x64' Java-17: javaVersion: '17' + JDK_DOWNLOAD_LINK: 'https://aka.ms/download-jdk/microsoft-jdk-$(JDK17_LINUX_VERSION)-linux-x64.tar.gz' + JAVA_VERSION: 'microsoft-jdk-$(JDK17_LINUX_VERSION)-linux-x64' Java-21: javaVersion: '21' + JDK_DOWNLOAD_LINK: 'https://aka.ms/download-jdk/microsoft-jdk-$(JDK21_LINUX_VERSION)-linux-x64.tar.gz' + JAVA_VERSION: 'microsoft-jdk-$(JDK21_LINUX_VERSION)-linux-x64' steps: + # The -min image ships no Maven. squashfs-tools backs the packaging in build-apps.ps1, and + # python3-venv provides the venv module the pip steps below rely on. + - bash: | + set -e + sudo apt-get update + sudo apt-get install -y maven squashfs-tools python3-venv + displayName: 'Install build tools' + # Maven resolves plugins and extensions before a pom's repositories are honored. Install the # mirror before MavenAuthenticate@0, which adds credentials to the same settings file. - pwsh: | @@ -36,17 +58,6 @@ jobs: displayName: 'Authenticate Maven to CFS' inputs: artifactsFeeds: upstream-public - - bash: | - echo "=== disk BEFORE cleanup ===" - df -h / - echo "=== freeing space (toolsets this job does not use) ===" - sudo rm -rf /usr/local/lib/android /opt/ghc /usr/local/.ghcup /usr/share/swift \ - /opt/hostedtoolcache/CodeQL /opt/hostedtoolcache/PyPy 2>/dev/null || true - sudo apt-get clean || true - echo "=== disk AFTER cleanup ===" - df -h / - displayName: 'Free up disk space' - - bash: | mvn --version displayName: 'Check Maven is installed' @@ -55,18 +66,31 @@ jobs: java -version displayName: 'Check default java version' - - task: JavaToolInstaller@0 + - pwsh: | # Download JDK for later installation + Invoke-WebRequest $(JDK_DOWNLOAD_LINK) -OutFile "$(JAVA_VERSION).tar.gz" + $current = get-location | select -ExpandProperty Path + Write-Host "##vso[task.setvariable variable=downloadPath;]$current" + displayName: 'Download jdk for Linux' + + - task: JavaToolInstaller@0 # Install the JDK downloaded above; -min has no preinstalled JDKs displayName: 'Install Java $(javaVersion)' inputs: versionSpec: '$(javaVersion)' jdkArchitectureOption: 'x64' - jdkSourceOption: 'PreInstalled' + jdkSourceOption: LocalDirectory + jdkFile: "$(downloadPath)/$(JAVA_VERSION).tar.gz" + jdkDestinationDirectory: "$(downloadPath)/externals" + cleanDestinationDirectory: true - - task: UsePythonVersion@0 - displayName: 'Use Python 3.11' - inputs: - versionSpec: '3.11' - addToPath: true + # -min carries no Python in the tool cache, so UsePythonVersion cannot resolve one. The + # system python3 satisfies the test kit (requires-python >= 3.8); install into a venv so + # pip is not refused by PEP 668 on the externally managed system interpreter. + - bash: | + set -e + python3 -m venv "$(Agent.TempDirectory)/venv" + echo "##vso[task.prependpath]$(Agent.TempDirectory)/venv/bin" + python3 --version + displayName: 'Create Python virtual environment' - task: PipAuthenticate@1 displayName: 'Authenticate pip to CFS' diff --git a/eng/ci/templates/jobs/run-emulated-tests-linux.yml b/eng/ci/templates/jobs/run-emulated-tests-linux.yml index edb97da5..6991e142 100644 --- a/eng/ci/templates/jobs/run-emulated-tests-linux.yml +++ b/eng/ci/templates/jobs/run-emulated-tests-linux.yml @@ -9,7 +9,7 @@ jobs: pool: name: ${{ parameters.poolName }} - image: 1es-ubuntu-22.04 + image: 1es-ubuntu-24.04-min os: linux variables: @@ -45,6 +45,16 @@ jobs: JAVA_VERSION_SPEC: '25' steps: + # The -min image ships no Maven and no Node, so install them up front. + - bash: | + set -e + sudo apt-get update + sudo apt-get install -y maven + displayName: 'Install Maven' + - task: NodeTool@0 + inputs: + versionSpec: '20.x' + displayName: 'Install Node.js' # Maven resolves plugins and extensions before a pom's repositories are honored. Install the # mirror before MavenAuthenticate@0, which adds credentials to the same settings file. - pwsh: | @@ -57,31 +67,11 @@ jobs: inputs: artifactsFeeds: upstream-public - bash: | - echo "=== disk BEFORE cleanup ===" df -h / - echo "=== top consumers on / (baseline) ===" - sudo du -xh --max-depth=2 / 2>/dev/null | sort -rh | head -20 || true - echo "=== freeing space (toolsets this job does not use) ===" - sudo rm -rf /usr/local/lib/android /opt/ghc /usr/local/.ghcup /usr/share/swift \ - /opt/hostedtoolcache/CodeQL /opt/hostedtoolcache/PyPy 2>/dev/null || true - docker image prune -af || true - sudo apt-get clean || true - echo "=== disk AFTER cleanup ===" - df -h / - displayName: 'Free up disk space' - - task: NuGetToolInstaller@1 - inputs: - checkLatest: true - displayName: 'Install NuGet Tool' + displayName: 'Report disk space' - pwsh: | Get-Command mvn displayName: 'Check Maven is installed' - - task: JavaToolInstaller@0 # This step is necessary as Linux image has Java 11 as default - inputs: - versionSpec: '8' - jdkArchitectureOption: 'x64' - jdkSourceOption: 'PreInstalled' - displayName: 'Setup Java for Linux' - pwsh: | java -version displayName: 'Check default java version' @@ -124,7 +114,9 @@ jobs: Write-Host "git tag not found. Setting package suffix to '$buildNumber'" } Write-Host "##vso[task.setvariable variable=buildNumber;isOutput=true;]$buildNumber" - .\package-pipeline.ps1 -buildNumber $buildNumber + # -skipNuget: these tests consume the worker from target/, never the .nupkg, and the + # nuget CLI needs mono, which the -min image does not ship. + .\package-pipeline.ps1 -buildNumber $buildNumber -skipNuget displayName: 'Executing build script' - pwsh: | ./eng/scripts/Export-JavaCacerts.ps1 -OutputPath "./emulatedtests/confluent_cloud_cacert.pem" diff --git a/eng/ci/templates/jobs/run-emulated-tests-windows.yml b/eng/ci/templates/jobs/run-emulated-tests-windows.yml index 4c8490d6..00f52087 100644 --- a/eng/ci/templates/jobs/run-emulated-tests-windows.yml +++ b/eng/ci/templates/jobs/run-emulated-tests-windows.yml @@ -56,10 +56,6 @@ jobs: displayName: 'Authenticate Maven to CFS' inputs: artifactsFeeds: upstream-public - - task: NuGetToolInstaller@1 - inputs: - checkLatest: true - displayName: 'Install NuGet Tool' - pwsh: | Get-Command mvn displayName: 'Check Maven is installed' @@ -105,7 +101,8 @@ jobs: Write-Host "git tag not found. Setting package suffix to '$buildNumber'" } Write-Host "##vso[task.setvariable variable=buildNumber;isOutput=true;]$buildNumber" - .\package-pipeline.ps1 -buildNumber $buildNumber + # -skipNuget: these tests consume the worker from target/, never the .nupkg. + .\package-pipeline.ps1 -buildNumber $buildNumber -skipNuget displayName: 'Executing build script' - pwsh: | ./eng/scripts/Export-JavaCacerts.ps1 -OutputPath "./emulatedtests/confluent_cloud_cacert.pem" diff --git a/installMavenPluginLocally.ps1 b/installMavenPluginLocally.ps1 deleted file mode 100644 index e221640d..00000000 --- a/installMavenPluginLocally.ps1 +++ /dev/null @@ -1,20 +0,0 @@ -# Variables for second repository -$repoUrl2 = 'https://github.com/ahmedmuhsin/azure-maven-plugins.git' -$branchName2 = 'sdk-types' -$repoName2 = 'azure-maven-plugins' - -# Clone the second repository -git clone $repoUrl2 - -# Change directory to the cloned repository -Set-Location $repoName2 - -# Checkout the desired branch -git checkout $branchName2 - -# Run Maven command to build/install, skipping tests and javadoc -if ($IsWindows) { - & "mvn" "clean" "install" "-DskipTests" "-Dmaven.javadoc.skip=true" -} else { - bash -c "mvn clean install -DskipTests -Dmaven.javadoc.skip=true" -}