From 8833cc7617c5d64929778df619acc5c411618222 Mon Sep 17 00:00:00 2001 From: Andreas Stefl Date: Mon, 17 Aug 2026 17:25:59 +0200 Subject: [PATCH 1/4] Photograph one simulator at a time, and give the test room to finish The screenshot run failed on whichever device was behind. Nothing was wrong with it: two simulators on a runner with three cores and seven gigabytes took ten minutes over the six screens, and XCTest allows a test ten. Both devices were killed at the same second. So one device per xcodebuild instead of both at once, and a ceiling high enough that only a real hang reaches it. The job gets a bound as well, and a failed run now keeps whatever it managed to photograph. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01E6tn8P25cBDmrA67zrhfwN --- .github/workflows/release.yml | 7 ++++++- OpenDocumentReaderUITests/ScreenshotTests.swift | 4 ++++ fastlane/Fastfile | 6 +++++- 3 files changed, 15 insertions(+), 2 deletions(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 407ad68..6b26b2a 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -186,6 +186,8 @@ jobs: # the artifact is the only way to look at the pictures before the store does. screenshots: runs-on: macos-26 + # long by nature, but the default is six hours for a wedged simulator to sit in + timeout-minutes: 180 steps: - name: checkout uses: actions/checkout@v7 @@ -223,11 +225,14 @@ jobs: # and what they were framed from, which is where to look when a picture # comes out wrong - name: archive the raw captures + # also when the lane failed: a half finished set is what says which + # language it got to + if: ${{ !cancelled() }} uses: actions/upload-artifact@v7 with: name: screenshots path: fastlane/screenshots - if-no-files-found: error + if-no-files-found: warn compression-level: 0 - uses: actions/upload-artifact@v7 diff --git a/OpenDocumentReaderUITests/ScreenshotTests.swift b/OpenDocumentReaderUITests/ScreenshotTests.swift index 969be0b..5cdb500 100644 --- a/OpenDocumentReaderUITests/ScreenshotTests.swift +++ b/OpenDocumentReaderUITests/ScreenshotTests.swift @@ -19,6 +19,10 @@ final class ScreenshotTests: XCTestCase { override func setUpWithError() throws { continueAfterFailure = false + + // A ceiling, not a target: six launches on a shared runner outrun + // XCTest's ten minute default and get killed mid-test. + executionTimeAllowance = 1800 } @MainActor diff --git a/fastlane/Fastfile b/fastlane/Fastfile index 3728204..b8ac4c3 100644 --- a/fastlane/Fastfile +++ b/fastlane/Fastfile @@ -178,7 +178,11 @@ platform :ios do skip_helper_version_check: true, # the runner has no browser to open the summary in, and no one watching it skip_open_summary: true, - stop_after_first_error: true + stop_after_first_error: true, + # one device per xcodebuild rather than both at once: two simulators + # rendering on a three core runner took ten minutes over one, and ran the + # test out of the time XCTest allows it + concurrent_simulators: false ) # A raw capture is not what the store shows. The framing is separate from the From 6a4cf0a478f345ac5eb6782322b3e36bcbff5c9f Mon Sep 17 00:00:00 2001 From: Andreas Stefl Date: Mon, 17 Aug 2026 18:57:53 +0200 Subject: [PATCH 2/4] Build the app once, and let the simulator speak the language too snapshot builds and tests on every run, and there are eighteen of them: two devices in nine languages. The build is the same every time and checking it costs about two minutes each on a runner, so it is built once up front and the runs only photograph. CI keeps the build between runs as well, which is why it goes somewhere with a name rather than the temp directory snapshot picks. The simulator is also told which language it is in. It was only ever the app that knew, so an iPad - which shows the date up there - said Mon Aug 17 above a Russian document. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01E6tn8P25cBDmrA67zrhfwN --- .github/workflows/release.yml | 10 ++++++++++ fastlane/Fastfile | 31 ++++++++++++++++++++++++++++++- 2 files changed, 40 insertions(+), 1 deletion(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 6b26b2a..e2798a5 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -207,6 +207,16 @@ jobs: python-version: "3.13" - run: python3 -m pip install --quiet Pillow + # The lane builds once into build/screenshots and photographs eighteen + # times without building again. Keeping it means a rerun starts at the + # first simulator instead of at the compiler. Keyed on everything the + # build reads, so a stale one is never used - only ever missed. + - name: reuse the last build + uses: actions/cache@v4 + with: + path: build/screenshots + key: screenshots-${{ runner.os }}-${{ env.xcode_version }}-${{ hashFiles('OpenDocumentReader.xcodeproj/project.pbxproj', 'OpenDocumentReader.xcodeproj/project.xcworkspace/xcshareddata/swiftpm/Package.resolved', 'OpenDocumentReader/**', 'OpenDocumentReaderUITests/**', 'Ads/**', 'NoAds/**', 'configs/**', 'scripts/make-screenshot-documents.py') }} + # the lane checks the set it produced, so a language that came out short # fails here rather than half way up to App Store Connect - name: photograph both devices in every locale diff --git a/fastlane/Fastfile b/fastlane/Fastfile index b8ac4c3..f870c88 100644 --- a/fastlane/Fastfile +++ b/fastlane/Fastfile @@ -30,6 +30,11 @@ MAKE_DOCUMENTS = File.expand_path("../scripts/make-screenshot-documents.py").fre SCREENSHOT_DIR = File.expand_path("screenshots").freeze FRAMED_DIR = File.expand_path("framed").freeze +# Named, where snapshot would pick a fresh temp directory each run. The app is +# built once into it and then photographed eighteen times without building +# again, and CI keeps it between runs. +SCREENSHOT_BUILD_DIR = File.expand_path("../build/screenshots").freeze + # The two sizes App Store Connect asks an app that runs on both for, named by # whatever the runner's Xcode calls them. Newest first, and the first one the # runner actually has wins - a simulator's name changes with every Xcode, the @@ -149,6 +154,23 @@ platform :ios do # it starts - a file written later lands in the next build, not this one. sh(MAKE_DOCUMENTS) + # Built once, here. snapshot asks xcodebuild to build and then test on every + # run, and there are eighteen of them - two devices in nine languages. The + # build is the same every time, and checking it is still up to date costs + # about two minutes on a runner. + scan( + project: "OpenDocumentReader.xcodeproj", + scheme: "ODR Screenshots", + # not a device: this is the one build both simulators are photographed from + destination: "generic/platform=iOS Simulator", + derived_data_path: SCREENSHOT_BUILD_DIR, + build_for_testing: true, + skip_detect_devices: true, + # what snapshot reads the build with. xcpretty is the default here and + # says itself that it can swallow a build error. + xcodebuild_formatter: "xcbeautify" + ) + # The Pro scheme, so no ad sdk is linked and no consent form can come up in # front of a picture. The two apps are the same app, and what differs - the # banner Lite carries - is not in a screenshot either way, so one set of @@ -182,7 +204,14 @@ platform :ios do # one device per xcodebuild rather than both at once: two simulators # rendering on a three core runner took ten minutes over one, and ran the # test out of the time XCTest allows it - concurrent_simulators: false + concurrent_simulators: false, + # the build above, photographed as it stands + derived_data_path: SCREENSHOT_BUILD_DIR, + test_without_building: true, + # the language the app is given is not the language the status bar is + # drawn in, and an iPad's status bar carries the date. Without this it + # reads Mon Aug 17 over a Russian document. + localize_simulator: true ) # A raw capture is not what the store shows. The framing is separate from the From 350b5a81e7efa0371fbb33547d0ab17bd546e027 Mon Sep 17 00:00:00 2001 From: Andreas Stefl Date: Mon, 17 Aug 2026 19:16:05 +0200 Subject: [PATCH 3/4] Give each device its own runner Two simulators never sat on one machine again by accident: they are on two now, a device to a runner, and the two halves photograph at the same time. What used to be an hour and a quarter is about half that. Neither runner can see the other's pictures, so the set is checked once they are back together rather than by the lane. A run started by hand is unchanged - both devices, every language, checked at the end. The edit screen also stopped waiting two seconds for a keyboard tutorial that the launch argument had already turned off. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01E6tn8P25cBDmrA67zrhfwN --- .github/workflows/release.yml | 74 +++++++++++++++++-- .../ScreenshotTests.swift | 5 +- fastlane/Fastfile | 43 +++++++---- 3 files changed, 100 insertions(+), 22 deletions(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index e2798a5..17ac1f6 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -184,10 +184,19 @@ jobs: # Beside the build rather than behind it: it signs nothing and uploads nothing, # it just drives simulators, and it takes about as long. On a dry run too - # the artifact is the only way to look at the pictures before the store does. + # A runner per device rather than one doing both. Two simulators on the seven + # gigabytes a runner has spent their time swapping, and the runs took five + # times as long as they do alone; a device to a machine is that, structurally. + # It halves the wall clock as well, since the two halves photograph at once. screenshots: runs-on: macos-26 # long by nature, but the default is six hours for a wedged simulator to sit in timeout-minutes: 180 + strategy: + # one device failing should not throw away the other's hour of work + fail-fast: false + matrix: + device: [iphone, ipad] steps: - name: checkout uses: actions/checkout@v7 @@ -217,16 +226,18 @@ jobs: path: build/screenshots key: screenshots-${{ runner.os }}-${{ env.xcode_version }}-${{ hashFiles('OpenDocumentReader.xcodeproj/project.pbxproj', 'OpenDocumentReader.xcodeproj/project.xcworkspace/xcshareddata/swiftpm/Package.resolved', 'OpenDocumentReader/**', 'OpenDocumentReaderUITests/**', 'Ads/**', 'NoAds/**', 'configs/**', 'scripts/make-screenshot-documents.py') }} - # the lane checks the set it produced, so a language that came out short - # fails here rather than half way up to App Store Connect - - name: photograph both devices in every locale + # half a set, so the lane does not check it - the job below does, once + # both halves are in + - name: photograph the ${{ matrix.device }} in every locale + env: + ODR_SCREENSHOT_DEVICE: ${{ matrix.device }} run: bundle exec fastlane ios screenshots # what the store is given - name: archive the framed screenshots uses: actions/upload-artifact@v7 with: - name: framed + name: framed-${{ matrix.device }} path: fastlane/framed if-no-files-found: error # png, so there is nothing left to squeeze out of them @@ -240,7 +251,7 @@ jobs: if: ${{ !cancelled() }} uses: actions/upload-artifact@v7 with: - name: screenshots + name: screenshots-${{ matrix.device }} path: fastlane/screenshots if-no-files-found: warn compression-level: 0 @@ -248,10 +259,59 @@ jobs: - uses: actions/upload-artifact@v7 if: failure() with: - name: screenshot-log + name: screenshot-log-${{ matrix.device }} path: ~/Library/Logs/snapshot if-no-files-found: warn + # The two halves put back together, and only now checked: a set is both + # devices in every locale, and neither runner above can see the other's. + # Republished under the names the release reads. + screenshot-set: + needs: screenshots + runs-on: ubuntu-24.04 + steps: + - name: checkout + uses: actions/checkout@v7 + + - uses: actions/setup-python@v6 + with: + python-version: "3.13" + + - name: fetch both halves + uses: actions/download-artifact@v8 + with: + pattern: framed-* + merge-multiple: true + path: fastlane/framed + + - name: fetch what they were framed from + uses: actions/download-artifact@v8 + with: + pattern: screenshots-* + merge-multiple: true + path: fastlane/screenshots + + # the check the lane cannot do on half a set: every locale, both devices, + # at a size App Store Connect takes + - name: check the set + run: scripts/store_screenshots.py --screenshots fastlane/framed + + - name: archive the framed screenshots + uses: actions/upload-artifact@v7 + with: + name: framed + path: fastlane/framed + if-no-files-found: error + compression-level: 0 + + - name: archive the raw captures + uses: actions/upload-artifact@v7 + with: + name: screenshots + path: fastlane/screenshots + if-no-files-found: warn + compression-level: 0 + # a job per app, fail-fast off, so "Re-run failed jobs" can retry one half upload: needs: build @@ -290,7 +350,7 @@ jobs: # while a build cannot be uploaded twice. No macOS runner: this touches the # listing, not the app listing: - needs: [upload, screenshots] + needs: [upload, screenshot-set] if: ${{ !inputs.dry_run }} runs-on: ubuntu-24.04 strategy: diff --git a/OpenDocumentReaderUITests/ScreenshotTests.swift b/OpenDocumentReaderUITests/ScreenshotTests.swift index 5cdb500..c464d63 100644 --- a/OpenDocumentReaderUITests/ScreenshotTests.swift +++ b/OpenDocumentReaderUITests/ScreenshotTests.swift @@ -129,7 +129,10 @@ final class ScreenshotTests: XCTestCase { private func dismissTheKeyboardTutorial(in app: XCUIApplication) { let continueButton = app.buttons["Continue"] - if continueButton.waitForExistence(timeout: 2) { + // Short, because this waits its whole timeout on every run that does not + // need it - which is all of them while the launch argument holds. The + // panel comes up with the keyboard, and the keyboard is already up here. + if continueButton.waitForExistence(timeout: 0.5) { continueButton.tap() } } diff --git a/fastlane/Fastfile b/fastlane/Fastfile index f870c88..6b2d763 100644 --- a/fastlane/Fastfile +++ b/fastlane/Fastfile @@ -39,10 +39,10 @@ SCREENSHOT_BUILD_DIR = File.expand_path("../build/screenshots").freeze # whatever the runner's Xcode calls them. Newest first, and the first one the # runner actually has wins - a simulator's name changes with every Xcode, the # number of pixels it has does not, and store_screenshots.py checks that. -SCREENSHOT_DEVICES = [ - ["iPhone 17 Pro Max", "iPhone 16 Pro Max", "iPhone 15 Pro Max"], - ["iPad Pro 13-inch (M5)", "iPad Pro 13-inch (M4)", "iPad Pro (12.9-inch) (6th generation)"], -].freeze +SCREENSHOT_DEVICES = { + "iphone" => ["iPhone 17 Pro Max", "iPhone 16 Pro Max", "iPhone 15 Pro Max"], + "ipad" => ["iPad Pro 13-inch (M5)", "iPad Pro 13-inch (M4)", "iPad Pro (12.9-inch) (6th generation)"], +}.freeze def dry_run? ENV["ODR_DRY_RUN"].to_s.strip == "true" @@ -99,7 +99,8 @@ def screenshot_devices(ios) available = simulators_by_ios.fetch(ios, []) - SCREENSHOT_DEVICES.map do |candidates| + screenshot_kinds.map do |kind| + candidates = SCREENSHOT_DEVICES.fetch(kind) candidates.find { |name| available.include?(name) } || UI.user_error!( "none of #{candidates.join(', ')} runs iOS #{ios}. That runtime has: " \ @@ -109,6 +110,19 @@ def screenshot_devices(ios) end end +# Which of the two the run is for. Both unless one is named, so a run started by +# hand is the whole set - only the release splits them, a device to a runner. +def screenshot_kinds + given = ENV["ODR_SCREENSHOT_DEVICE"].to_s.strip.downcase + return SCREENSHOT_DEVICES.keys if given.empty? + + unless SCREENSHOT_DEVICES.key?(given) + UI.user_error!("no such device: #{given}. One of #{SCREENSHOT_DEVICES.keys.join(', ')}.") + end + + [given] +end + platform :ios do desc "Build a signed .ipa of the paid app" lane :buildPro do @@ -165,10 +179,7 @@ platform :ios do destination: "generic/platform=iOS Simulator", derived_data_path: SCREENSHOT_BUILD_DIR, build_for_testing: true, - skip_detect_devices: true, - # what snapshot reads the build with. xcpretty is the default here and - # says itself that it can swallow a build error. - xcodebuild_formatter: "xcbeautify" + skip_detect_devices: true ) # The Pro scheme, so no ad sdk is linked and no consent form can come up in @@ -222,12 +233,16 @@ platform :ios do # what came out is what the store would be given, so it is checked here # rather than at upload time on the other side of the run. A run narrowed to - # a few languages by hand has nothing to check against: half a set is what it - # was asked for. - if ENV["ODR_SCREENSHOT_LANGUAGES"].to_s.strip.empty? - sh(STORE_SCREENSHOTS, "--screenshots", FRAMED_DIR) + # some of the languages or to one of the devices has nothing to check + # against: half a set is what it was asked for, and the release checks the + # two halves together once both runners have handed theirs in. + narrowed = %w[ODR_SCREENSHOT_LANGUAGES ODR_SCREENSHOT_DEVICE ODR_SCREENSHOT_DEVICES] + .any? { |name| !ENV[name].to_s.strip.empty? } + + if narrowed + UI.important("#{devices.join(', ')} in #{languages.join(', ')} is not a full set, so it is not checked") else - UI.important("only #{languages.join(', ')} were captured, so the set is not checked") + sh(STORE_SCREENSHOTS, "--screenshots", FRAMED_DIR) end end From 844325ad505b4238218994580ee2e77d8f265339 Mon Sep 17 00:00:00 2001 From: Andreas Stefl Date: Mon, 17 Aug 2026 21:23:07 +0200 Subject: [PATCH 4/4] Keep a build per device rather than one they overwrite Both runners saved under the same key, so the second one found it taken and the two halves shared whichever build got there first. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01E6tn8P25cBDmrA67zrhfwN --- .github/workflows/release.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 17ac1f6..295df10 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -224,7 +224,7 @@ jobs: uses: actions/cache@v4 with: path: build/screenshots - key: screenshots-${{ runner.os }}-${{ env.xcode_version }}-${{ hashFiles('OpenDocumentReader.xcodeproj/project.pbxproj', 'OpenDocumentReader.xcodeproj/project.xcworkspace/xcshareddata/swiftpm/Package.resolved', 'OpenDocumentReader/**', 'OpenDocumentReaderUITests/**', 'Ads/**', 'NoAds/**', 'configs/**', 'scripts/make-screenshot-documents.py') }} + key: screenshots-${{ matrix.device }}-${{ runner.os }}-${{ env.xcode_version }}-${{ hashFiles('OpenDocumentReader.xcodeproj/project.pbxproj', 'OpenDocumentReader.xcodeproj/project.xcworkspace/xcshareddata/swiftpm/Package.resolved', 'OpenDocumentReader/**', 'OpenDocumentReaderUITests/**', 'Ads/**', 'NoAds/**', 'configs/**', 'scripts/make-screenshot-documents.py') }} # half a set, so the lane does not check it - the job below does, once # both halves are in