Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
76 changes: 76 additions & 0 deletions src/commands/cloud.ts
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,7 @@ import {
import { MoropoService } from '../services/moropo.service.js';
import { ReportDownloadService } from '../services/report-download.service.js';
import {
deviceFromResultRow,
ResultsPollingService,
RunFailedError,
} from '../services/results-polling.service.js';
Expand All @@ -31,8 +32,10 @@ import {
EAndroidDevices,
EiOSDevices,
EiOSVersions,
isIosMatrixConfig,
} from '../types/domain/device.types.js';
import { resolveAuth } from '../utils/auth.js';
import { matrixIsIos, parseDeviceMatrix } from '../utils/device-matrix.js';
import { detectCiContext, isCI } from '../utils/ci.js';
import {
CliError,
Expand Down Expand Up @@ -211,6 +214,10 @@ export const cloudCommand = defineCommand({
'android-device',
);
const androidNoSnapshot = Boolean(args['android-no-snapshot']);
// Repeatable device-matrix flags: one validated cell each, no cross-product.
const iosConfigFlags = collectRepeatedFlag(rawArgs, ['--ios-config']);
const androidConfigFlags = collectRepeatedFlag(rawArgs, ['--android-config']);
const deviceMatrix = parseDeviceMatrix(iosConfigFlags, androidConfigFlags);
const json = Boolean(args.json);
const jsonFileFlag = Boolean(args['json-file']);
const jsonFileName = args['json-file-name'] as string | undefined;
Expand Down Expand Up @@ -502,6 +509,28 @@ export const cloudCommand = defineCommand({
{ debug, logger: (m: string) => out(m) },
);

// Validate every device-matrix cell up front — each on its own, against
// the same compatibility matrix — so an unsupported cell fails fast,
// naming it, before anything is uploaded.
for (const cfg of deviceMatrix) {
if (isIosMatrixConfig(cfg)) {
deviceValidationService.validateiOSDevice(
cfg.iOSVersion,
cfg.iOSDevice,
compatibilityData,
{ debug, logger: (m: string) => out(m) },
);
} else {
deviceValidationService.validateAndroidDevice(
cfg.androidApiLevel,
cfg.androidDevice,
cfg.googlePlay ?? googlePlay,
compatibilityData,
{ debug, logger: (m: string) => out(m) },
);
}
}

if (maestroChromeOnboarding && !androidApiLevel && !androidDevice) {
warnOut(
'The --maestro-chrome-onboarding flag only applies to Android tests and will be ignored for iOS tests.',
Expand Down Expand Up @@ -639,6 +668,8 @@ export const cloudCommand = defineCommand({
'include-tags': includeTags,
'exclude-tags': excludeTags,
'exclude-flows': excludeFlows,
'ios-config': iosConfigFlags,
'android-config': androidConfigFlags,
};
for (const [k, v] of Object.entries(args)) {
if (!canonicalFlagKeys.has(k)) continue;
Expand Down Expand Up @@ -762,6 +793,7 @@ export const cloudCommand = defineCommand({
continueOnFailure,
debug,
deviceLocale,
deviceMatrix,
env,
executionPlan,
flowFile,
Expand All @@ -784,6 +816,49 @@ export const cloudCommand = defineCommand({
disableAnimations,
});

// Device-matrix cost preview: the server prices the exact fan-out (quote
// == charge) so the user sees the cell count and estimated cost before the
// flow zip is uploaded. An unsupported cell fails fast here. Skipped when
// there is no matrix, and tolerant of older APIs that lack the endpoint.
if (deviceMatrix.length > 0) {
const estimate = await ApiGateway.estimateMatrix(apiUrl, auth, fields);
if (estimate) {
const osPrefix = matrixIsIos(deviceMatrix) ? 'iOS' : 'API';
const rows = ui.fields([
['cells', colors.highlight(String(estimate.cellCount))],
['est. cost', colors.highlight(`$${estimate.totalCost.toFixed(2)}`)],
]);
for (const col of estimate.columns) {
const label = [
col.deviceName,
col.osVersion && `${osPrefix} ${col.osVersion}`,
col.googlePlay && 'Play',
]
.filter(Boolean)
.join(' · ');
rows.push(
...ui.fields([
[
label,
colors.dim(
`${col.flowCount} flow${col.flowCount === 1 ? '' : 's'} · $${col.cost.toFixed(2)}`,
),
],
]),
);
}
if (estimate.excludedFlows.length > 0) {
rows.push(
colors.dim(
`${estimate.excludedFlows.length} flow${estimate.excludedFlows.length === 1 ? '' : 's'} target their own device (excluded from the matrix)`,
),
);
}
out(ui.section('Device matrix'));
out(ui.branch(rows));
}
}

// New path: upload the zip directly to storage, then submit a JSON test
// referencing it. Older API deployments lack these endpoints — a real API
// 404s (route undefined), some proxies 405 (path/method not allowed); in
Expand Down Expand Up @@ -865,6 +940,7 @@ export const cloudCommand = defineCommand({
consoleUrl: url,
status: 'PENDING',
tests: results.map((r) => ({
device: deviceFromResultRow(r),
fileName: r.test_file_name,
flowName:
testMetadataMap[r.test_file_name]?.flowName ||
Expand Down
8 changes: 8 additions & 0 deletions src/config/flags/device.flags.ts
Original file line number Diff line number Diff line change
Expand Up @@ -42,6 +42,14 @@ export const deviceFlags = {
type: 'string',
description: `[iOS only] iOS version to run your flow against (options: ${iosVersions})`,
},
'ios-config': {
type: 'string',
description: `[iOS only] Device-matrix cell as <device>:<version>, e.g. iphone-16:18. Repeatable — every flow runs once per cell (no cross-product). Cannot be combined with --android-config.`,
},
'android-config': {
type: 'string',
description: `[Android only] Device-matrix cell as <device>:<apiLevel> (append :play for Google Play), e.g. pixel-7:34 or pixel-7:34:play. Repeatable — every flow runs once per cell (no cross-product). Cannot be combined with --ios-config.`,
},
orientation: {
type: 'string',
description:
Expand Down
46 changes: 46 additions & 0 deletions src/gateways/api-gateway.ts
Original file line number Diff line number Diff line change
Expand Up @@ -588,6 +588,52 @@ export const ApiGateway = {
}
},

/**
* Dry-run cost + cell-count estimate for a (possibly device-matrix)
* submission. Runs the same resolve → validate → fan-out → price core as the
* submit path server-side, without persisting. The preview is best-effort:
* an API that predates this endpoint 404s (route undefined) or 405s (route
* matched a sibling path, no POST) — in either case return null so the caller
* proceeds without a preview. A 400 (an invalid config) still surfaces as a
* normal API error so the CLI can fail fast before uploading the flow zip.
*/
async estimateMatrix(baseUrl: string, auth: AuthContext, body: Record<string, unknown>) {
try {
const res = await fetch(`${baseUrl}/uploads/estimateMatrix`, {
body: JSON.stringify(body),
headers: {
'content-type': 'application/json',
...auth.headers,
},
method: 'POST',
});
if (res.status === 404 || res.status === 405) {
return null;
}
if (!res.ok) {
await this.handleApiError(res, 'Failed to estimate device matrix');
}
return await parseJsonResponse<{
cellCount: number;
totalCost: number;
excludedFlows: string[];
columns: Array<{
deviceName: string;
osVersion: string;
googlePlay: boolean;
flowCount: number;
cost: number;
}>;
}>(res, 'Failed to estimate device matrix');
} catch (error) {
if (error instanceof TypeError && error.message === 'fetch failed') {
throw this.enhanceFetchError(error, `${baseUrl}/uploads/estimateMatrix`);
}

throw error;
}
},


/**
* Generic report download method that handles both junit and allure reports
Expand Down
42 changes: 42 additions & 0 deletions src/services/results-polling.service.ts
Original file line number Diff line number Diff line change
Expand Up @@ -47,10 +47,46 @@ export interface TestMetadata {
tags: string[];
}

/**
* The device a result ran on. Additive: single-device runs are unchanged, and
* a device matrix disambiguates two `tests[]` entries that share a `name` (the
* same flow on two devices) by their device.
*/
export interface TestDevice {
googlePlay?: boolean;
name?: string;
osVersion?: string;
}

/**
* Structured device for a result row. Prefers the friendly deviceName/osVersion
* the per-flow targeting / matrix fan-out stamps onto each result's config
* (#1097), falling back to the raw simulator_name for older rows. Returns
* undefined when neither is present, so single-device runs that predate the
* field simply omit `device`. Shared by the sync polling path and the async
* (--async --json) path so both emit an identical device shape.
*/
export function deviceFromResultRow(r: {
config?: unknown;
simulator_name?: string | null;
}): TestDevice | undefined {
const config = (r.config ?? {}) as { deviceName?: string; osVersion?: string };
const sim = r.simulator_name ?? undefined;
const name = config.deviceName ?? sim;
if (!name && !config.osVersion) return undefined;
return {
name,
osVersion: config.osVersion,
googlePlay: sim ? /(_PLAY|-play)$/.test(sim) : undefined,
};
}

export interface PollingResult {
consoleUrl: string;
status: 'FAILED' | 'PASSED';
tests: Array<{
/** Device this result ran on (present when the API reports it). */
device?: TestDevice;
durationSeconds: null | number;
failReason?: string;
/** File path of the test (same as name, for clarity) */
Expand Down Expand Up @@ -311,6 +347,12 @@ export class ResultsPollingService {
? 'PASSED'
: 'FAILED',
tests: resultsWithoutEarlierTries.map((r) => ({
// r carries config/simulator_name at runtime; the committed generated
// types lag the API (regenerated wholesale from dev's swagger), so read
// them through the helper's structural type.
device: deviceFromResultRow(
r as { config?: unknown; simulator_name?: string | null },
),
durationSeconds: r.duration_seconds ?? null,
failReason:
r.status === 'FAILED' ? r.fail_reason || 'No reason provided' : undefined,
Expand Down
21 changes: 20 additions & 1 deletion src/services/test-submission.service.ts
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@ import { createHash } from 'node:crypto';
import * as path from 'node:path';

import { compressFilesFromRelativePath } from '../methods.js';
import { DeviceMatrixConfig } from '../types/domain/device.types.js';
import { toPortableRelativePath } from '../utils/paths.js';
import { IExecutionPlan } from './execution-plan.service.js';

Expand All @@ -15,6 +16,7 @@ export interface TestSubmissionConfig {
continueOnFailure?: boolean;
debug?: boolean;
deviceLocale?: string;
deviceMatrix?: DeviceMatrixConfig[];
disableAnimations?: boolean;
env?: string[];
executionPlan: IExecutionPlan;
Expand Down Expand Up @@ -85,6 +87,7 @@ export class TestSubmissionService {
maestroChromeOnboarding,
raw,
disableAnimations,
deviceMatrix,
debug = false,
logger,
} = config;
Expand Down Expand Up @@ -183,7 +186,23 @@ export class TestSubmissionService {
// Note: googlePlay is now included in configPayload below instead of as a separate field
// to work around a FormData parsing issue in the API

const targetPlatform = iOSDevice || iOSVersion ? 'ios' : 'android';
// Explicit device matrix (one upload, N cells). Only sent when present, so
// single-device submissions stay byte-identical.
if (deviceMatrix && deviceMatrix.length > 0) {
fields.deviceMatrix = JSON.stringify(deviceMatrix);
}

// Platform used only to pick which workspace-config disableAnimations flag
// applies. A device matrix is single-platform; its first cell decides. Fall
// back to the scalar iOS flags for single-device submissions.
const matrixPlatform =
deviceMatrix && deviceMatrix.length > 0
? 'iOSDevice' in deviceMatrix[0]
? 'ios'
: 'android'
: undefined;
const targetPlatform =
matrixPlatform ?? (iOSDevice || iOSVersion ? 'ios' : 'android');
const configYamlDisableAnimations =
targetPlatform === 'ios'
? Boolean(workspaceConfig?.platform?.ios?.disableAnimations)
Expand Down
14 changes: 14 additions & 0 deletions src/types/domain/device.types.ts
Original file line number Diff line number Diff line change
Expand Up @@ -40,3 +40,17 @@ export enum EAndroidApiLevels {
'thirtyTwo' = '32',
'twentyNine' = '29',
}

/**
* One explicit device-matrix cell. iOS entries carry {iOSDevice, iOSVersion};
* Android entries carry {androidDevice, androidApiLevel} plus an optional Play
* channel. Sent to the API as the `deviceMatrix` array; every non-targeted flow
* runs once per cell. There is no cross-product — each entry is one cell.
*/
export type DeviceMatrixConfig =
| { iOSDevice: string; iOSVersion: string }
| { androidApiLevel: string; androidDevice: string; googlePlay?: boolean };

export const isIosMatrixConfig = (
c: DeviceMatrixConfig,
): c is { iOSDevice: string; iOSVersion: string } => 'iOSDevice' in c;
68 changes: 68 additions & 0 deletions src/utils/device-matrix.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,68 @@
import {
DeviceMatrixConfig,
isIosMatrixConfig,
} from '../types/domain/device.types.js';
import { CliError } from './cli.js';

/**
* Parse repeated `--ios-config <device>:<version>` and
* `--android-config <device>:<apiLevel>[:play]` flags into an explicit device
* matrix. Each entry is one validated cell — there is NO cross-product, because
* the compatibility matrix is ragged and a cross-product would invent cells the
* user never asked for.
*
* A device matrix is single-platform (one upload, one binary), so mixing iOS
* and Android configs is rejected here before anything is uploaded.
*
* @throws CliError on malformed syntax or a mixed-platform matrix.
*/
export function parseDeviceMatrix(
iosConfigs: string[],
androidConfigs: string[],
): DeviceMatrixConfig[] {
if (iosConfigs.length > 0 && androidConfigs.length > 0) {
throw new CliError(
'A device matrix cannot mix platforms: use either --ios-config or --android-config, not both. One upload runs one binary.',
);
}

const configs: DeviceMatrixConfig[] = [];

for (const raw of iosConfigs) {
const parts = raw.split(':');
if (parts.length !== 2 || !parts[0] || !parts[1]) {
throw new CliError(
`Invalid --ios-config "${raw}". Expected <device>:<version>, e.g. iphone-16:18.`,
);
}
configs.push({ iOSDevice: parts[0], iOSVersion: parts[1] });
}

for (const raw of androidConfigs) {
const parts = raw.split(':');
// <device>:<apiLevel> with an optional trailing :play for a Play cell.
if (
parts.length < 2 ||
parts.length > 3 ||
!parts[0] ||
!parts[1] ||
(parts.length === 3 && parts[2] !== 'play')
) {
throw new CliError(
`Invalid --android-config "${raw}". Expected <device>:<apiLevel> or <device>:<apiLevel>:play, e.g. pixel-7:34 or pixel-7:34:play.`,
);
}
configs.push({
androidDevice: parts[0],
androidApiLevel: parts[1],
...(parts.length === 3 ? { googlePlay: true } : {}),
});
}

return configs;
}

/** True when the matrix targets iOS (used to pick the validation lookup). */
export function matrixIsIos(configs: DeviceMatrixConfig[]): boolean {
return configs.length > 0 && isIosMatrixConfig(configs[0]);
}
Loading
Loading