diff --git a/tools/ensemble_test_runner/README.md b/tools/ensemble_test_runner/README.md index 540e40e39..9e3246648 100644 --- a/tools/ensemble_test_runner/README.md +++ b/tools/ensemble_test_runner/README.md @@ -18,6 +18,13 @@ initialState: name: first: John last: Doe + # Ensemble encrypted public storage. Keys are logical keys; values are + # encrypted at bootstrap and persisted internally as enc_. + secureStorage: + onboardingComplete: true + # Platform FlutterSecureStorage, used by keychain/readSecurely actions. + keychain: + authToken: test-token steps: - expectVisible: id: greeting_text @@ -123,7 +130,9 @@ logStorage: `mocks` and `initialState` in `config.yaml` apply to every test. Test-file `mocks` / `initialState` values override suite values for the same API name or -storage/keychain/env key. +storage/secureStorage/keychain/env key. `storage`, `secureStorage`, and +`keychain` are separate backends: `secureStorage` is Ensemble's encrypted +public-storage namespace, while `keychain` is platform secure storage. When `devices` is set, each test expands to one run per device (ids look like `home[android_nl]` when there is more than one device). Device `locale` sets @@ -337,6 +346,10 @@ override suite keys: initialState: storage: apiUrl: http://ensemble.test/ws/NeMo/Intf/lan:getMIBs + secureStorage: + onboardingComplete: true + keychain: + authToken: test-token env: APP_LOCALE: nl @@ -355,7 +368,7 @@ steps: ### Reusable authenticated session The session producer runs once. After it passes, the runner captures public -storage, keychain values, and locale in memory. Each consumer restores that +storage, encrypted secure-storage values, keychain values, and locale in memory. Each consumer restores that snapshot, runs its `setup`, and mounts a fresh requested screen. ```yaml diff --git a/tools/ensemble_test_runner/STEP_VOCABULARY.md b/tools/ensemble_test_runner/STEP_VOCABULARY.md index bb4af6d75..a574bed35 100644 --- a/tools/ensemble_test_runner/STEP_VOCABULARY.md +++ b/tools/ensemble_test_runner/STEP_VOCABULARY.md @@ -148,7 +148,7 @@ with `--report=json` or `--report-file=build/ensemble_test_results.json`. Each `*.test.yaml` file is a single test case and must provide: - `startScreen` — cold-starts the app on the given screen and runs steps -- `session` — optional; runs the referenced test once, restores its captured storage/keychain/locale for this test, runs `setup`, and mounts the requested `startScreen` +- `session` — optional; runs the referenced test once, restores its captured public storage/secure storage/keychain/locale for this test, runs `setup`, and mounts the requested `startScreen` - `retry` — number of additional attempts after a failed run, e.g. `retry: 3` Root-level `setup` supports `httpRequest`, `group`, and `optional`. It runs diff --git a/tools/ensemble_test_runner/assets/schema/ensemble_test_config_schema.json b/tools/ensemble_test_runner/assets/schema/ensemble_test_config_schema.json index 24e4f6dad..221b76d8e 100644 --- a/tools/ensemble_test_runner/assets/schema/ensemble_test_config_schema.json +++ b/tools/ensemble_test_runner/assets/schema/ensemble_test_config_schema.json @@ -75,7 +75,7 @@ }, "initialState": { "$ref": "#/$defs/initialState", - "description": "Suite-wide storage/keychain/env applied before each test initialState. Test values override suite values per key." + "description": "Suite-wide storage/secureStorage/keychain/env applied before each test initialState. Test values override suite values per key." }, "profiles": { "type": "object", @@ -213,6 +213,18 @@ } ] }, + "secureStorage": { + "oneOf": [ + { + "type": "object", + "additionalProperties": true + }, + { + "type": "string", + "pattern": "^\\$\\{(inputs|scenario)\\.[A-Za-z0-9_.-]+\\}$" + } + ] + }, "keychain": { "oneOf": [ { diff --git a/tools/ensemble_test_runner/assets/schema/ensemble_tests_schema.json b/tools/ensemble_test_runner/assets/schema/ensemble_tests_schema.json index 024a415d5..883cf54c6 100644 --- a/tools/ensemble_test_runner/assets/schema/ensemble_tests_schema.json +++ b/tools/ensemble_test_runner/assets/schema/ensemble_tests_schema.json @@ -155,6 +155,18 @@ } ] }, + "secureStorage": { + "oneOf": [ + { + "type": "object", + "additionalProperties": true + }, + { + "type": "string", + "pattern": "^\\$\\{(inputs|scenario)\\.[A-Za-z0-9_.-]+\\}$" + } + ] + }, "keychain": { "oneOf": [ { diff --git a/tools/ensemble_test_runner/lib/assertions/assertion_engine.dart b/tools/ensemble_test_runner/lib/assertions/assertion_engine.dart index 8b37882f2..8b104419b 100644 --- a/tools/ensemble_test_runner/lib/assertions/assertion_engine.dart +++ b/tools/ensemble_test_runner/lib/assertions/assertion_engine.dart @@ -192,7 +192,11 @@ class AssertionEngine { ); } final isEnabled = _readSemantics( - () => tester.getSemantics(finder).hasFlag(SemanticsFlag.isEnabled), + () => tester + .getSemantics(finder) + .getSemanticsData() + .flagsCollection + .isEnabled, ); if (isEnabled != enabled) { throw EnsembleTestFailure( @@ -358,7 +362,11 @@ class AssertionEngine { throw EnsembleTestFailure('expectChecked: widget "$id" not found.'); } final isChecked = _readSemantics( - () => tester.getSemantics(finder).hasFlag(SemanticsFlag.isChecked), + () => tester + .getSemantics(finder) + .getSemanticsData() + .flagsCollection + .isChecked, ); if (isChecked != expected) { throw EnsembleTestFailure( diff --git a/tools/ensemble_test_runner/lib/discovery/ensemble_test_execution_planner.dart b/tools/ensemble_test_runner/lib/discovery/ensemble_test_execution_planner.dart index aa847e7a3..e9758daa8 100644 --- a/tools/ensemble_test_runner/lib/discovery/ensemble_test_execution_planner.dart +++ b/tools/ensemble_test_runner/lib/discovery/ensemble_test_execution_planner.dart @@ -635,7 +635,7 @@ class EnsembleTestExecutionPlanner { } /// Suite [initialState] is the base; test values override per key within - /// `storage`, `keychain`, and `env`. + /// `storage`, `secureStorage`, `keychain`, and `env`. @visibleForTesting static Map mergedInitialState( Map suite, @@ -654,7 +654,7 @@ class EnsembleTestExecutionPlanner { } final merged = {}; - for (final key in const ['storage', 'keychain', 'env']) { + for (final key in const ['storage', 'secureStorage', 'keychain', 'env']) { final value = section(key); if (value.isNotEmpty) { merged[key] = value; diff --git a/tools/ensemble_test_runner/lib/reporters/html_test_report_app_js.dart b/tools/ensemble_test_runner/lib/reporters/html_test_report_app_js.dart index e244c9c30..fcecfe1a4 100644 --- a/tools/ensemble_test_runner/lib/reporters/html_test_report_app_js.dart +++ b/tools/ensemble_test_runner/lib/reporters/html_test_report_app_js.dart @@ -11,6 +11,7 @@ const ensembleHtmlTestReportAppJs = r''' let activeModalTab = 'api'; let currentModalCardId = ''; let currentModalStepIndex = -1; + let activeStorageSubTab = 'public'; function escapeHtml(str) { return String(str).replace(/&/g, '&').replace(//g, '>').replace(/"/g, '"').replace(/'/g, '''); @@ -37,6 +38,25 @@ const ensembleHtmlTestReportAppJs = r''' return String(testId).replace(/[^A-Za-z0-9_-]+/g, '_'); } + /** Strip a shared screen-name prefix (e.g. ExtenderPositioning_) for readable journey chips. */ + function shortenScreenLabels(names) { + const list = (names || []).map((n) => String(n || '')); + if (list.length < 2) return { prefix: '', labels: list }; + let prefix = list[0]; + for (let i = 1; i < list.length; i++) { + while (prefix && !list[i].startsWith(prefix)) { + prefix = prefix.slice(0, -1); + } + if (!prefix) break; + } + const cut = Math.max(prefix.lastIndexOf('_'), prefix.lastIndexOf('/'), prefix.lastIndexOf('.')); + if (cut < 2) return { prefix: '', labels: list }; + const shared = prefix.slice(0, cut + 1); + const labels = list.map((n) => n.slice(shared.length) || n); + if (labels.some((l) => !l)) return { prefix: '', labels: list }; + return { prefix: shared.slice(0, -1), labels: labels }; + } + function resolveBlobValue(value, blobs) { if (value && typeof value === 'object' && !Array.isArray(value)) { if (Object.keys(value).length === 1 && typeof value['$b'] === 'string') { @@ -64,21 +84,29 @@ const ensembleHtmlTestReportAppJs = r''' apiCalls: step.apiCalls || [], appLogs: step.appLogs || [], storageChanges: step.storageChanges || [], + secureStorageChanges: step.secureStorageChanges || [], + keychainChanges: step.keychainChanges || [], screenshots: step.screenshots || [] }; step.apiCalls = parent.apiCalls; step.appLogs = parent.appLogs; step.storageChanges = parent.storageChanges; + step.secureStorageChanges = parent.secureStorageChanges; + step.keychainChanges = parent.keychainChanges; step.screenshots = parent.screenshots; } else if (parent) { step.apiCalls = parent.apiCalls; step.appLogs = parent.appLogs; step.storageChanges = parent.storageChanges; + step.secureStorageChanges = parent.secureStorageChanges; + step.keychainChanges = parent.keychainChanges; step.screenshots = parent.screenshots; } else { step.apiCalls = []; step.appLogs = []; step.storageChanges = []; + step.secureStorageChanges = []; + step.keychainChanges = []; step.screenshots = []; } steps[i] = step; @@ -179,7 +207,6 @@ const ensembleHtmlTestReportAppJs = r''' pr_number AS prNumber FROM runs ORDER BY id DESC - LIMIT 25 `)); return { runs }; } catch (e) { @@ -286,6 +313,7 @@ const ensembleHtmlTestReportAppJs = r''' } window.stepData = {}; + window.storageSnapshots = {}; const listPane = document.getElementById('test-list-pane'); const detailPane = document.getElementById('test-detail-pane'); listPane.innerHTML = ''; @@ -568,7 +596,7 @@ const ensembleHtmlTestReportAppJs = r''' html += ''; html += '
'; - html += '

Test Execution History

' + runs.length + ' recent runs
'; + html += '

Test Execution History

' + runs.length + ' total runs
'; html += '
'; html += ''; @@ -763,6 +791,7 @@ const ensembleHtmlTestReportAppJs = r''' runs.forEach((test, i) => { const stepKey = cardId + '-' + i; window.stepData[stepKey] = test.steps || []; + window.storageSnapshots[stepKey] = test.storage || {}; html += buildRunBlock(base, test, cardId, i, stepKey); }); wrap.innerHTML = html; @@ -858,17 +887,46 @@ const ensembleHtmlTestReportAppJs = r''' const visited = report.screensVisited || []; if (visited.length > 0) { - html += '
Flow Journey
'; + const screensMap = report.screens || {}; + const shortened = shortenScreenLabels(visited); + const uniqueCount = new Set(visited).size; + const seenCounts = {}; + html += '
'; + html += '
Flow Journey
'; + html += '' + visited.length + ' screen' + (visited.length === 1 ? '' : 's') + ''; + if (uniqueCount !== visited.length) { + html += '' + uniqueCount + ' unique'; + } + if (shortened.prefix) { + html += '' + escapeHtml(shortened.prefix) + ''; + } + html += '
'; visited.forEach((s, j) => { - const screensMap = report.screens || {}; + seenCounts[s] = (seenCounts[s] || 0) + 1; + const visitNum = seenCounts[s]; + const isRevisit = visitNum > 1; const hasData = screensMap[s] !== undefined; - if (hasData) { - const escapedScreen = s.replace(/'/g, "\\'"); - html += '' + escapeHtml(s) + ''; - } else { - html += '' + escapeHtml(s) + ''; - } - if (j < visited.length - 1) html += ''; + const label = shortened.labels[j] || s; + const classes = ['flow-node']; + if (j === 0) classes.push('start'); + if (j === visited.length - 1) classes.push('end'); + if (isRevisit) classes.push('revisit'); + if (hasData) classes.push('interactive'); + const titleParts = [s]; + if (isRevisit) titleParts.push('visit #' + visitNum); + if (hasData) titleParts.push('Click for widget tree & performance'); + const title = titleParts.join(' · '); + const tag = hasData ? 'button' : 'span'; + const typeAttr = hasData ? ' type="button"' : ''; + const clickAttr = hasData + ? ' onclick="openScreenDialog(\'' + anchorId(test.id) + '\', \'' + String(s).replace(/\\/g, '\\\\').replace(/'/g, "\\'") + '\')"' + : ''; + html += '<' + tag + typeAttr + ' class="' + classes.join(' ') + '" title="' + escapeHtml(title) + '"' + clickAttr + '>'; + html += '' + (j + 1) + ''; + html += '' + escapeHtml(label) + ''; + if (isRevisit) html += '×' + visitNum + ''; + if (hasData) html += ''; + html += ''; }); html += '
'; } @@ -976,6 +1034,8 @@ const ensembleHtmlTestReportAppJs = r''' const events = flattenStepField(test, 'apiCalls'); const storage = test.storage || {}; const keys = storage.keys || {}; + const secureKeys = storage.secureStorageKeys || {}; + const keychainKeys = storage.keychainKeys || {}; let storageContent = ''; try { storageContent = JSON.stringify(keys, null, 2); } catch (e) { storageContent = String(keys); } @@ -988,10 +1048,28 @@ const ensembleHtmlTestReportAppJs = r''' html += '
'; html += '
' + renderApiRows(events) + '
'; - if (Object.keys(keys).length > 0) { - html += '
💾 Local State Storage'; - html += '
'; - html += '
' + escapeHtml(storageContent) + '
'; + const hasKeys = Object.keys(keys).length > 0; + const hasSecure = Object.keys(secureKeys).length > 0; + const hasKeychain = Object.keys(keychainKeys).length > 0; + + if (hasKeys || hasSecure || hasKeychain) { + html += '
'; + if (hasKeys) { + html += '
💾 Local State Storage'; + html += '
'; + html += '
' + escapeHtml(storageContent) + '
'; + } + if (hasSecure) { + html += '
🔐 Secure Storage'; + html += '
'; + html += '
' + escapeHtml(JSON.stringify(secureKeys, null, 2)) + '
'; + } + if (hasKeychain) { + html += '
🔑 Keychain'; + html += '
'; + html += '
' + escapeHtml(JSON.stringify(keychainKeys, null, 2)) + '
'; + } + html += '
'; } return html; } @@ -1105,21 +1183,24 @@ const ensembleHtmlTestReportAppJs = r''' } // --- Step modal (retargeted to window.stepData) --- - function getStorageStateAtStep(cardId, targetStepIndex) { + function getStorageStateAtStep(cardId, targetStepIndex, field) { const deviceData = window.stepData && window.stepData[cardId]; if (!deviceData) return {}; const stepKeys = Object.keys(deviceData).map(k => parseInt(k, 10)).filter(n => !isNaN(n)).sort((a, b) => a - b); - const state = {}; - for (const stepKey of stepKeys) { - if (stepKey > targetStepIndex) break; - const stepObj = deviceData[stepKey]; - const changes = (stepObj && stepObj.storageChanges) || []; + const snapshot = window.storageSnapshots && window.storageSnapshots[cardId]; + const snapshotField = field === 'secureStorageChanges' ? 'secureStorageKeys' : field === 'keychainChanges' ? 'keychainKeys' : 'keys'; + const state = Object.assign({}, (snapshot && snapshot[snapshotField]) || {}); + // Rewind the final snapshot to the state immediately after the requested + // step so initial values remain visible even if they never changed. + for (let i = stepKeys.length - 1; i > targetStepIndex; i--) { + const stepObj = deviceData[stepKeys[i]]; + const changes = (stepObj && stepObj[field || 'storageChanges']) || []; for (const change of changes) { const key = change.key; if (!key) continue; const kind = (change.change || '').toLowerCase(); - if (kind === 'removed') delete state[key]; - else if (kind === 'added' || kind === 'modified') state[key] = change.after; + if (kind === 'added') delete state[key]; + else if (kind === 'removed' || kind === 'modified') state[key] = change.before; } } return state; @@ -1204,12 +1285,47 @@ const ensembleHtmlTestReportAppJs = r''' const storageList = document.getElementById('modal-storage-list'); storageList.innerHTML = ''; + storageList.className = 'modal-list storage-tab-content'; + const storageSubTabs = document.createElement('div'); + storageSubTabs.className = 'storage-sub-tabs'; + const storagePanels = document.createElement('div'); + storagePanels.className = 'storage-sub-panels'; + const publicStoragePanel = document.createElement('div'); + const secureStoragePanel = document.createElement('div'); + const keychainPanel = document.createElement('div'); + publicStoragePanel.className = 'storage-sub-panel logs-terminal'; + secureStoragePanel.className = 'storage-sub-panel'; + keychainPanel.className = 'storage-sub-panel'; + const publicStorageList = publicStoragePanel; + const storageSubTabDefinitions = [ + { id: 'public', label: 'Public Storage', panel: publicStoragePanel }, + { id: 'secure', label: 'Secure Storage', panel: secureStoragePanel }, + { id: 'keychain', label: 'Keychain', panel: keychainPanel } + ]; + storageSubTabDefinitions.forEach(definition => { + const button = document.createElement('button'); + button.className = 'storage-sub-tab-btn' + (definition.id === activeStorageSubTab ? ' active' : ''); + button.textContent = definition.label; + button.onclick = () => { + activeStorageSubTab = definition.id; + storageSubTabDefinitions.forEach(item => { + item.panel.style.display = item.id === activeStorageSubTab ? 'block' : 'none'; + }); + storageSubTabs.querySelectorAll('.storage-sub-tab-btn').forEach(item => item.classList.remove('active')); + button.classList.add('active'); + }; + storageSubTabs.appendChild(button); + definition.panel.style.display = definition.id === activeStorageSubTab ? 'block' : 'none'; + storagePanels.appendChild(definition.panel); + }); + storageList.appendChild(storageSubTabs); + storageList.appendChild(storagePanels); const storageChanges = data.storageChanges || []; document.getElementById('modal-storage-count').textContent = storageChanges.length; const currentState = getStorageStateAtStep(cardId, stepIndex); const changedKeys = new Set(storageChanges.map(c => c.key).filter(Boolean)); if (!storageChanges.length && Object.keys(currentState).length === 0) { - storageList.innerHTML = '
<no storage changes for this step>
'; + publicStorageList.innerHTML = '
<no storage changes for this step>
'; } else { storageChanges.forEach(change => { const key = change.key || '(unknown)'; @@ -1270,7 +1386,7 @@ const ensembleHtmlTestReportAppJs = r''' row.innerHTML = '' + badgeText + '' + escapeHtml(key) + '' + escapeHtml(formatStorageValue(change.before)) + '' + escapeHtml(formatStorageValue(change.after)) + ''; } } - storageList.appendChild(row); + publicStorageList.appendChild(row); }); Object.keys(currentState).filter(k => !changedKeys.has(k)).sort().forEach(key => { const val = currentState[key]; @@ -1285,10 +1401,78 @@ const ensembleHtmlTestReportAppJs = r''' } else { row.innerHTML = 'VAL' + escapeHtml(key) + '' + escapeHtml(formatStorageValue(val)) + ''; } - storageList.appendChild(row); + publicStorageList.appendChild(row); }); } + function appendSimpleStorageSection(title, changesField, stateField, target) { + const changes = data[changesField] || []; + const state = getStorageStateAtStep(cardId, stepIndex, changesField); + const section = document.createElement('div'); + section.style.marginTop = '18px'; + const body = document.createElement('div'); + body.className = 'logs-terminal'; + if (!changes.length && !Object.keys(state).length) { + body.innerHTML = '
<no changes>
'; + } else { + changes.forEach(change => { + const key = change.key || '(unknown)'; + const row = document.createElement('div'); + const kind = (change.change || '').toLowerCase(); + let badgeClass = 'info', badgeText = 'MOD'; + if (kind === 'added') { + badgeClass = 'passed'; + badgeText = 'ADD'; + } else if (kind === 'removed') { + badgeClass = 'failed'; + badgeText = 'DEL'; + } + const beforeFormatted = change.before === undefined ? '' : prettyFormatStorageValue(change.before); + const afterFormatted = change.after === undefined ? '' : prettyFormatStorageValue(change.after); + const isLong = beforeFormatted.length > 40 || afterFormatted.length > 40 || beforeFormatted.includes('\n') || afterFormatted.includes('\n'); + if (isLong) { + row.className = 'terminal-row storage-collapsible'; + let details = ''; + if (kind === 'modified') { + details = '
' + + '
Before
' + escapeHtml(beforeFormatted) + '
' + + '
After
' + escapeHtml(afterFormatted) + '
' + + '
'; + } else { + const value = kind === 'removed' ? beforeFormatted : afterFormatted; + details = '
' + escapeHtml(value) + '
'; + } + const summary = kind === 'added' ? 'Value added (click to view)' : kind === 'removed' ? 'Value removed (click to view)' : 'Value modified (click to compare)'; + row.innerHTML = '
' + badgeText + '' + escapeHtml(key) + '' + summary + '
'; + } else if (kind === 'modified') { + row.innerHTML = '' + badgeText + '' + escapeHtml(key) + '' + escapeHtml(beforeFormatted) + '' + escapeHtml(afterFormatted) + ''; + } else { + const value = kind === 'removed' ? beforeFormatted : afterFormatted; + const color = kind === 'added' ? 'var(--pass)' : kind === 'removed' ? 'var(--fail)' : 'var(--accent)'; + row.innerHTML = '' + badgeText + '' + escapeHtml(key) + '' + escapeHtml(value) + ''; + } + body.appendChild(row); + }); + Object.keys(state).filter(k => !changes.some(c => c.key === k)).sort().forEach(key => { + const formatted = prettyFormatStorageValue(state[key]); + const row = document.createElement('div'); + row.className = 'terminal-row'; + const isLong = formatted.length > 80 || formatted.includes('\n'); + if (isLong) { + row.className = 'terminal-row storage-collapsible'; + row.innerHTML = '
VAL' + escapeHtml(key) + 'Value view (click to inspect)
'; + } else { + row.innerHTML = 'VAL' + escapeHtml(key) + '' + escapeHtml(formatStorageValue(state[key])) + ''; + } + body.appendChild(row); + }); + } + section.appendChild(body); + target.appendChild(section); + } + appendSimpleStorageSection('Secure Storage', 'secureStorageChanges', 'secureStorage', secureStoragePanel); + appendSimpleStorageSection('Keychain', 'keychainChanges', 'keychain', keychainPanel); + const shotsList = document.getElementById('modal-screenshots-list'); shotsList.innerHTML = ''; const screenshots = data.screenshots || []; diff --git a/tools/ensemble_test_runner/lib/reporters/html_test_report_css.dart b/tools/ensemble_test_runner/lib/reporters/html_test_report_css.dart index cccd905a0..b3c1cf2c3 100644 --- a/tools/ensemble_test_runner/lib/reporters/html_test_report_css.dart +++ b/tools/ensemble_test_runner/lib/reporters/html_test_report_css.dart @@ -57,7 +57,12 @@ body { .hero { padding-top: 24px; - padding-bottom: 8px; + padding-bottom: 16px; + display: flex; + justify-content: space-between; + align-items: flex-end; + flex-wrap: wrap; + gap: 16px; } .hero-header h1 { margin: 0; @@ -541,6 +546,43 @@ body { border-color: var(--accent); color: #000; } +.storage-sub-tabs { + display: flex; + gap: 8px; + padding: 0 0 12px; + border-bottom: 1px solid var(--border); + flex-wrap: wrap; +} +.storage-sub-tab-btn { + background: rgba(255, 255, 255, 0.03); + border: 1px solid var(--border); + border-radius: 8px; + color: var(--text-muted); + padding: 7px 14px; + font-size: 0.78rem; + font-weight: 800; + cursor: pointer; + font-family: var(--font-ui); + transition: all 0.2s ease; +} +.storage-sub-tab-btn:hover { + background: rgba(255, 255, 255, 0.08); + color: #fff; +} +.storage-sub-tab-btn.active { + background: var(--accent); + border-color: var(--accent); + color: #000; +} +.storage-sub-panels { + min-height: 80px; +} +.storage-sub-panel { + min-height: 80px; +} +.storage-sub-panel.logs-terminal { + padding: 12px 16px; +} @media (max-width: 800px) { .run-selector-row { @@ -716,38 +758,194 @@ body { /* Screens Flow Timeline Journey */ .flow-timeline { - background: rgba(0, 0, 0, 0.2); + background: linear-gradient(180deg, rgba(6, 182, 212, 0.04), rgba(0, 0, 0, 0.22)); border: 1px solid var(--border); - border-radius: 12px; - padding: 16px 20px; + border-radius: 14px; + padding: 16px 18px 18px; margin-bottom: 20px; } +.flow-header { + display: flex; + align-items: center; + justify-content: space-between; + gap: 12px; + flex-wrap: wrap; + margin-bottom: 14px; +} .flow-label { font-size: 0.7rem; font-weight: 800; text-transform: uppercase; color: var(--text-muted); - letter-spacing: 0.05em; - margin-bottom: 8px; + letter-spacing: 0.08em; + margin: 0; } -.flow-track { +.flow-meta { display: flex; align-items: center; + gap: 8px; flex-wrap: wrap; - gap: 8px 12px; - font-size: 0.9rem; } -.flow-node { - background: rgba(255, 255, 255, 0.05); +.flow-meta-chip { + display: inline-flex; + align-items: center; + gap: 6px; + font-size: 0.72rem; + font-weight: 600; + color: var(--text-muted); + background: rgba(255, 255, 255, 0.04); border: 1px solid var(--border); - border-radius: 6px; - padding: 4px 10px; + border-radius: 999px; + padding: 3px 10px; +} +.flow-meta-chip strong { + color: var(--text); font-weight: 700; - color: #fff; } -.flow-arrow { +.flow-meta-chip.prefix { color: var(--accent); - font-weight: 800; + background: rgba(6, 182, 212, 0.08); + border-color: rgba(6, 182, 212, 0.2); + font-family: var(--font-code); + font-size: 0.68rem; + max-width: 280px; + overflow: hidden; + text-overflow: ellipsis; + white-space: nowrap; +} +.flow-track { + display: flex; + align-items: stretch; + flex-wrap: wrap; + gap: 10px 0; + row-gap: 12px; +} +.flow-node { + position: relative; + appearance: none; + -webkit-appearance: none; + display: inline-flex; + align-items: center; + gap: 8px; + max-width: 100%; + background: rgba(255, 255, 255, 0.035); + border: 1px solid rgba(255, 255, 255, 0.08); + border-radius: 10px; + padding: 7px 12px 7px 8px; + color: var(--text); + font-family: var(--font-ui); + font-size: 0.82rem; + font-weight: 600; + line-height: 1.2; + margin: 0 28px 0 0; + transition: border-color 0.15s ease, background 0.15s ease, box-shadow 0.15s ease, transform 0.15s ease; +} +.flow-node:last-child { + margin-right: 0; +} +.flow-node:not(:last-child)::after { + content: ''; + position: absolute; + left: calc(100% + 8px); + top: 50%; + width: 12px; + height: 2px; + background: linear-gradient(90deg, rgba(6, 182, 212, 0.65), rgba(6, 182, 212, 0.15)); + border-radius: 2px; + transform: translateY(-50%); + pointer-events: none; +} +.flow-node:not(:last-child)::before { + content: ''; + position: absolute; + left: calc(100% + 16px); + top: 50%; + width: 5px; + height: 5px; + border-right: 2px solid rgba(6, 182, 212, 0.55); + border-top: 2px solid rgba(6, 182, 212, 0.55); + transform: translateY(-50%) rotate(45deg); + pointer-events: none; +} +.flow-idx { + flex-shrink: 0; + display: inline-flex; + align-items: center; + justify-content: center; + width: 22px; + height: 22px; + border-radius: 50%; + background: rgba(6, 182, 212, 0.12); + border: 1px solid rgba(6, 182, 212, 0.28); + color: var(--accent); + font-family: var(--font-code); + font-size: 0.68rem; + font-weight: 700; +} +.flow-name { + overflow: hidden; + text-overflow: ellipsis; + white-space: nowrap; + max-width: 180px; +} +.flow-node.start { + border-color: rgba(16, 185, 129, 0.35); + background: rgba(16, 185, 129, 0.08); +} +.flow-node.start .flow-idx { + background: rgba(16, 185, 129, 0.18); + border-color: rgba(16, 185, 129, 0.4); + color: var(--pass); +} +.flow-node.end { + border-color: rgba(6, 182, 212, 0.4); + background: rgba(6, 182, 212, 0.08); +} +.flow-node.revisit { + border-style: dashed; +} +.flow-revisit { + flex-shrink: 0; + font-size: 0.62rem; + font-weight: 700; + letter-spacing: 0.02em; + text-transform: uppercase; + color: var(--text-muted); + background: rgba(255, 255, 255, 0.06); + border-radius: 999px; + padding: 2px 6px; +} +.flow-node.interactive { + cursor: pointer; + text-align: left; +} +.flow-node.interactive:hover { + border-color: rgba(6, 182, 212, 0.55); + background: rgba(6, 182, 212, 0.12); + box-shadow: 0 0 0 3px rgba(6, 182, 212, 0.1); + transform: translateY(-1px); +} +.flow-node.interactive:focus-visible { + outline: 2px solid var(--accent); + outline-offset: 2px; +} +.flow-inspect { + flex-shrink: 0; + width: 6px; + height: 6px; + border-radius: 50%; + background: var(--accent); + box-shadow: 0 0 6px rgba(6, 182, 212, 0.7); + opacity: 0.85; +} +@media (max-width: 640px) { + .flow-name { + max-width: 140px; + } + .flow-node { + margin-right: 22px; + padding: 6px 10px 6px 7px; + } } /* Timeline steps */ @@ -920,6 +1118,12 @@ body { gap: 16px; margin-top: 24px; } +.storage-grid-container { + display: grid; + grid-template-columns: repeat(auto-fit, minmax(320px, 1fr)); + gap: 16px; + margin-top: 16px; +} .logs-card-pane { background: rgba(0, 0, 0, 0.35); border: 1px solid var(--border); @@ -1806,38 +2010,28 @@ a:hover { /* App Tab Navigation & History Details */ .app-tab-navigation { display: flex; - gap: 8px; - background: rgba(17, 24, 39, 0.45); - border: 1px solid rgba(255, 255, 255, 0.08); - border-radius: 30px; - padding: 4px; - margin: 16px auto 24px auto; - width: fit-content; - box-shadow: 0 4px 12px rgba(0, 0, 0, 0.2); + gap: 6px; } .app-tab-btn { - background: transparent; - border: none; - color: #9ca3af; - padding: 8px 20px; + background: rgba(255, 255, 255, 0.03); + border: 1px solid var(--border); + border-radius: 8px; + color: var(--text-muted); + padding: 8px 16px; + font-size: 0.85rem; + font-weight: 700; cursor: pointer; - font-size: 0.88rem; font-family: var(--font-ui); - font-weight: 600; - border-radius: 20px; transition: all 0.2s ease; - display: flex; - align-items: center; - gap: 6px; } .app-tab-btn:hover { + background: rgba(255, 255, 255, 0.08); color: #fff; - background: rgba(255, 255, 255, 0.05); } .app-tab-btn.active { - color: #fff; background: var(--accent); - box-shadow: 0 0 12px rgba(6, 182, 212, 0.35); + border-color: var(--accent); + color: #000; } .history-caret { display: inline-block; diff --git a/tools/ensemble_test_runner/lib/reporters/html_test_reporter.dart b/tools/ensemble_test_runner/lib/reporters/html_test_reporter.dart index 2e2a6206b..4b2d6faba 100644 --- a/tools/ensemble_test_runner/lib/reporters/html_test_reporter.dart +++ b/tools/ensemble_test_runner/lib/reporters/html_test_reporter.dart @@ -120,13 +120,13 @@ class HtmlTestReporter { ..writeln('

Ensemble Test Runner

') ..writeln('

') ..writeln(' ') - ..writeln('') - ..writeln('
') + ..writeln('
') ..writeln( - ' ') + ' ') ..writeln( - ' ') - ..writeln('
') + ' ') + ..writeln('
') + ..writeln('') ..writeln('
') ..writeln('
') ..writeln('
') diff --git a/tools/ensemble_test_runner/lib/reporters/report_json_optimizer.dart b/tools/ensemble_test_runner/lib/reporters/report_json_optimizer.dart index 769788d3e..12eb52945 100644 --- a/tools/ensemble_test_runner/lib/reporters/report_json_optimizer.dart +++ b/tools/ensemble_test_runner/lib/reporters/report_json_optimizer.dart @@ -153,6 +153,19 @@ class ReportJsonOptimizer { out['storageChanges'] = optimizedChanges; } } + for (final field in const ['secureStorageChanges', 'keychainChanges']) { + final secureChanges = step[field]; + if (secureChanges is List && secureChanges.isNotEmpty) { + final optimizedChanges = >[]; + for (final c in secureChanges) { + if (c is! Map) continue; + final optimized = + optimizeStorageChange(Map.from(c)); + if (optimized != null) optimizedChanges.add(optimized); + } + if (optimizedChanges.isNotEmpty) out[field] = optimizedChanges; + } + } final shots = step['screenshots']; if (shots is List && shots.isNotEmpty) { @@ -222,16 +235,22 @@ class ReportJsonOptimizer { } final storage = test['storage']; - if (storage is Map && storage['keys'] is Map) { - final keys = storage['keys'] as Map; - if (keys.isNotEmpty) { - out['storage'] = { - 'keys': { + if (storage is Map) { + final storageOut = {}; + for (final entry in const [ + ('keys', 'keys'), + ('secureStorageKeys', 'secureStorageKeys'), + ('keychainKeys', 'keychainKeys'), + ]) { + final keys = storage[entry.$1]; + if (keys is Map && keys.isNotEmpty) { + storageOut[entry.$2] = { for (final e in keys.entries) e.key.toString(): maybeIntern(e.value), - }, - }; + }; + } } + if (storageOut.isNotEmpty) out['storage'] = storageOut; } final steps = test['steps']; @@ -329,21 +348,30 @@ class ReportJsonOptimizer { 'apiCalls': step['apiCalls'] ?? const [], 'appLogs': step['appLogs'] ?? const [], 'storageChanges': step['storageChanges'] ?? const [], + 'secureStorageChanges': step['secureStorageChanges'] ?? const [], + 'keychainChanges': step['keychainChanges'] ?? const [], 'screenshots': step['screenshots'] ?? const [], }; step.putIfAbsent('apiCalls', () => const []); step.putIfAbsent('appLogs', () => const []); step.putIfAbsent('storageChanges', () => const []); + step.putIfAbsent('secureStorageChanges', () => const []); + step.putIfAbsent('keychainChanges', () => const []); step.putIfAbsent('screenshots', () => const []); } else if (parentPayload != null) { step['apiCalls'] = parentPayload['apiCalls']; step['appLogs'] = parentPayload['appLogs']; step['storageChanges'] = parentPayload['storageChanges']; + step['secureStorageChanges'] = + parentPayload['secureStorageChanges']; + step['keychainChanges'] = parentPayload['keychainChanges']; step['screenshots'] = parentPayload['screenshots']; } else { step['apiCalls'] = const []; step['appLogs'] = const []; step['storageChanges'] = const []; + step['secureStorageChanges'] = const []; + step['keychainChanges'] = const []; step['screenshots'] = const []; } } diff --git a/tools/ensemble_test_runner/lib/reporters/step_log_grouping.dart b/tools/ensemble_test_runner/lib/reporters/step_log_grouping.dart index e0a23c07b..a4233f16e 100644 --- a/tools/ensemble_test_runner/lib/reporters/step_log_grouping.dart +++ b/tools/ensemble_test_runner/lib/reporters/step_log_grouping.dart @@ -67,6 +67,8 @@ List> groupLogsByStep({ required List> apiEvents, required List rawConsoleLines, List> storageSteps = const [], + List> secureStorageSteps = const [], + List> keychainSteps = const [], List> screenshotFrames = const [], }) { final result = >[]; @@ -84,6 +86,8 @@ List> groupLogsByStep({ 'apiCalls': >[], 'appLogs': [], 'storageChanges': >[], + 'secureStorageChanges': >[], + 'keychainChanges': >[], 'screenshots': >[], }, ); @@ -166,21 +170,30 @@ List> groupLogsByStep({ (buckets[outlineIndex]['appLogs'] as List).add(line); } - for (final step in storageSteps) { - final targetTopLevel = resolveTopLevel(step); - if (targetTopLevel == null) continue; - final outlineIndex = outlineIndexForTopLevel(targetTopLevel); - if (outlineIndex == null) continue; - final changes = step['changes']; - if (changes is! List) continue; - final bucket = buckets[outlineIndex]['storageChanges'] as List; - for (final change in changes) { - if (change is Map) { - bucket.add(Map.from(change)); + void addStorageSteps( + List> steps, + String bucketName, + ) { + for (final step in steps) { + final targetTopLevel = resolveTopLevel(step); + if (targetTopLevel == null) continue; + final outlineIndex = outlineIndexForTopLevel(targetTopLevel); + if (outlineIndex == null) continue; + final changes = step['changes']; + if (changes is! List) continue; + final bucket = buckets[outlineIndex][bucketName] as List; + for (final change in changes) { + if (change is Map) { + bucket.add(Map.from(change)); + } } } } + addStorageSteps(storageSteps, 'storageChanges'); + addStorageSteps(secureStorageSteps, 'secureStorageChanges'); + addStorageSteps(keychainSteps, 'keychainChanges'); + for (final frame in screenshotFrames) { final targetTopLevel = resolveTopLevel(frame); if (targetTopLevel == null) continue; @@ -201,6 +214,8 @@ List> groupLogsByStep({ 'apiCalls': >[], 'appLogs': [], 'storageChanges': >[], + 'secureStorageChanges': >[], + 'keychainChanges': >[], 'screenshots': >[], }); } diff --git a/tools/ensemble_test_runner/lib/reporters/test_report_document.dart b/tools/ensemble_test_runner/lib/reporters/test_report_document.dart index 2dd5dcab7..d8259112d 100644 --- a/tools/ensemble_test_runner/lib/reporters/test_report_document.dart +++ b/tools/ensemble_test_runner/lib/reporters/test_report_document.dart @@ -170,7 +170,17 @@ class TestReportDocument { stepStartTimes: report.stepStartTimes, apiEvents: apiEvents, rawConsoleLines: console, - storageSteps: (storage['steps'] as List?) + storageSteps: (storage['storageSteps'] as List?) + ?.whereType() + .map((e) => Map.from(e)) + .toList() ?? + const [], + secureStorageSteps: (storage['secureStorageSteps'] as List?) + ?.whereType() + .map((e) => Map.from(e)) + .toList() ?? + const [], + keychainSteps: (storage['keychainSteps'] as List?) ?.whereType() .map((e) => Map.from(e)) .toList() ?? @@ -204,8 +214,14 @@ class TestReportDocument { if (test.failedStepIndex != null) 'failedStepIndex': test.failedStepIndex, if (report != null) 'report': report.toJson(), 'storage': { - 'keys': storage['keys'] is Map - ? Map.from(storage['keys'] as Map) + 'keys': storage['storageKeys'] is Map + ? Map.from(storage['storageKeys'] as Map) + : {}, + 'secureStorageKeys': storage['secureStorageKeys'] is Map + ? Map.from(storage['secureStorageKeys'] as Map) + : {}, + 'keychainKeys': storage['keychainKeys'] is Map + ? Map.from(storage['keychainKeys'] as Map) : {}, }, 'steps': steps, @@ -264,36 +280,88 @@ class TestReportDocument { String displayRoot, ) { final ref = _first(artifacts, 'storage'); - if (ref == null) return {'keys': {}, 'steps': []}; + if (ref == null) return _emptyStorageReport(); final fsPath = filesystemPath(ref.path, artifactRoot: artifactRoot, displayRoot: displayRoot); if (fsPath == null || !File(fsPath).existsSync()) { - return {'keys': {}, 'steps': []}; + return _emptyStorageReport(); } try { final decoded = json.decode(File(fsPath).readAsStringSync()); if (decoded is Map) { + if (decoded['storage'] is Map || decoded['secureStorage'] is Map) { + final publicSection = decoded['storage'] is Map + ? Map.from(decoded['storage'] as Map) + : {}; + final secureSection = decoded['secureStorage'] is Map + ? Map.from(decoded['secureStorage'] as Map) + : {}; + final keychainSection = decoded['keychain'] is Map + ? Map.from(decoded['keychain'] as Map) + : {}; + return { + 'storageKeys': publicSection['keys'] is Map + ? Map.from(publicSection['keys'] as Map) + : {}, + 'secureStorageKeys': secureSection['keys'] is Map + ? Map.from(secureSection['keys'] as Map) + : {}, + 'keychainKeys': keychainSection['keys'] is Map + ? Map.from(keychainSection['keys'] as Map) + : {}, + 'storageSteps': [ + for (final step in (publicSection['steps'] as List? ?? const [])) + if (step is Map) Map.from(step), + ], + 'secureStorageSteps': [ + for (final step in (secureSection['steps'] as List? ?? const [])) + if (step is Map) Map.from(step), + ], + 'keychainSteps': [ + for (final step + in (keychainSection['steps'] as List? ?? const [])) + if (step is Map) Map.from(step), + ], + }; + } if (decoded['keys'] is Map || decoded['steps'] is List) { return { - 'keys': decoded['keys'] is Map + 'storageKeys': decoded['keys'] is Map ? Map.from(decoded['keys'] as Map) : {}, - 'steps': [ + 'secureStorageKeys': {}, + 'keychainKeys': {}, + 'storageSteps': [ for (final step in (decoded['steps'] as List? ?? const [])) if (step is Map) Map.from(step), ], + 'secureStorageSteps': >[], + 'keychainSteps': >[], }; } // Legacy flat storage snapshot. return { - 'keys': Map.from(decoded), - 'steps': >[], + 'storageKeys': Map.from(decoded), + 'secureStorageKeys': {}, + 'keychainKeys': {}, + 'storageSteps': >[], + 'secureStorageSteps': >[], + 'keychainSteps': >[], }; } } catch (_) {} - return {'keys': {}, 'steps': []}; + return _emptyStorageReport(); } + static Map _emptyStorageReport() => { + 'storageKeys': {}, + 'secureStorageKeys': {}, + 'keychainKeys': {}, + 'storageSteps': >[], + 'secureStorageSteps': >[], + 'keychainSteps': >[], + }; + static List> _readScreenshotFrames( List<_ArtifactRef> artifacts, String artifactRoot, diff --git a/tools/ensemble_test_runner/lib/runner/app_session_snapshot.dart b/tools/ensemble_test_runner/lib/runner/app_session_snapshot.dart index 983672900..bc677f36a 100644 --- a/tools/ensemble_test_runner/lib/runner/app_session_snapshot.dart +++ b/tools/ensemble_test_runner/lib/runner/app_session_snapshot.dart @@ -7,11 +7,13 @@ import 'package:flutter/widgets.dart'; /// In-memory app state captured from a successful session-producing test. class AppSessionSnapshot { final Map publicStorage; + final Map secureStorage; final Map keychain; final Locale? locale; const AppSessionSnapshot({ required this.publicStorage, + required this.secureStorage, required this.keychain, this.locale, }); @@ -19,12 +21,18 @@ class AppSessionSnapshot { static Future capture() async { final storage = StorageManager(); final publicStorage = {}; + final secureStorage = {}; for (final key in storage.getKeys()) { - publicStorage[key] = _copy(storage.read(key)); + if (key.startsWith('enc_')) { + secureStorage[key] = _copy(storage.read(key)); + } else { + publicStorage[key] = _copy(storage.read(key)); + } } final keychain = await storage.getAllFromKeychain(); return AppSessionSnapshot( publicStorage: publicStorage, + secureStorage: secureStorage, keychain: { for (final entry in keychain.entries) entry.key: _copy(entry.value), }, @@ -35,6 +43,10 @@ class AppSessionSnapshot { Future restore() async { final storage = StorageManager(); await storage.clearPublicStorage(); + for (final key + in storage.getKeys().where((key) => key.startsWith('enc_')).toList()) { + await storage.remove(key); + } final currentKeychain = await storage.getAllFromKeychain(); for (final key in currentKeychain.keys) { await storage.removeSecurely(key); @@ -42,6 +54,9 @@ class AppSessionSnapshot { for (final entry in publicStorage.entries) { await storage.write(entry.key, _copy(entry.value)); } + for (final entry in secureStorage.entries) { + await storage.write(entry.key, _copy(entry.value)); + } for (final entry in keychain.entries) { await storage.writeSecurely(key: entry.key, value: _copy(entry.value)); } diff --git a/tools/ensemble_test_runner/lib/runner/debug_artifact_logs.dart b/tools/ensemble_test_runner/lib/runner/debug_artifact_logs.dart index b076619a4..1067824c5 100644 --- a/tools/ensemble_test_runner/lib/runner/debug_artifact_logs.dart +++ b/tools/ensemble_test_runner/lib/runner/debug_artifact_logs.dart @@ -159,6 +159,8 @@ Future writeStorageLog( filePrefix: context.testCase.id, key: key, stepDiffs: context.runtime.storageStepDiffs, + secureStepDiffs: context.runtime.secureStorageStepDiffs, + keychainStepDiffs: context.runtime.keychainStepDiffs, ); } @@ -189,8 +191,10 @@ Future writeStorageLogFile({ required String filePrefix, String? key, List stepDiffs = const [], + List secureStepDiffs = const [], + List keychainStepDiffs = const [], Map? keys, -}) { +}) async { if (key != null && key.isNotEmpty) { return logger.writeLogFile( testId: filePrefix, @@ -200,15 +204,27 @@ Future writeStorageLogFile({ ); } - final snapshot = keys ?? - { - for (final storageKey in StorageManager().getKeys()) - storageKey: StorageManager().read(storageKey), - }; - final content = _prettyJson({ + final snapshot = keys ?? capturePublicStorage(); + final contentMap = { + // Keep the legacy top-level fields for consumers that only understand + // public storage artifacts. 'keys': snapshot, 'steps': [for (final diff in stepDiffs) diff.toJson()], - }); + 'storage': { + 'keys': snapshot, + 'steps': [for (final diff in stepDiffs) diff.toJson()], + }, + 'secureStorage': { + 'keys': keys == null ? captureSecureStorage() : {}, + 'steps': [for (final diff in secureStepDiffs) diff.toJson()], + }, + 'keychain': { + 'keys': + keys == null ? await captureKeychainStorage() : {}, + 'steps': [for (final diff in keychainStepDiffs) diff.toJson()], + }, + }; + final content = _prettyJson(contentMap); return logger.writeLogFile( testId: filePrefix, name: 'storage', diff --git a/tools/ensemble_test_runner/lib/runner/ensemble_test_context.dart b/tools/ensemble_test_runner/lib/runner/ensemble_test_context.dart index c1e0316a1..b5eb20f57 100644 --- a/tools/ensemble_test_runner/lib/runner/ensemble_test_context.dart +++ b/tools/ensemble_test_runner/lib/runner/ensemble_test_context.dart @@ -37,6 +37,7 @@ class EnsembleTestContext { ); final storage = testCase.initialState['storage']; + final secureStorage = testCase.initialState['secureStorage']; final keychain = testCase.initialState['keychain']; final env = testCase.initialState['env']; @@ -47,6 +48,9 @@ class EnsembleTestContext { envOverrides: envMap.isEmpty ? null : envMap, initialPublicStorage: storage is Map ? Map.from(storage) : null, + initialSecureStorage: secureStorage is Map + ? Map.from(secureStorage) + : null, initialKeychain: keychain is Map ? Map.from(keychain) : null, ); diff --git a/tools/ensemble_test_runner/lib/runner/ensemble_test_harness.dart b/tools/ensemble_test_runner/lib/runner/ensemble_test_harness.dart index 386274d88..93825de9b 100644 --- a/tools/ensemble_test_runner/lib/runner/ensemble_test_harness.dart +++ b/tools/ensemble_test_runner/lib/runner/ensemble_test_harness.dart @@ -8,6 +8,7 @@ import 'package:ensemble/framework/apiproviders/http_api_provider.dart'; import 'package:ensemble/framework/definition_providers/local_provider.dart'; import 'package:ensemble/framework/screen_tracker.dart'; import 'package:ensemble/framework/storage_manager.dart'; +import 'package:ensemble/framework/encrypted_storage_manager.dart'; import 'package:ensemble/page_model.dart'; import 'package:ensemble/screen_controller.dart'; import 'package:ensemble/util/utils.dart'; @@ -31,11 +32,13 @@ import 'package:yaml/yaml.dart'; class EnsembleTestSetup { final Map? envOverrides; final Map? initialPublicStorage; + final Map? initialSecureStorage; final Map? initialKeychain; const EnsembleTestSetup({ this.envOverrides, this.initialPublicStorage, + this.initialSecureStorage, this.initialKeychain, }); } @@ -54,6 +57,20 @@ Future applyYamlTestStorageBootstrap(EnsembleTestSetup setup) async { const Iterable>.empty()) { await StorageManager().write(entry.key, entry.value); } + for (final entry in setup.initialSecureStorage?.entries ?? + const Iterable>.empty()) { + EncryptedStorageManager.setSecureStorage({ + 'key': entry.key, + 'value': entry.value, + }); + // The runtime API is synchronous for compatibility with released + // Ensemble versions, while GetStorage persists asynchronously. Wait for + // the backend entry to become observable before mounting the app. + for (var attempt = 0; attempt < 50; attempt++) { + if (StorageManager().read('enc_${entry.key}') != null) break; + await Future.delayed(const Duration(milliseconds: 10)); + } + } for (final entry in setup.initialKeychain?.entries ?? const Iterable>.empty()) { await StorageManager().writeSecurely(key: entry.key, value: entry.value); diff --git a/tools/ensemble_test_runner/lib/runner/ensemble_test_runner.dart b/tools/ensemble_test_runner/lib/runner/ensemble_test_runner.dart index b33d01cf0..db4e1ad2c 100644 --- a/tools/ensemble_test_runner/lib/runner/ensemble_test_runner.dart +++ b/tools/ensemble_test_runner/lib/runner/ensemble_test_runner.dart @@ -447,6 +447,8 @@ class EnsembleTestRunner { ctx.runtime.currentStepIndex = i; stepStartTimes.add(startTime.toIso8601String()); final storageBefore = capturePublicStorage(); + final secureStorageBefore = captureSecureStorage(); + final keychainBefore = await captureKeychainStorage(); var capturedStep = false; try { _throwIfUnexpectedFlutterExceptions( @@ -546,10 +548,12 @@ class EnsembleTestRunner { capturedStep = true; } await YamlTestSession.navigationFlow.flushPending(); - _recordStorageStepDiff( + await _recordStorageStepDiff( ctx: ctx, stepIndex: i, before: storageBefore, + secureBefore: secureStorageBefore, + keychainBefore: keychainBefore, ); stepDurationsMs.add( DateTime.now().difference(startTime).inMilliseconds, @@ -565,10 +569,12 @@ class EnsembleTestRunner { ); _captureScreenArtifacts(ctx); } catch (error, stackTrace) { - _recordStorageStepDiff( + await _recordStorageStepDiff( ctx: ctx, stepIndex: i, before: storageBefore, + secureBefore: secureStorageBefore, + keychainBefore: keychainBefore, ); stepDurationsMs.add( DateTime.now().difference(startTime).inMilliseconds, @@ -1414,20 +1420,40 @@ class EnsembleTestRunner { } } - void _recordStorageStepDiff({ + Future _recordStorageStepDiff({ required EnsembleTestContext ctx, required int stepIndex, required Map before, - }) { + required Map secureBefore, + required Map keychainBefore, + }) async { final changes = diffStorage(before, capturePublicStorage()); - if (changes.isEmpty) return; - ctx.runtime.storageStepDiffs.add( - StorageStepDiff( + final secureChanges = diffStorage(secureBefore, captureSecureStorage()); + final keychainChanges = diffStorage( + keychainBefore, + await captureKeychainStorage(), + ); + if (changes.isNotEmpty) { + ctx.runtime.storageStepDiffs.add(StorageStepDiff( stepIndex: stepIndex, timestamp: DateTime.now(), changes: changes, - ), - ); + )); + } + if (secureChanges.isNotEmpty) { + ctx.runtime.secureStorageStepDiffs.add(StorageStepDiff( + stepIndex: stepIndex, + timestamp: DateTime.now(), + changes: secureChanges, + )); + } + if (keychainChanges.isNotEmpty) { + ctx.runtime.keychainStepDiffs.add(StorageStepDiff( + stepIndex: stepIndex, + timestamp: DateTime.now(), + changes: keychainChanges, + )); + } } void _replaceArtifactLog(TestLogger logger, String label, String path) { diff --git a/tools/ensemble_test_runner/lib/runner/storage_step_diff.dart b/tools/ensemble_test_runner/lib/runner/storage_step_diff.dart index 5ceefe931..5984a309d 100644 --- a/tools/ensemble_test_runner/lib/runner/storage_step_diff.dart +++ b/tools/ensemble_test_runner/lib/runner/storage_step_diff.dart @@ -1,6 +1,7 @@ import 'dart:convert'; import 'package:ensemble/framework/storage_manager.dart'; +import 'package:ensemble/framework/encrypted_storage_manager.dart'; /// One public-storage key change during a test step. class StorageKeyChange { @@ -48,11 +49,42 @@ Map capturePublicStorage() { final storage = StorageManager(); final result = {}; for (final key in storage.getKeys()) { - result[key] = _copy(storage.read(key)); + if (!key.startsWith('enc_')) { + result[key] = _copy(storage.read(key)); + } } return result; } +/// Captures Ensemble encrypted public-storage values using their logical keys. +/// +/// The encrypted backend stores values under `enc_` in public GetStorage. +/// Reports expose the logical key and decoded value instead. +Map captureSecureStorage() { + final storage = StorageManager(); + final result = {}; + for (final backendKey in storage.getKeys()) { + if (!backendKey.startsWith('enc_')) continue; + final key = backendKey.substring('enc_'.length); + final raw = storage.read(backendKey); + final decoded = EncryptedStorageManager.getSecureStorage(key); + result[key] = decoded ?? + { + 'value': _copy(raw), + 'decodeError': true, + }; + } + return result; +} + +/// Captures platform keychain / FlutterSecureStorage values. +Future> captureKeychainStorage() async { + final values = await StorageManager().getAllFromKeychain(); + return { + for (final entry in values.entries) entry.key: _copy(entry.value), + }; +} + /// Diff of public storage maps. Returns only keys that changed. List diffStorage( Map before, diff --git a/tools/ensemble_test_runner/lib/runner/test_runtime_state.dart b/tools/ensemble_test_runner/lib/runner/test_runtime_state.dart index 643ef28f6..0bda52a38 100644 --- a/tools/ensemble_test_runner/lib/runner/test_runtime_state.dart +++ b/tools/ensemble_test_runner/lib/runner/test_runtime_state.dart @@ -24,6 +24,8 @@ class TestRuntimeState { /// Public-storage diffs captured at the end of each top-level step. final List storageStepDiffs = []; + final List secureStorageStepDiffs = []; + final List keychainStepDiffs = []; /// Map from screen name to its captured artifacts (debugTree, performance markers, etc.) final Map> screenArtifacts = {}; @@ -42,6 +44,8 @@ class TestRuntimeState { themeMode = null; currentStepIndex = null; storageStepDiffs.clear(); + secureStorageStepDiffs.clear(); + keychainStepDiffs.clear(); screenArtifacts.clear(); } diff --git a/tools/ensemble_test_runner/lib/schema/ensemble_test_schema_builder.dart b/tools/ensemble_test_runner/lib/schema/ensemble_test_schema_builder.dart index 96d797f9c..f5ecca159 100644 --- a/tools/ensemble_test_runner/lib/schema/ensemble_test_schema_builder.dart +++ b/tools/ensemble_test_runner/lib/schema/ensemble_test_schema_builder.dart @@ -25,6 +25,7 @@ class EnsembleTestSchemaBuilder { 'additionalProperties': false, 'properties': { 'storage': _mapOrPlaceholderDef(), + 'secureStorage': _mapOrPlaceholderDef(), 'keychain': _mapOrPlaceholderDef(), 'env': _mapOrPlaceholderDef(), }, @@ -376,7 +377,7 @@ class EnsembleTestSchemaBuilder { 'initialState': { '\$ref': '#/\$defs/initialState', 'description': - 'Suite-wide storage/keychain/env applied before each test ' + 'Suite-wide storage/secureStorage/keychain/env applied before each test ' 'initialState. Test values override suite values per key.', }, 'profiles': { diff --git a/tools/ensemble_test_runner/test/app_session_snapshot_test.dart b/tools/ensemble_test_runner/test/app_session_snapshot_test.dart index d22d83fe3..0184761eb 100644 --- a/tools/ensemble_test_runner/test/app_session_snapshot_test.dart +++ b/tools/ensemble_test_runner/test/app_session_snapshot_test.dart @@ -6,7 +6,8 @@ import 'package:flutter_test/flutter_test.dart'; void main() { TestWidgetsFlutterBinding.ensureInitialized(); - test('restores public storage and keychain from an isolated copy', () async { + test('restores public, secure storage, and keychain from an isolated copy', + () async { EnsembleTestHarness.ensureTestPlugins(); final storage = StorageManager(); await storage.init(); @@ -19,6 +20,7 @@ void main() { await storage.write('nested', { 'values': [1, 2] }); + await storage.write('enc_secureToken', 'encrypted-value'); await storage.writeSecurely(key: 'sahCookie', value: 'original'); final snapshot = await AppSessionSnapshot.capture(); @@ -26,6 +28,7 @@ void main() { final nested = storage.read('nested')!; (nested['values'] as List).add(3); await storage.write('extra', true); + await storage.write('enc_extraSecureToken', 'remove-me'); await storage.writeSecurely(key: 'sahCookie', value: 'changed'); await storage.writeSecurely(key: 'extraSecret', value: 'remove-me'); @@ -36,6 +39,8 @@ void main() { 'values': [1, 2] }); expect(storage.read('extra'), isNull); + expect(storage.read('enc_secureToken'), 'encrypted-value'); + expect(storage.read('enc_extraSecureToken'), isNull); expect(await storage.readSecurely('sahCookie'), 'original'); expect(await storage.readSecurely('extraSecret'), isNull); }); diff --git a/tools/ensemble_test_runner/test/ensemble_test_harness_test.dart b/tools/ensemble_test_runner/test/ensemble_test_harness_test.dart index 9a3b49780..fdfd59fec 100644 --- a/tools/ensemble_test_runner/test/ensemble_test_harness_test.dart +++ b/tools/ensemble_test_runner/test/ensemble_test_harness_test.dart @@ -1,6 +1,8 @@ import 'dart:io'; import 'package:ensemble/framework/storage_manager.dart'; +import 'package:ensemble/framework/encrypted_storage_manager.dart'; +import 'package:ensemble/framework/secrets.dart'; import 'package:ensemble_test_runner/runner/ensemble_test_harness.dart'; import 'package:flutter_test/flutter_test.dart'; @@ -103,4 +105,27 @@ appName=KPN InHome Dev {'token': 'abc'}, ); }); + + testWidgets('initial secureStorage state is encrypted into public storage', + (tester) async { + EnsembleTestHarness.ensureTestPlugins(); + SecretsStore().secretCache['encryptionKey'] = + '7OPUScfQ3OTmGZXx9EZ5q6lTsSDwiCUA'; + + await tester.runAsync( + () => applyYamlTestStorageBootstrap( + const EnsembleTestSetup( + initialSecureStorage: { + 'dnsBackup': {'hasBackup': true}, + }, + ), + ), + ); + + expect(StorageManager().read('enc_dnsBackup'), isA()); + expect( + EncryptedStorageManager.getSecureStorage('dnsBackup'), + {'hasBackup': true}, + ); + }); } diff --git a/tools/ensemble_test_runner/test/ensemble_test_parser_test.dart b/tools/ensemble_test_runner/test/ensemble_test_parser_test.dart index 213b77960..e3f0d97e5 100644 --- a/tools/ensemble_test_runner/test/ensemble_test_parser_test.dart +++ b/tools/ensemble_test_runner/test/ensemble_test_parser_test.dart @@ -182,6 +182,31 @@ steps: ); }); + test('parses secureStorage independently from keychain', () { + const yaml = ''' +id: encrypted_state +startScreen: Home +initialState: + storage: + publicFlag: true + secureStorage: + dnsBackup: + hasBackup: true + keychain: + authToken: secret +steps: + - expectVisible: + id: home +'''; + + final test = EnsembleTestParser.parseString(yaml); + expect(test.initialState['storage'], {'publicFlag': true}); + expect(test.initialState['secureStorage'], { + 'dnsBackup': {'hasBackup': true}, + }); + expect(test.initialState['keychain'], {'authToken': 'secret'}); + }); + test('parses a reusable session and pre-screen setup', () { const yaml = ''' id: home_from_session diff --git a/tools/ensemble_test_runner/test/ensemble_test_schema_test.dart b/tools/ensemble_test_runner/test/ensemble_test_schema_test.dart index 0692bb4b8..136cdc868 100644 --- a/tools/ensemble_test_runner/test/ensemble_test_schema_test.dart +++ b/tools/ensemble_test_runner/test/ensemble_test_schema_test.dart @@ -145,7 +145,7 @@ void main() { }); test( - 'initialState schema accepts storage, keychain, and env maps/placeholders', + 'initialState schema accepts storage, secureStorage, keychain, and env maps/placeholders', () { final schema = EnsembleTestSchemaBuilder.build(); final initialState = @@ -153,6 +153,7 @@ void main() { final properties = initialState['properties'] as Map; expect(properties, contains('storage')); + expect(properties, contains('secureStorage')); expect(properties, contains('keychain')); expect(properties, contains('env')); expect( diff --git a/tools/ensemble_test_runner/test/step_log_grouping_test.dart b/tools/ensemble_test_runner/test/step_log_grouping_test.dart index 7e1560eed..7237fe2a1 100644 --- a/tools/ensemble_test_runner/test/step_log_grouping_test.dart +++ b/tools/ensemble_test_runner/test/step_log_grouping_test.dart @@ -172,6 +172,40 @@ void main() { expect(changes[1]['change'], 'modified'); }); + test('places secure storage and keychain changes under matching step', () { + final grouped = groupLogsByStep( + stepsOutline: ['tap(a)', 'tap(b)'], + stepDurationsMs: [10, 10], + stepStartTimes: [ + '2026-07-22T12:00:00.000', + '2026-07-22T12:00:01.000', + ], + apiEvents: const [], + rawConsoleLines: const [], + secureStorageSteps: [ + { + 'stepIndex': 1, + 'changes': [ + {'key': 'dnsBackup', 'change': 'added', 'after': true}, + ], + }, + ], + keychainSteps: [ + { + 'stepIndex': 1, + 'changes': [ + {'key': 'authToken', 'change': 'modified', 'after': 'new'}, + ], + }, + ], + ); + + expect((grouped[1]['secureStorageChanges'] as List).single['key'], + 'dnsBackup'); + expect( + (grouped[1]['keychainChanges'] as List).single['key'], 'authToken'); + }); + test('places screenshot frames under matching stepIndex', () { final grouped = groupLogsByStep( stepsOutline: ['tap(a)', 'waitForNavigation(Home)'],
Run IDDate & TimeStatusDurationTestsBranch