From 0e4539e800c3db18ebd455774e99822d5c051cdd Mon Sep 17 00:00:00 2001 From: Matthew Thomas Date: Tue, 7 Apr 2026 23:11:37 +0100 Subject: [PATCH 1/8] spec: add identity and lineage model - Add normative Section 4.9 Identity and Lineage Model with uniqueness and stability tables for all seven identity layers - Add runId (root): logical test run identifier - Add testId (test): stable test-case identifier (string, not UUID-constrained) - Add executionId (test): per-execution identifier within a run - Add attemptId (retryAttempt): per-retry-attempt identifier - Add attachmentId (attachment): per-attachment-reference identifier - Add shardId (environment): shard/partition label - Update reportId description to clarify as document artifact identifier - Deprecate tests[].id in favour of testId - Update Section 2.4 design principle to cover full identity model - Update Section 19 conformance (producer and consumer) - Update Appendix B references from id to testId - Update Appendix D.5 comprehensive example with all identity fields - Update schema with all new optional fields and deprecation note --- schema/ctrf.schema.json | 32 ++++- spec/ctrf.md | 284 ++++++++++++++++++++++++++++++---------- 2 files changed, 246 insertions(+), 70 deletions(-) diff --git a/schema/ctrf.schema.json b/schema/ctrf.schema.json index 2ac8d8d..7387b42 100644 --- a/schema/ctrf.schema.json +++ b/schema/ctrf.schema.json @@ -15,10 +15,14 @@ "pattern": "^[0-9]+\\.[0-9]+\\.[0-9]+$" }, "reportId": { - "description": "Unique identifier for this report instance (UUID)", + "description": "Unique identifier for this CTRF document instance (UUID). Identifies the serialized artifact.", "type": "string", "format": "uuid" }, + "runId": { + "description": "Identifier for the logical test run. Shared across shards/documents belonging to the same run.", + "type": "string" + }, "timestamp": { "description": "Report generation time (RFC 3339 / ISO 8601)", "type": "string", @@ -132,10 +136,18 @@ "required": [ "name", "status", "duration" ], "properties": { "id": { - "description": "Unique, stable identifier for the test case (UUID)", + "description": "Deprecated: use testId instead. Unique, stable identifier for the test case (UUID)", "type": "string", "format": "uuid" }, + "testId": { + "description": "Stable identifier for the logical test case. Should be deterministic and stable across runs.", + "type": "string" + }, + "executionId": { + "description": "Unique identifier for this specific execution of the test case within a run. UUID recommended.", + "type": "string" + }, "name": { "description": "Name or title of the test case", "type": "string", @@ -255,6 +267,10 @@ "type": "integer", "minimum": 1 }, + "attemptId": { + "description": "Unique identifier for this individual retry attempt. UUID recommended.", + "type": "string" + }, "status": { "description": "Outcome of this attempt", "enum": [ @@ -315,6 +331,10 @@ "type": "object", "required": [ "name", "contentType", "path" ], "properties": { + "attachmentId": { + "description": "Unique identifier for this attachment reference instance. UUID recommended.", + "type": "string" + }, "name": { "description": "Display name of the attachment", "type": "string" @@ -385,6 +405,10 @@ "type": "object", "required": [ "name", "contentType", "path" ], "properties": { + "attachmentId": { + "description": "Unique identifier for this attachment reference instance. UUID recommended.", + "type": "string" + }, "name": { "description": "Display name of the attachment", "type": "string" @@ -546,6 +570,10 @@ "description": "Logical test environment (e.g., 'staging', 'production')", "type": "string" }, + "shardId": { + "description": "Shard or partition label identifying which part of a distributed run produced this document", + "type": "string" + }, "healthy": { "description": "Indicates if the run is considered healthy", "type": "boolean" diff --git a/spec/ctrf.md b/spec/ctrf.md index 1c1c21a..5d57573 100644 --- a/spec/ctrf.md +++ b/spec/ctrf.md @@ -143,18 +143,19 @@ Consumers MAY reconstruct hierarchical views for presentation purposes, but such --- -### 2.4. Deterministic Test Identity +### 2.4. Identity and Lineage -CTRF supports stable identification of test cases across runs. +CTRF provides a layered identity model that supports stable identification of test cases across runs, correlation of documents belonging to the same logical run, and traceability from individual attempts back to their parent execution and test case. -While test identifiers are OPTIONAL, producers are encouraged to generate deterministic identifiers (for example, using UUID version 5) derived from stable test attributes such as name, suite path, and file location. +Identity fields are OPTIONAL. When provided, they enable: -Deterministic identifiers enable: - -- cross-run analysis +- cross-run analysis and trend detection - flakiness detection -- historical trend analysis -- baseline comparisons +- shard and parallel-execution merging +- historical baseline comparisons +- artifact lineage tracking + +Producers are encouraged to assign identity fields when the corresponding correlation is meaningful for their workflow. The normative requirements for identity fields are defined in Section 4.9. --- @@ -411,6 +412,55 @@ Unless otherwise specified in this document, all timestamps and duration values This applies to all fields representing points in time or elapsed durations, including (but not limited to) `start`, `stop`, and `duration` fields at all levels. +### 4.9. Identity and Lineage Model + +CTRF defines a layered identity model that enables correlation of test data across documents, runs, test cases, and individual executions. + +Each layer addresses a distinct identification concern: + +1. **Document**: `reportId` identifies the emitted CTRF document as a serialized artifact. Each CTRF file SHOULD have exactly one `reportId`. +2. **Run**: `runId` identifies the logical test run. Multiple CTRF documents MAY share the same `runId` when they represent shards or partitions of a single coordinated execution. +3. **Test case**: `testId` identifies the logical test case. It SHOULD be deterministic and stable across runs, enabling cross-run analysis, trending, and flake detection. +4. **Execution**: `executionId` identifies a concrete execution of a test case within a run. An execution MAY include multiple retry attempts represented in `retryAttempts`. +5. **Attempt**: `attemptId` identifies an individual retry attempt within an execution. +6. **Attachment**: `attachmentId` identifies a specific attachment reference instance. +7. **Shard**: `shardId` labels the partition or shard that produced this document within a logical run. + +All identity fields are OPTIONAL. Producers SHOULD assign them when the corresponding correlation is meaningful for their workflow. + +#### 4.9.1. Uniqueness + +Each identity field has a defined uniqueness scope: + +| Field | Uniqueness Scope | +|---|---| +| `reportId` | Globally unique | +| `runId` | Globally unique | +| `testId` | Unique within the document | +| `executionId` | Globally unique | +| `attemptId` | Globally unique | +| `attachmentId` | Unique within the containing parent object | +| `shardId` | Unique within a logical run | + +Producers MUST NOT emit duplicate values for fields designated as globally unique. +Producers SHOULD NOT emit duplicate values for fields with narrower uniqueness scopes. + +#### 4.9.2. Stability + +Identity fields differ in whether their values persist across runs or are unique to a single emission: + +| Field | Stability | +|---|---| +| `reportId` | Unique per document; not stable across retransmissions | +| `runId` | Stable across all documents belonging to the same logical run | +| `testId` | Stable across runs for the same logical test case | +| `executionId` | Unique per execution; not stable across runs | +| `attemptId` | Unique per attempt; not stable across runs | +| `attachmentId` | Unique per attachment reference instance | +| `shardId` | Stable across retransmissions of the same shard | + +Producers that generate `testId` values SHOULD ensure stability by deriving identifiers from deterministic inputs such as the test name, suite path, and file path (see Appendix B). + --- ## 5. Top-Level Fields @@ -457,17 +507,35 @@ Consumers MUST NOT assume compatibility across different MAJOR versions. ### 5.3. `reportId` **Description:** -A unique identifier for this report instance. +A unique identifier for this CTRF document instance. + +`reportId` identifies the serialized artifact — the emitted file or payload — rather than the logical test run or any entity within it. See Section 4.9 for the full identity model. **Requirements:** `reportId` is OPTIONAL. -If present, it MUST be a valid UUID as defined in [RFC4122]. +If present, it MUST be a valid UUID as defined in [RFC4122]. +Each emitted document SHOULD receive a distinct `reportId`. Each emitted CTRF document represents a distinct report instance. If a new CTRF document is produced through post-processing, it SHOULD use a different `reportId` value than the source document when `reportId` is used. --- -### 5.4. `timestamp` +### 5.4. `runId` + +**Description:** +Identifies the logical test run to which this document belongs. + +A logical run MAY span multiple CTRF documents (for example, when tests are distributed across shards or parallel workers). All documents belonging to the same coordinated execution SHOULD share the same `runId`. + +**Requirements:** +`runId` is OPTIONAL. +If present, it MUST be a string. +Producers SHOULD ensure that `runId` is globally unique across distinct runs. +Consumers MAY use `runId` to merge or correlate documents from the same logical run. + +--- + +### 5.5. `timestamp` **Description:** The time at which the report was generated. @@ -478,7 +546,7 @@ If present, it MUST be a valid RFC 3339 / ISO 8601 date-time string. --- -### 5.5. `generatedBy` +### 5.6. `generatedBy` **Description:** Identifies the tool, framework, or system that produced the CTRF document. @@ -489,7 +557,7 @@ No specific format or structure is required. --- -### 5.6. `results` +### 5.7. `results` **Description:** Contains the results of a single test execution run. @@ -500,7 +568,7 @@ It MUST follow the structure defined in Section 6 (Results Object). --- -### 5.7. `insights` +### 5.8. `insights` **Description:** Contains aggregated or derived metrics computed across multiple test runs. @@ -511,7 +579,7 @@ If present, it MUST follow the structure described in Section 14 (Run-Level Insi --- -### 5.8. `baseline` +### 5.9. `baseline` **Description:** Identifies a previous report used as a comparison reference when computing insights. @@ -522,7 +590,7 @@ If present, it MUST follow the structure described in Section 17 (Baseline Objec --- -### 5.9. `extra` +### 5.10. `extra` **Description:** An extensibility object containing arbitrary metadata. @@ -786,19 +854,47 @@ Each field within the test object is described in the subsections below. ### 9.1. `id` +> **Deprecated.** Use `testId` (Section 9.2) instead. `id` will be removed in a future major version. + **Description:** A unique, stable identifier for the test case. **Requirements:** `id` is OPTIONAL. -If present, it MUST be a valid UUID as defined in [RFC4122]. -If a producer chooses to assign test identifiers, they SHOULD be deterministic for a given test case and stable across runs, machines, and environments. +If present, it MUST be a valid UUID as defined in [RFC4122]. +Producers SHOULD prefer `testId` for new implementations. +When both `id` and `testId` are present, consumers SHOULD use `testId`. -CTRF RECOMMENDS using UUID version 5 (name-based UUID) derived from stable test properties such as the test name, suite path, and file path. +### 9.2. `testId` -Consumers MUST treat the `id` value as opaque. +**Description:** +A stable identifier for the logical test case. -### 9.2. `name` +`testId` identifies the test case itself, independent of any particular run or execution. The same logical test case SHOULD produce the same `testId` across runs, machines, and environments. + +**Requirements:** +`testId` is OPTIONAL. +If present, it MUST be a string. +`testId` SHOULD be deterministic and stable across runs for the same test case. +It MAY be a UUID, a hash, or a canonical string. +Producers that generate UUID-based identifiers SHOULD use UUID version 5 derived from stable test properties (see Appendix B). + +Consumers MUST treat the `testId` value as opaque. + +### 9.3. `executionId` + +**Description:** +A unique identifier for the specific reported execution of a logical test case within a run. + +An execution represents a single reported outcome for a test case in this document. An execution MAY include multiple retry attempts represented in `retryAttempts`. + +**Requirements:** +`executionId` is OPTIONAL. +If present, it MUST be a string. +UUID is RECOMMENDED. +`executionId` SHOULD be globally unique and MUST NOT be reused across runs. + +### 9.4. `name` **Description:** The name or title of the test case. @@ -807,7 +903,7 @@ The name or title of the test case. `name` MUST be present. It MUST be a non-empty string. -### 9.3. `status` +### 9.5. `status` **Description:** The final outcome of the test case. @@ -816,7 +912,7 @@ The final outcome of the test case. `status` MUST be present. It MUST be one of: `passed`, `failed`, `skipped`, `pending`, or `other`. -### 9.4. `duration` +### 9.6. `duration` **Description:** The total execution time for the test case. @@ -825,7 +921,7 @@ The total execution time for the test case. `duration` MUST be present. It MUST be an integer representing milliseconds. -### 9.5. `start` +### 9.7. `start` **Description:** The timestamp when test execution began. @@ -834,7 +930,7 @@ The timestamp when test execution began. `start` is OPTIONAL. If present, it MUST be an integer representing milliseconds since the Unix epoch. -### 9.6. `stop` +### 9.8. `stop` **Description:** The timestamp when test execution ended. @@ -844,7 +940,7 @@ The timestamp when test execution ended. If present, it MUST be an integer representing milliseconds since the Unix epoch. If both `start` and `stop` are present, `stop` SHOULD be greater than or equal to `start`. -### 9.7. `suite` +### 9.9. `suite` **Description:** An ordered list of suite or grouping names to which this test belongs, ordered from the top-level suite to the immediate parent of the test. @@ -854,7 +950,7 @@ An ordered list of suite or grouping names to which this test belongs, ordered f If present, it MUST be an array of strings containing at least one entry. The array MUST be ordered from the top-level suite to the immediate parent suite of the test. -### 9.8. `message` +### 9.10. `message` **Description:** A user-visible error or failure message associated with the test result. @@ -863,7 +959,7 @@ A user-visible error or failure message associated with the test result. `message` is OPTIONAL. If present, it MUST be a string. -### 9.9. `trace` +### 9.11. `trace` **Description:** A stack trace or structured trace information describing the failure. @@ -872,7 +968,7 @@ A stack trace or structured trace information describing the failure. `trace` is OPTIONAL. If present, it MUST be a string. -### 9.10. `snippet` +### 9.12. `snippet` **Description:** A code snippet or relevant source excerpt associated with the failure. @@ -881,7 +977,7 @@ A code snippet or relevant source excerpt associated with the failure. `snippet` is OPTIONAL. If present, it MUST be a string. -### 9.11. `ai` +### 9.13. `ai` **Description:** AI-generated diagnostic data, commentary, or suggestions related to the test. @@ -890,7 +986,7 @@ AI-generated diagnostic data, commentary, or suggestions related to the test. `ai` is OPTIONAL. If present, it MUST be a string. -### 9.12. `line` +### 9.14. `line` **Description:** The line number associated with the test definition. @@ -899,7 +995,7 @@ The line number associated with the test definition. `line` is OPTIONAL. If present, it MUST be an integer. -### 9.13. `rawStatus` +### 9.15. `rawStatus` **Description:** The unmodified status reported by the source tool before CTRF normalization. @@ -908,7 +1004,7 @@ The unmodified status reported by the source tool before CTRF normalization. `rawStatus` is OPTIONAL. If present, it MUST be a string. -### 9.14. `tags` +### 9.16. `tags` **Description:** A list of simple, keyless classifications associated with the test. @@ -920,7 +1016,7 @@ Producers SHOULD use `tags` for simple, keyless categorization, such as `"smoke" --- -### 9.15. `labels` +### 9.17. `labels` **Description:** Structured key-value metadata associated with the test, such as ownership, priority, severity, external identifiers, or other named attributes. @@ -933,7 +1029,7 @@ Array values MAY contain more than one primitive type, but producers SHOULD use Consumers MUST treat label keys as opaque and MUST NOT require any particular keys to be present. Producers SHOULD use `labels` whenever the metadata consists of named attributes and associated values. -### 9.16. `type` +### 9.18. `type` **Description:** A classification or category for the test (for example, `"unit"`, `"integration"`, or `"e2e"`). @@ -942,7 +1038,7 @@ A classification or category for the test (for example, `"unit"`, `"integration" `type` is OPTIONAL. If present, it MUST be a string. -### 9.17. `filePath` +### 9.19. `filePath` **Description:** The path to the file that defines the test case. @@ -951,7 +1047,7 @@ The path to the file that defines the test case. `filePath` is OPTIONAL. If present, it MUST be a string. -### 9.18. `retries` +### 9.20. `retries` **Description:** The number of retries performed for this test case. @@ -961,7 +1057,7 @@ The number of retries performed for this test case. If present, it MUST be a non-negative integer. It SHOULD equal the count of entries in `retryAttempts`. -### 9.19. `retryAttempts` +### 9.21. `retryAttempts` **Description:** A list of retry attempts performed for this test case. @@ -971,7 +1067,7 @@ A list of retry attempts performed for this test case. If present, it MUST be an array. Each entry MUST follow the structure defined in Section 11 (Retry Attempt Object). -### 9.20. `flaky` +### 9.22. `flaky` **Description:** Indicates whether the test is considered flaky. @@ -982,7 +1078,7 @@ A test is considered flaky only if its final status is `passed` and it experienc `flaky` is OPTIONAL. If present, it MUST be a boolean. -### 9.21. `stdout` +### 9.23. `stdout` **Description:** Lines of standard output generated during test execution. @@ -991,7 +1087,7 @@ Lines of standard output generated during test execution. `stdout` is OPTIONAL. If present, it MUST be an array of strings. -### 9.22. `stderr` +### 9.24. `stderr` **Description:** Lines of standard error output generated during test execution. @@ -1000,7 +1096,7 @@ Lines of standard error output generated during test execution. `stderr` is OPTIONAL. If present, it MUST be an array of strings. -### 9.23. `threadId` +### 9.25. `threadId` **Description:** Identifies the thread or worker on which the test executed. @@ -1009,7 +1105,7 @@ Identifies the thread or worker on which the test executed. `threadId` is OPTIONAL. If present, it MUST be a string. -### 9.24. `browser` +### 9.26. `browser` **Description:** Identifies the browser used during test execution (for browser-based tests). @@ -1018,7 +1114,7 @@ Identifies the browser used during test execution (for browser-based tests). `browser` is OPTIONAL. If present, it MUST be a string. -### 9.25. `device` +### 9.27. `device` **Description:** Identifies the device or device profile used during test execution. @@ -1027,7 +1123,7 @@ Identifies the device or device profile used during test execution. `device` is OPTIONAL. If present, it MUST be a string. -### 9.26. `screenshot` +### 9.28. `screenshot` **Description:** A single base64-encoded screenshot captured during execution of the test case. @@ -1044,7 +1140,7 @@ Producers MUST use the `attachments` array if providing additional screenshots o Screenshots are base64-encoded and may significantly increase document size. Producers SHOULD prefer file-based attachments for large images. -### 9.27. `attachments` +### 9.29. `attachments` **Description:** A list of additional artifacts associated with the test case (screenshots, logs, videos, etc.). @@ -1054,7 +1150,7 @@ A list of additional artifacts associated with the test case (screenshots, logs, If present, it MUST be an array. Each entry MUST follow the structure defined in Section 12 (Attachment Object). -### 9.28. `parameters` +### 9.30. `parameters` **Description:** A set of test parameters, input values, or contextual data relevant to the execution. @@ -1064,7 +1160,7 @@ A set of test parameters, input values, or contextual data relevant to the execu If present, it MUST be an object. It MAY contain arbitrary fields. -### 9.29. `steps` +### 9.31. `steps` **Description:** A list of test steps or sub-operations performed during the test. @@ -1074,7 +1170,7 @@ A list of test steps or sub-operations performed during the test. If present, it MUST be an array. Each element MUST follow the structure defined in Section 13 (Step Object). -### 9.30. `insights` +### 9.32. `insights` **Description:** Derived or aggregated insights specific to this test case. @@ -1083,7 +1179,7 @@ Derived or aggregated insights specific to this test case. `insights` is OPTIONAL. If present, it MUST follow the structure defined in Section 15 (Test-Level Insights Object). -### 9.31. `extra` +### 9.33. `extra` **Description:** An extensibility object containing arbitrary metadata. @@ -1242,7 +1338,20 @@ A label indicating the environment under which tests were executed (e.g., local, `testEnvironment` is OPTIONAL. If present, it MUST be a string. -### 10.16. `healthy` +### 10.16. `shardId` + +**Description:** +A label identifying the shard or partition that produced this document within a logical run. + +When a test run is distributed across multiple shards, workers, or machines, `shardId` identifies which partition produced this specific document. + +**Requirements:** +`shardId` is OPTIONAL. +If present, it MUST be a string. +`shardId` values SHOULD be unique within documents sharing the same `runId`. +The value MAY be human-readable (e.g. `"shard-1"`, `"worker-3"`). + +### 10.17. `healthy` **Description:** Indicates whether the overall system or environment was considered healthy during the test run. @@ -1251,7 +1360,7 @@ Indicates whether the overall system or environment was considered healthy durin `healthy` is OPTIONAL. If present, it MUST be a boolean. -### 10.17. `extra` +### 10.18. `extra` **Description:** An extensibility object containing arbitrary metadata. @@ -1286,7 +1395,18 @@ The attempt number for this execution of the test case. It MUST be an integer greater than or equal to 1. Attempt number 1 represents the first execution; higher numbers represent retries. -### 11.2. `status` +### 11.2. `attemptId` + +**Description:** +A unique identifier for this individual retry attempt. + +**Requirements:** +`attemptId` is OPTIONAL. +If present, it MUST be a string. +UUID is RECOMMENDED. +`attemptId` SHOULD be globally unique. + +### 11.3. `status` **Description:** The outcome of this retry attempt. @@ -1295,7 +1415,7 @@ The outcome of this retry attempt. `status` MUST be present. It MUST be one of the following values: `passed`, `failed`, `skipped`, `pending`, or `other`. -### 11.3. `duration` +### 11.4. `duration` **Description:** The execution time of this retry attempt, in milliseconds. @@ -1304,7 +1424,7 @@ The execution time of this retry attempt, in milliseconds. `duration` is OPTIONAL. If present, it MUST be an integer. -### 11.4. `message` +### 11.5. `message` **Description:** A human-readable message associated with the attempt outcome, commonly used for error or failure messages. @@ -1313,7 +1433,7 @@ A human-readable message associated with the attempt outcome, commonly used for `message` is OPTIONAL. If present, it MUST be a string. -### 11.5. `trace` +### 11.6. `trace` **Description:** A stack trace or diagnostic trace captured during the attempt. @@ -1322,7 +1442,7 @@ A stack trace or diagnostic trace captured during the attempt. `trace` is OPTIONAL. If present, it MUST be a string. -### 11.6. `line` +### 11.7. `line` **Description:** The line number in the source file where the failure occurred, if applicable. @@ -1331,7 +1451,7 @@ The line number in the source file where the failure occurred, if applicable. `line` is OPTIONAL. If present, it MUST be a number. -### 11.7. `snippet` +### 11.8. `snippet` **Description:** A source code snippet or excerpt relevant to the attempt. @@ -1340,7 +1460,7 @@ A source code snippet or excerpt relevant to the attempt. `snippet` is OPTIONAL. If present, it MUST be a string. -### 11.8. `stdout` +### 11.9. `stdout` **Description:** A list of standard output lines captured during the attempt. @@ -1349,7 +1469,7 @@ A list of standard output lines captured during the attempt. `stdout` is OPTIONAL. If present, it MUST be an array of strings. -### 11.9. `stderr` +### 11.10. `stderr` **Description:** A list of standard error lines captured during the attempt. @@ -1358,7 +1478,7 @@ A list of standard error lines captured during the attempt. `stderr` is OPTIONAL. If present, it MUST be an array of strings. -### 11.10. `start` +### 11.11. `start` **Description:** The timestamp indicating when the attempt began, in milliseconds since the Unix epoch. @@ -1367,7 +1487,7 @@ The timestamp indicating when the attempt began, in milliseconds since the Unix `start` is OPTIONAL. If present, it MUST be an integer. -### 11.11. `stop` +### 11.12. `stop` **Description:** The timestamp indicating when the attempt ended, in milliseconds since the Unix epoch. @@ -1377,7 +1497,7 @@ The timestamp indicating when the attempt ended, in milliseconds since the Unix If present, it MUST be an integer. If both `start` and `stop` are present, `stop` SHOULD be greater than or equal to `start`. -### 11.12. `attachments` +### 11.13. `attachments` **Description:** An array of attachment objects associated with this retry attempt. @@ -1387,7 +1507,7 @@ An array of attachment objects associated with this retry attempt. If present, it MUST be an array. Each element MUST follow the structure defined in Section 12 (Attachment Object). -### 11.13. `extra` +### 11.14. `extra` **Description:** An extensibility object containing arbitrary metadata. @@ -1441,7 +1561,18 @@ The specification does not impose any particular URI or path semantics. Consumers MUST treat attachment paths as opaque references and MUST NOT assume local filesystem access unless explicitly documented by the producer. -### 12.4. `extra` +### 12.4. `attachmentId` + +**Description:** +A unique identifier for this attachment reference instance. + +**Requirements:** +`attachmentId` is OPTIONAL. +If present, it MUST be a string. +UUID is RECOMMENDED. +`attachmentId` SHOULD be unique within the containing parent object. + +### 12.5. `extra` **Description:** An extensibility object containing arbitrary metadata. @@ -1823,6 +1954,7 @@ Producers: - MUST follow rules for retry attempts - MUST NOT introduce fields outside `extra` objects - MUST NOT emit invalid enum values (status, etc.) +- MUST NOT emit duplicate values for identity fields designated as globally unique (see Section 4.9.1) - MAY include `insights` if historical or aggregate data is available Producers SHOULD: @@ -1832,6 +1964,8 @@ Producers SHOULD: - provide diagnostics when available - treat emitted CTRF documents as immutable artifacts - emit a new CTRF document when performing post-processing after initial report generation and, when `reportId` is used, assign a different `reportId` value rather than modifying an emitted CTRF document +- assign identity fields (`testId`, `executionId`, `attemptId`, `attachmentId`) when traceability is meaningful for their workflow +- prefer `testId` over the deprecated `id` field for new implementations --- @@ -1848,6 +1982,9 @@ Consumers: - SHOULD compute insights when sufficient historical data is available - SHOULD assume that CTRF documents are immutable once emitted - MUST NOT rely on in-place modification of existing reports +- SHOULD use `testId` in preference to `id` when both are present +- MAY use `runId` to merge or correlate documents from the same logical run +- MUST NOT assume that identity fields are present; all identity fields are OPTIONAL --- @@ -2561,11 +2698,11 @@ to this specification. ## Appendix B - Test Identifier Generation (Informative) -Although the `id` field of a test object is OPTIONAL, producers that assign test identifiers SHOULD ensure that they are stable and deterministic across runs. +Although `testId` (and the deprecated `id` field) of a test object is OPTIONAL, producers that assign test identifiers SHOULD ensure that they are stable and deterministic across runs. Producers SHOULD normalize input values (e.g., trimming whitespace and using consistent path separators) before generating deterministic identifiers. -CTRF RECOMMENDS the following approach: +CTRF RECOMMENDS the following approach for `testId`: - Use UUID version 5 (name-based UUID, SHA-1). - Construct a stable identifier string using test properties such as: @@ -2574,7 +2711,7 @@ CTRF RECOMMENDS the following approach: - file path - Combine these properties into a canonical string and generate a UUID v5 using a fixed namespace UUID. -This approach ensures that the same test retains the same identifier across runs, enabling consumers to analyze historical trends, flakiness, and stability. +This approach ensures that the same test retains the same `testId` across runs, enabling consumers to analyze historical trends, flakiness, and stability. When generating deterministic test identifiers, producers SHOULD: @@ -2584,6 +2721,8 @@ When generating deterministic test identifiers, producers SHOULD: The namespace UUID is intentionally not fixed by this specification. +Note: `testId` accepts any string format (UUID, hash, or canonical string). UUID v5 is recommended but not required. Producers MAY use other deterministic schemes provided the output is stable for a given test case. + --- ## Appendix C - Metrics Reference (Informative) @@ -3020,6 +3159,7 @@ It includes: "reportFormat": "CTRF", "specVersion": "0.0.0", "reportId": "9d2c6a10-3f7a-4e22-9a8f-1a2b3c4d5e6f", + "runId": "run-20251124-e2e-staging", "timestamp": "2025-11-24T12:00:00Z", "generatedBy": "example-ci", "extra": { @@ -3059,6 +3199,8 @@ It includes: "tests": [ { "id": "c6c9f8c0-8b5e-5f7a-9e2d-3c91f8a7d7c1", + "testId": "c6c9f8c0-8b5e-5f7a-9e2d-3c91f8a7d7c1", + "executionId": "a1b2c3d4-1111-4000-a000-000000000001", "name": "user can log in", "suite": ["auth", "login"], "filePath": "tests/auth/login.test.js", @@ -3075,6 +3217,7 @@ It includes: "retryAttempts": [ { "attempt": 1, + "attemptId": "f1e2d3c4-2222-4000-b000-000000000001", "status": "failed", "duration": 1500, "message": "Timeout waiting for response", @@ -3083,6 +3226,7 @@ It includes: }, { "attempt": 2, + "attemptId": "f1e2d3c4-2222-4000-b000-000000000002", "status": "passed", "duration": 1500 } @@ -3102,6 +3246,8 @@ It includes: }, { "id": "1e2d4a40-92a4-5d3f-9e3d-3a77fbc2c123", + "testId": "1e2d4a40-92a4-5d3f-9e3d-3a77fbc2c123", + "executionId": "a1b2c3d4-1111-4000-a000-000000000002", "name": "invalid login is rejected", "suite": ["auth", "login"], "filePath": "tests/auth/login.test.js", @@ -3114,6 +3260,7 @@ It includes: "screenshot": "iVBORw0KGgoAAAANSUhEUgAAAAUA", "attachments": [ { + "attachmentId": "b3c4d5e6-3333-4000-c000-000000000001", "name": "browser-log.txt", "contentType": "text/plain", "path": "artifacts/browser-log.txt" @@ -3153,6 +3300,7 @@ It includes: "osRelease": "ubuntu", "osVersion": "22.04", "testEnvironment": "staging", + "shardId": "shard-1", "healthy": false, "extra": { "myorg.ci": { From 0feadacda5d4884d257e51df5d3d734cbb7c2d1f Mon Sep 17 00:00:00 2001 From: Matthew Thomas <25988124+Ma11hewThomas@users.noreply.github.com> Date: Sat, 27 Jun 2026 13:47:00 +0100 Subject: [PATCH 2/8] Add identity examples and validation tests --- examples/comprehensive.json | 14 ++++ tests/normative/type-validation.test.json | 80 +++++++++++++++++++++++ 2 files changed, 94 insertions(+) diff --git a/examples/comprehensive.json b/examples/comprehensive.json index a1a0e6e..53772b6 100644 --- a/examples/comprehensive.json +++ b/examples/comprehensive.json @@ -2,6 +2,7 @@ "reportFormat": "CTRF", "specVersion": "1.0.0", "reportId": "9d2c6a10-3f7a-4e22-9a8f-1a2b3c4d5e6f", + "runId": "run-2025-11-24-pr-123", "timestamp": "2025-11-24T12:00:00Z", "generatedBy": "example-ci", "extra": { @@ -36,6 +37,8 @@ "tests": [ { "id": "550e8400-e29b-41d4-a716-446655440000", + "testId": "auth-login-user-can-login", + "executionId": "2cf7e0b4-ea0c-4ef4-a18a-b9d6f6f0e100", "name": "user can login", "status": "passed", "duration": 3500, @@ -71,6 +74,8 @@ }, { "id": "550e8400-e29b-41d4-a716-446655440001", + "testId": "checkout-user-can-checkout", + "executionId": "2cf7e0b4-ea0c-4ef4-a18a-b9d6f6f0e101", "name": "user can checkout", "status": "passed", "duration": 5200, @@ -93,6 +98,7 @@ "retryAttempts": [ { "attempt": 1, + "attemptId": "48c3f90b-95a9-4c13-a42f-0888bb0b1001", "status": "failed", "duration": 5100, "start": 1609459203500, @@ -101,6 +107,7 @@ "trace": "Error: Payment gateway timeout\n at processPayment (payment.js:23:10)", "attachments": [ { + "attachmentId": "72d531af-9d77-4f3b-bf67-0d8ec26c2001", "name": "attempt-1-screenshot.png", "contentType": "image/png", "path": "/artifacts/retry-1.png" @@ -109,6 +116,7 @@ }, { "attempt": 2, + "attemptId": "48c3f90b-95a9-4c13-a42f-0888bb0b1002", "status": "passed", "duration": 5200, "start": 1609459208700, @@ -131,6 +139,8 @@ }, { "id": "550e8400-e29b-41d4-a716-446655440002", + "testId": "admin-dashboard-loads", + "executionId": "2cf7e0b4-ea0c-4ef4-a18a-b9d6f6f0e102", "name": "admin dashboard loads", "status": "failed", "duration": 8000, @@ -163,16 +173,19 @@ ], "attachments": [ { + "attachmentId": "72d531af-9d77-4f3b-bf67-0d8ec26c2002", "name": "failure-screenshot.png", "contentType": "image/png", "path": "/artifacts/screenshots/admin-dashboard-fail.png" }, { + "attachmentId": "72d531af-9d77-4f3b-bf67-0d8ec26c2003", "name": "trace.zip", "contentType": "application/zip", "path": "/artifacts/traces/admin-dashboard.trace.zip" }, { + "attachmentId": "72d531af-9d77-4f3b-bf67-0d8ec26c2004", "name": "video.webm", "contentType": "video/webm", "path": "/artifacts/videos/admin-dashboard.webm" @@ -214,6 +227,7 @@ "repositoryUrl": "https://github.com/example/example-app", "commit": "a1b2c3d4e5f6", "branchName": "feature/new-dashboard", + "shardId": "shard-1", "osPlatform": "linux", "osRelease": "5.10.0", "osVersion": "Ubuntu 20.04", diff --git a/tests/normative/type-validation.test.json b/tests/normative/type-validation.test.json index 7d8e44b..f1c13c6 100644 --- a/tests/normative/type-validation.test.json +++ b/tests/normative/type-validation.test.json @@ -42,6 +42,20 @@ } } }, + { + "description": "Document with runId as number instead of string", + "valid": false, + "data": { + "reportFormat": "CTRF", + "specVersion": "1.0.0", + "runId": 12345, + "results": { + "tool": { "name": "runner" }, + "summary": { "tests": 1, "passed": 1, "failed": 0, "pending": 0, "skipped": 0, "other": 0, "start": 1609459200000, "stop": 1609459201000 }, + "tests": [{ "name": "test", "status": "passed", "duration": 100 }] + } + } + }, { "description": "Document with timestamp as number instead of string", "valid": false, @@ -327,6 +341,32 @@ } } }, + { + "description": "Test with testId as number instead of string", + "valid": false, + "data": { + "reportFormat": "CTRF", + "specVersion": "1.0.0", + "results": { + "tool": { "name": "runner" }, + "summary": { "tests": 1, "passed": 1, "failed": 0, "pending": 0, "skipped": 0, "other": 0, "start": 1609459200000, "stop": 1609459201000 }, + "tests": [{ "name": "test", "testId": 12345, "status": "passed", "duration": 100 }] + } + } + }, + { + "description": "Test with executionId as number instead of string", + "valid": false, + "data": { + "reportFormat": "CTRF", + "specVersion": "1.0.0", + "results": { + "tool": { "name": "runner" }, + "summary": { "tests": 1, "passed": 1, "failed": 0, "pending": 0, "skipped": 0, "other": 0, "start": 1609459200000, "stop": 1609459201000 }, + "tests": [{ "name": "test", "executionId": 12345, "status": "passed", "duration": 100 }] + } + } + }, { "description": "Test with status as number instead of string", "valid": false, @@ -691,6 +731,19 @@ } } }, + { + "description": "RetryAttempt with attemptId as number instead of string", + "valid": false, + "data": { + "reportFormat": "CTRF", + "specVersion": "1.0.0", + "results": { + "tool": { "name": "runner" }, + "summary": { "tests": 1, "passed": 1, "failed": 0, "pending": 0, "skipped": 0, "other": 0, "start": 1609459200000, "stop": 1609459201000 }, + "tests": [{ "name": "test", "status": "passed", "duration": 100, "retryAttempts": [{ "attempt": 1, "attemptId": 12345, "status": "failed" }] }] + } + } + }, { "description": "RetryAttempt with status as number instead of string", "valid": false, @@ -795,6 +848,19 @@ } } }, + { + "description": "Attachment with attachmentId as number instead of string", + "valid": false, + "data": { + "reportFormat": "CTRF", + "specVersion": "1.0.0", + "results": { + "tool": { "name": "runner" }, + "summary": { "tests": 1, "passed": 1, "failed": 0, "pending": 0, "skipped": 0, "other": 0, "start": 1609459200000, "stop": 1609459201000 }, + "tests": [{ "name": "test", "status": "passed", "duration": 100, "attachments": [{ "attachmentId": 12345, "name": "file", "contentType": "text/plain", "path": "/path" }] }] + } + } + }, { "description": "Test with parameters as array instead of object", "valid": false, @@ -888,6 +954,20 @@ } } }, + { + "description": "Environment with shardId as number instead of string", + "valid": false, + "data": { + "reportFormat": "CTRF", + "specVersion": "1.0.0", + "results": { + "tool": { "name": "runner" }, + "summary": { "tests": 1, "passed": 1, "failed": 0, "pending": 0, "skipped": 0, "other": 0, "start": 1609459200000, "stop": 1609459201000 }, + "tests": [{ "name": "test", "status": "passed", "duration": 100 }], + "environment": { "shardId": 1 } + } + } + }, { "description": "Environment with healthy as string instead of boolean", "valid": false, From 6fdd22c51de3be34f8f905fc268abe98301b8668 Mon Sep 17 00:00:00 2001 From: Matthew Thomas <25988124+Ma11hewThomas@users.noreply.github.com> Date: Sun, 26 Jul 2026 17:25:56 +0100 Subject: [PATCH 3/8] Prepare identity model for PR --- CHANGELOG.md | 1 + spec/ctrf.md | 58 ++++++++++++++++++++++++++++++++++++++-------------- 2 files changed, 44 insertions(+), 15 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 21098b5..48f6678 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Changed +- Added an optional identity and lineage model for CTRF documents, logical runs, test cases, executions, attempts, attachments, and shards. - Clarified namespace guidance for `extra` extension keys and examples ([#56](https://github.com/ctrf-io/ctrf/pull/56)). - Clarified immutability guidance for emitted CTRF report artifacts ([#55](https://github.com/ctrf-io/ctrf/pull/55)). - Clarified `tags` as simple keyless classifications and `labels` as structured key-value test metadata ([#54](https://github.com/ctrf-io/ctrf/pull/54)). diff --git a/spec/ctrf.md b/spec/ctrf.md index 5d57573..3d2d8c3 100644 --- a/spec/ctrf.md +++ b/spec/ctrf.md @@ -418,7 +418,7 @@ CTRF defines a layered identity model that enables correlation of test data acro Each layer addresses a distinct identification concern: -1. **Document**: `reportId` identifies the emitted CTRF document as a serialized artifact. Each CTRF file SHOULD have exactly one `reportId`. +1. **Document**: `reportId` identifies the emitted CTRF document as a serialized artifact. When `reportId` is used, each CTRF document SHOULD have exactly one `reportId`. 2. **Run**: `runId` identifies the logical test run. Multiple CTRF documents MAY share the same `runId` when they represent shards or partitions of a single coordinated execution. 3. **Test case**: `testId` identifies the logical test case. It SHOULD be deterministic and stable across runs, enabling cross-run analysis, trending, and flake detection. 4. **Execution**: `executionId` identifies a concrete execution of a test case within a run. An execution MAY include multiple retry attempts represented in `retryAttempts`. @@ -430,20 +430,20 @@ All identity fields are OPTIONAL. Producers SHOULD assign them when the correspo #### 4.9.1. Uniqueness -Each identity field has a defined uniqueness scope: +Each identity field has an intended uniqueness scope: | Field | Uniqueness Scope | |---|---| -| `reportId` | Globally unique | -| `runId` | Globally unique | +| `reportId` | Unique per document | +| `runId` | Unique per logical run | | `testId` | Unique within the document | -| `executionId` | Globally unique | -| `attemptId` | Globally unique | +| `executionId` | Unique per execution | +| `attemptId` | Unique per attempt | | `attachmentId` | Unique within the containing parent object | | `shardId` | Unique within a logical run | -Producers MUST NOT emit duplicate values for fields designated as globally unique. -Producers SHOULD NOT emit duplicate values for fields with narrower uniqueness scopes. +Producers SHOULD NOT knowingly reuse identity values outside their intended scope. +Producers SHOULD NOT emit duplicate identity values within a single CTRF document when those duplicates would make correlation ambiguous. #### 4.9.2. Stability @@ -514,7 +514,7 @@ A unique identifier for this CTRF document instance. **Requirements:** `reportId` is OPTIONAL. If present, it MUST be a valid UUID as defined in [RFC4122]. -Each emitted document SHOULD receive a distinct `reportId`. +When `reportId` is used, each emitted document SHOULD receive a distinct `reportId`. Each emitted CTRF document represents a distinct report instance. If a new CTRF document is produced through post-processing, it SHOULD use a different `reportId` value than the source document when `reportId` is used. @@ -530,7 +530,7 @@ A logical run MAY span multiple CTRF documents (for example, when tests are dist **Requirements:** `runId` is OPTIONAL. If present, it MUST be a string. -Producers SHOULD ensure that `runId` is globally unique across distinct runs. +Producers SHOULD ensure that `runId` is unique across distinct logical runs. Consumers MAY use `runId` to merge or correlate documents from the same logical run. --- @@ -892,7 +892,7 @@ An execution represents a single reported outcome for a test case in this docume `executionId` is OPTIONAL. If present, it MUST be a string. UUID is RECOMMENDED. -`executionId` SHOULD be globally unique and MUST NOT be reused across runs. +`executionId` SHOULD be unique across executions and SHOULD NOT be reused across runs. ### 9.4. `name` @@ -1404,7 +1404,7 @@ A unique identifier for this individual retry attempt. `attemptId` is OPTIONAL. If present, it MUST be a string. UUID is RECOMMENDED. -`attemptId` SHOULD be globally unique. +`attemptId` SHOULD be unique across attempts. ### 11.3. `status` @@ -1954,7 +1954,6 @@ Producers: - MUST follow rules for retry attempts - MUST NOT introduce fields outside `extra` objects - MUST NOT emit invalid enum values (status, etc.) -- MUST NOT emit duplicate values for identity fields designated as globally unique (see Section 4.9.1) - MAY include `insights` if historical or aggregate data is available Producers SHOULD: @@ -1965,6 +1964,7 @@ Producers SHOULD: - treat emitted CTRF documents as immutable artifacts - emit a new CTRF document when performing post-processing after initial report generation and, when `reportId` is used, assign a different `reportId` value rather than modifying an emitted CTRF document - assign identity fields (`testId`, `executionId`, `attemptId`, `attachmentId`) when traceability is meaningful for their workflow +- avoid duplicate identity values when duplicates would make correlation ambiguous - prefer `testId` over the deprecated `id` field for new implementations --- @@ -2037,10 +2037,14 @@ to this specification. "pattern": "^[0-9]+\\.[0-9]+\\.[0-9]+$" }, "reportId": { - "description": "Unique identifier for this report instance (UUID)", + "description": "Unique identifier for this CTRF document instance (UUID). Identifies the serialized artifact.", "type": "string", "format": "uuid" }, + "runId": { + "description": "Identifier for the logical test run. Shared across shards/documents belonging to the same run.", + "type": "string" + }, "timestamp": { "description": "Report generation time (RFC 3339 / ISO 8601)", "type": "string", @@ -2154,10 +2158,18 @@ to this specification. "required": [ "name", "status", "duration" ], "properties": { "id": { - "description": "Unique, stable identifier for the test case (UUID)", + "description": "Deprecated: use testId instead. Unique, stable identifier for the test case (UUID)", "type": "string", "format": "uuid" }, + "testId": { + "description": "Stable identifier for the logical test case. Should be deterministic and stable across runs.", + "type": "string" + }, + "executionId": { + "description": "Unique identifier for this specific execution of the test case within a run. UUID recommended.", + "type": "string" + }, "name": { "description": "Name or title of the test case", "type": "string", @@ -2277,6 +2289,10 @@ to this specification. "type": "integer", "minimum": 1 }, + "attemptId": { + "description": "Unique identifier for this individual retry attempt. UUID recommended.", + "type": "string" + }, "status": { "description": "Outcome of this attempt", "enum": [ @@ -2337,6 +2353,10 @@ to this specification. "type": "object", "required": [ "name", "contentType", "path" ], "properties": { + "attachmentId": { + "description": "Unique identifier for this attachment reference instance. UUID recommended.", + "type": "string" + }, "name": { "description": "Display name of the attachment", "type": "string" @@ -2407,6 +2427,10 @@ to this specification. "type": "object", "required": [ "name", "contentType", "path" ], "properties": { + "attachmentId": { + "description": "Unique identifier for this attachment reference instance. UUID recommended.", + "type": "string" + }, "name": { "description": "Display name of the attachment", "type": "string" @@ -2568,6 +2592,10 @@ to this specification. "description": "Logical test environment (e.g., 'staging', 'production')", "type": "string" }, + "shardId": { + "description": "Shard or partition label identifying which part of a distributed run produced this document", + "type": "string" + }, "healthy": { "description": "Indicates if the run is considered healthy", "type": "boolean" From c7afecb60a130ecdf193358146c6f53b572233ed Mon Sep 17 00:00:00 2001 From: Matthew Thomas <25988124+Ma11hewThomas@users.noreply.github.com> Date: Sun, 26 Jul 2026 17:38:07 +0100 Subject: [PATCH 4/8] Link identity model changelog entry --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 48f6678..58625dd 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,7 +9,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Changed -- Added an optional identity and lineage model for CTRF documents, logical runs, test cases, executions, attempts, attachments, and shards. +- Added an optional identity and lineage model for CTRF documents, logical runs, test cases, executions, attempts, attachments, and shards ([#57](https://github.com/ctrf-io/ctrf/pull/57)). - Clarified namespace guidance for `extra` extension keys and examples ([#56](https://github.com/ctrf-io/ctrf/pull/56)). - Clarified immutability guidance for emitted CTRF report artifacts ([#55](https://github.com/ctrf-io/ctrf/pull/55)). - Clarified `tags` as simple keyless classifications and `labels` as structured key-value test metadata ([#54](https://github.com/ctrf-io/ctrf/pull/54)). From 6f065c089e5116624e50598d9dbb51fe8d4b4e7e Mon Sep 17 00:00:00 2001 From: Matthew Thomas <25988124+Ma11hewThomas@users.noreply.github.com> Date: Sun, 26 Jul 2026 17:56:50 +0100 Subject: [PATCH 5/8] Refine testId generation guidance --- schema/ctrf.schema.json | 16 +++++++------- spec/ctrf.md | 47 +++++++++++++++++++++++------------------ 2 files changed, 34 insertions(+), 29 deletions(-) diff --git a/schema/ctrf.schema.json b/schema/ctrf.schema.json index 7387b42..b900868 100644 --- a/schema/ctrf.schema.json +++ b/schema/ctrf.schema.json @@ -15,12 +15,12 @@ "pattern": "^[0-9]+\\.[0-9]+\\.[0-9]+$" }, "reportId": { - "description": "Unique identifier for this CTRF document instance (UUID). Identifies the serialized artifact.", + "description": "Unique identifier for this CTRF document instance (UUID). Identifies the serialized artifact", "type": "string", "format": "uuid" }, "runId": { - "description": "Identifier for the logical test run. Shared across shards/documents belonging to the same run.", + "description": "Identifier for the logical test run. Shared across shards/documents belonging to the same run", "type": "string" }, "timestamp": { @@ -136,16 +136,16 @@ "required": [ "name", "status", "duration" ], "properties": { "id": { - "description": "Deprecated: use testId instead. Unique, stable identifier for the test case (UUID)", + "description": "Legacy identifier for the test case (UUID). New producers should prefer testId", "type": "string", "format": "uuid" }, "testId": { - "description": "Stable identifier for the logical test case. Should be deterministic and stable across runs.", + "description": "Stable identifier for the logical test case. Should be derived from stable test attributes when generated by a producer", "type": "string" }, "executionId": { - "description": "Unique identifier for this specific execution of the test case within a run. UUID recommended.", + "description": "Unique identifier for this specific execution of the test case within a run. UUID recommended", "type": "string" }, "name": { @@ -268,7 +268,7 @@ "minimum": 1 }, "attemptId": { - "description": "Unique identifier for this individual retry attempt. UUID recommended.", + "description": "Unique identifier for this individual retry attempt. UUID recommended", "type": "string" }, "status": { @@ -332,7 +332,7 @@ "required": [ "name", "contentType", "path" ], "properties": { "attachmentId": { - "description": "Unique identifier for this attachment reference instance. UUID recommended.", + "description": "Unique identifier for this attachment reference instance. UUID recommended", "type": "string" }, "name": { @@ -406,7 +406,7 @@ "required": [ "name", "contentType", "path" ], "properties": { "attachmentId": { - "description": "Unique identifier for this attachment reference instance. UUID recommended.", + "description": "Unique identifier for this attachment reference instance. UUID recommended", "type": "string" }, "name": { diff --git a/spec/ctrf.md b/spec/ctrf.md index 3d2d8c3..db57156 100644 --- a/spec/ctrf.md +++ b/spec/ctrf.md @@ -854,7 +854,7 @@ Each field within the test object is described in the subsections below. ### 9.1. `id` -> **Deprecated.** Use `testId` (Section 9.2) instead. `id` will be removed in a future major version. +> **Legacy.** `id` is retained for backward compatibility. New producers SHOULD prefer `testId` (Section 9.2). **Description:** A unique, stable identifier for the test case. @@ -876,8 +876,8 @@ A stable identifier for the logical test case. `testId` is OPTIONAL. If present, it MUST be a string. `testId` SHOULD be deterministic and stable across runs for the same test case. -It MAY be a UUID, a hash, or a canonical string. -Producers that generate UUID-based identifiers SHOULD use UUID version 5 derived from stable test properties (see Appendix B). +When generated by a producer, `testId` SHOULD be derived from stable test attributes such as file path, suite path, and test name. +A producer MAY use a canonical string directly, or MAY encode that canonical string as a deterministic identifier such as a UUID version 5 or hash. Consumers MUST treat the `testId` value as opaque. @@ -2037,12 +2037,12 @@ to this specification. "pattern": "^[0-9]+\\.[0-9]+\\.[0-9]+$" }, "reportId": { - "description": "Unique identifier for this CTRF document instance (UUID). Identifies the serialized artifact.", + "description": "Unique identifier for this CTRF document instance (UUID). Identifies the serialized artifact", "type": "string", "format": "uuid" }, "runId": { - "description": "Identifier for the logical test run. Shared across shards/documents belonging to the same run.", + "description": "Identifier for the logical test run. Shared across shards/documents belonging to the same run", "type": "string" }, "timestamp": { @@ -2158,16 +2158,16 @@ to this specification. "required": [ "name", "status", "duration" ], "properties": { "id": { - "description": "Deprecated: use testId instead. Unique, stable identifier for the test case (UUID)", + "description": "Legacy identifier for the test case (UUID). New producers should prefer testId", "type": "string", "format": "uuid" }, "testId": { - "description": "Stable identifier for the logical test case. Should be deterministic and stable across runs.", + "description": "Stable identifier for the logical test case. Should be derived from stable test attributes when generated by a producer", "type": "string" }, "executionId": { - "description": "Unique identifier for this specific execution of the test case within a run. UUID recommended.", + "description": "Unique identifier for this specific execution of the test case within a run. UUID recommended", "type": "string" }, "name": { @@ -2290,7 +2290,7 @@ to this specification. "minimum": 1 }, "attemptId": { - "description": "Unique identifier for this individual retry attempt. UUID recommended.", + "description": "Unique identifier for this individual retry attempt. UUID recommended", "type": "string" }, "status": { @@ -2354,7 +2354,7 @@ to this specification. "required": [ "name", "contentType", "path" ], "properties": { "attachmentId": { - "description": "Unique identifier for this attachment reference instance. UUID recommended.", + "description": "Unique identifier for this attachment reference instance. UUID recommended", "type": "string" }, "name": { @@ -2428,7 +2428,7 @@ to this specification. "required": [ "name", "contentType", "path" ], "properties": { "attachmentId": { - "description": "Unique identifier for this attachment reference instance. UUID recommended.", + "description": "Unique identifier for this attachment reference instance. UUID recommended", "type": "string" }, "name": { @@ -2726,18 +2726,23 @@ to this specification. ## Appendix B - Test Identifier Generation (Informative) -Although `testId` (and the deprecated `id` field) of a test object is OPTIONAL, producers that assign test identifiers SHOULD ensure that they are stable and deterministic across runs. +Although `testId` (and the legacy `id` field) of a test object is OPTIONAL, producers that assign test identifiers SHOULD ensure that they are stable and deterministic across runs. Producers SHOULD normalize input values (e.g., trimming whitespace and using consistent path separators) before generating deterministic identifiers. -CTRF RECOMMENDS the following approach for `testId`: +CTRF RECOMMENDS deriving producer-generated `testId` values from stable test attributes such as: -- Use UUID version 5 (name-based UUID, SHA-1). -- Construct a stable identifier string using test properties such as: - - test name - - suite path - - file path -- Combine these properties into a canonical string and generate a UUID v5 using a fixed namespace UUID. +- file path +- suite path +- test name + +Producers MAY use the resulting canonical string directly. For example: + +```text +tests/auth/login.test.ts::auth > login::user can log in +``` + +Producers MAY also encode the canonical string as a deterministic identifier such as a UUID version 5 or hash. This approach ensures that the same test retains the same `testId` across runs, enabling consumers to analyze historical trends, flakiness, and stability. @@ -2747,9 +2752,9 @@ When generating deterministic test identifiers, producers SHOULD: - Use a fixed delimiter between components. - Normalize string inputs (e.g., case, path separators) consistently. -The namespace UUID is intentionally not fixed by this specification. +When UUID version 5 is used, the namespace UUID is intentionally not fixed by this specification. -Note: `testId` accepts any string format (UUID, hash, or canonical string). UUID v5 is recommended but not required. Producers MAY use other deterministic schemes provided the output is stable for a given test case. +Note: `testId` accepts any string format. Producers MAY use other deterministic schemes provided the output is stable for a given test case. --- From d97818a7d42bd59ff7ff91a7a1b85c4349b918b6 Mon Sep 17 00:00:00 2001 From: Matthew Thomas <25988124+Ma11hewThomas@users.noreply.github.com> Date: Sun, 26 Jul 2026 18:29:49 +0100 Subject: [PATCH 6/8] Tighten identity model semantics --- CHANGELOG.md | 2 +- schema/ctrf.schema.json | 24 +++-- spec/ctrf.md | 72 ++++++++----- tests/normative/string-constraints.test.json | 107 +++++++++++++++++++ tests/normative/valid-documents.test.json | 17 +++ 5 files changed, 187 insertions(+), 35 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 58625dd..512a5fc 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,7 +9,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Changed -- Added an optional identity and lineage model for CTRF documents, logical runs, test cases, executions, attempts, attachments, and shards ([#57](https://github.com/ctrf-io/ctrf/pull/57)). +- Added an optional identity model for CTRF documents, logical runs, test cases, executions, attempts, attachments, and shards ([#57](https://github.com/ctrf-io/ctrf/pull/57)). - Clarified namespace guidance for `extra` extension keys and examples ([#56](https://github.com/ctrf-io/ctrf/pull/56)). - Clarified immutability guidance for emitted CTRF report artifacts ([#55](https://github.com/ctrf-io/ctrf/pull/55)). - Clarified `tags` as simple keyless classifications and `labels` as structured key-value test metadata ([#54](https://github.com/ctrf-io/ctrf/pull/54)). diff --git a/schema/ctrf.schema.json b/schema/ctrf.schema.json index b900868..6184f37 100644 --- a/schema/ctrf.schema.json +++ b/schema/ctrf.schema.json @@ -17,11 +17,13 @@ "reportId": { "description": "Unique identifier for this CTRF document instance (UUID). Identifies the serialized artifact", "type": "string", + "minLength": 1, "format": "uuid" }, "runId": { "description": "Identifier for the logical test run. Shared across shards/documents belonging to the same run", - "type": "string" + "type": "string", + "minLength": 1 }, "timestamp": { "description": "Report generation time (RFC 3339 / ISO 8601)", @@ -138,15 +140,18 @@ "id": { "description": "Legacy identifier for the test case (UUID). New producers should prefer testId", "type": "string", + "minLength": 1, "format": "uuid" }, "testId": { "description": "Stable identifier for the logical test case. Should be derived from stable test attributes when generated by a producer", - "type": "string" + "type": "string", + "minLength": 1 }, "executionId": { "description": "Unique identifier for this specific execution of the test case within a run. UUID recommended", - "type": "string" + "type": "string", + "minLength": 1 }, "name": { "description": "Name or title of the test case", @@ -269,7 +274,8 @@ }, "attemptId": { "description": "Unique identifier for this individual retry attempt. UUID recommended", - "type": "string" + "type": "string", + "minLength": 1 }, "status": { "description": "Outcome of this attempt", @@ -333,7 +339,8 @@ "properties": { "attachmentId": { "description": "Unique identifier for this attachment reference instance. UUID recommended", - "type": "string" + "type": "string", + "minLength": 1 }, "name": { "description": "Display name of the attachment", @@ -407,7 +414,8 @@ "properties": { "attachmentId": { "description": "Unique identifier for this attachment reference instance. UUID recommended", - "type": "string" + "type": "string", + "minLength": 1 }, "name": { "description": "Display name of the attachment", @@ -572,7 +580,8 @@ }, "shardId": { "description": "Shard or partition label identifying which part of a distributed run produced this document", - "type": "string" + "type": "string", + "minLength": 1 }, "healthy": { "description": "Indicates if the run is considered healthy", @@ -639,6 +648,7 @@ "reportId": { "description": "UUID of the baseline report", "type": "string", + "minLength": 1, "format": "uuid" }, "timestamp": { diff --git a/spec/ctrf.md b/spec/ctrf.md index db57156..ca18415 100644 --- a/spec/ctrf.md +++ b/spec/ctrf.md @@ -143,9 +143,9 @@ Consumers MAY reconstruct hierarchical views for presentation purposes, but such --- -### 2.4. Identity and Lineage +### 2.4. Identity Model -CTRF provides a layered identity model that supports stable identification of test cases across runs, correlation of documents belonging to the same logical run, and traceability from individual attempts back to their parent execution and test case. +CTRF provides a layered identity model that supports stable identifiers for test cases, correlation of documents belonging to the same logical run, deduplication, aggregation, and future provenance support. Identity fields are OPTIONAL. When provided, they enable: @@ -153,7 +153,7 @@ Identity fields are OPTIONAL. When provided, they enable: - flakiness detection - shard and parallel-execution merging - historical baseline comparisons -- artifact lineage tracking +- artifact correlation Producers are encouraged to assign identity fields when the corresponding correlation is meaningful for their workflow. The normative requirements for identity fields are defined in Section 4.9. @@ -412,7 +412,7 @@ Unless otherwise specified in this document, all timestamps and duration values This applies to all fields representing points in time or elapsed durations, including (but not limited to) `start`, `stop`, and `duration` fields at all levels. -### 4.9. Identity and Lineage Model +### 4.9. Identity Model CTRF defines a layered identity model that enables correlation of test data across documents, runs, test cases, and individual executions. @@ -420,7 +420,7 @@ Each layer addresses a distinct identification concern: 1. **Document**: `reportId` identifies the emitted CTRF document as a serialized artifact. When `reportId` is used, each CTRF document SHOULD have exactly one `reportId`. 2. **Run**: `runId` identifies the logical test run. Multiple CTRF documents MAY share the same `runId` when they represent shards or partitions of a single coordinated execution. -3. **Test case**: `testId` identifies the logical test case. It SHOULD be deterministic and stable across runs, enabling cross-run analysis, trending, and flake detection. +3. **Test case**: `testId` identifies the logical test case within the producer's chosen scope. It SHOULD be deterministic and stable across runs within that scope, enabling cross-run analysis, trending, and flake detection. 4. **Execution**: `executionId` identifies a concrete execution of a test case within a run. An execution MAY include multiple retry attempts represented in `retryAttempts`. 5. **Attempt**: `attemptId` identifies an individual retry attempt within an execution. 6. **Attachment**: `attachmentId` identifies a specific attachment reference instance. @@ -428,6 +428,8 @@ Each layer addresses a distinct identification concern: All identity fields are OPTIONAL. Producers SHOULD assign them when the corresponding correlation is meaningful for their workflow. +This identity model provides primitives that can be used by future provenance and lineage extensions. It does not itself define derivation, parent/child report relationships, transformation history, or trust metadata. + #### 4.9.1. Uniqueness Each identity field has an intended uniqueness scope: @@ -436,9 +438,9 @@ Each identity field has an intended uniqueness scope: |---|---| | `reportId` | Unique per document | | `runId` | Unique per logical run | -| `testId` | Unique within the document | -| `executionId` | Unique per execution | -| `attemptId` | Unique per attempt | +| `testId` | Stable within the producer's chosen scope for the logical test case | +| `executionId` | Unique for one execution of a logical test case within a run | +| `attemptId` | Unique for one retry attempt of an execution | | `attachmentId` | Unique within the containing parent object | | `shardId` | Unique within a logical run | @@ -451,9 +453,9 @@ Identity fields differ in whether their values persist across runs or are unique | Field | Stability | |---|---| -| `reportId` | Unique per document; not stable across retransmissions | +| `reportId` | Stable across retransmissions of the same report; changes for materially changed report content | | `runId` | Stable across all documents belonging to the same logical run | -| `testId` | Stable across runs for the same logical test case | +| `testId` | Stable across runs for the same logical test case within the producer's chosen scope | | `executionId` | Unique per execution; not stable across runs | | `attemptId` | Unique per attempt; not stable across runs | | `attachmentId` | Unique per attachment reference instance | @@ -509,14 +511,16 @@ Consumers MUST NOT assume compatibility across different MAJOR versions. **Description:** A unique identifier for this CTRF document instance. -`reportId` identifies the serialized artifact — the emitted file or payload — rather than the logical test run or any entity within it. See Section 4.9 for the full identity model. +`reportId` identifies the report artifact or content — the emitted file or payload — rather than the logical test run or any entity within it. See Section 4.9 for the full identity model. **Requirements:** `reportId` is OPTIONAL. If present, it MUST be a valid UUID as defined in [RFC4122]. When `reportId` is used, each emitted document SHOULD receive a distinct `reportId`. -Each emitted CTRF document represents a distinct report instance. If a new CTRF document is produced through post-processing, it SHOULD use a different `reportId` value than the source document when `reportId` is used. +Retransmitting the same report means sending or uploading the same already-emitted CTRF document again without changing its content. A retransmission of the same report SHOULD preserve the same `reportId` when `reportId` is used. + +If a materially changed, regenerated, merged, filtered, or transformed CTRF document is produced, it SHOULD use a different `reportId` value than the source document when `reportId` is used. --- @@ -527,9 +531,11 @@ Identifies the logical test run to which this document belongs. A logical run MAY span multiple CTRF documents (for example, when tests are distributed across shards or parallel workers). All documents belonging to the same coordinated execution SHOULD share the same `runId`. +Multiple CTRF documents MAY share the same `runId` when they describe results from the same logical run. + **Requirements:** `runId` is OPTIONAL. -If present, it MUST be a string. +If present, it MUST be a non-empty string. Producers SHOULD ensure that `runId` is unique across distinct logical runs. Consumers MAY use `runId` to merge or correlate documents from the same logical run. @@ -864,22 +870,24 @@ A unique, stable identifier for the test case. If present, it MUST be a valid UUID as defined in [RFC4122]. Producers SHOULD prefer `testId` for new implementations. When both `id` and `testId` are present, consumers SHOULD use `testId`. +Consumers SHOULD treat `id` as legacy unless a producer documents stronger semantics. ### 9.2. `testId` **Description:** A stable identifier for the logical test case. -`testId` identifies the test case itself, independent of any particular run or execution. The same logical test case SHOULD produce the same `testId` across runs, machines, and environments. +`testId` identifies the test case itself, independent of any particular run or execution. Producers SHOULD keep `testId` stable for the same logical test within their own documented scope. **Requirements:** `testId` is OPTIONAL. -If present, it MUST be a string. +If present, it MUST be a non-empty string. `testId` SHOULD be deterministic and stable across runs for the same test case. When generated by a producer, `testId` SHOULD be derived from stable test attributes such as file path, suite path, and test name. A producer MAY use a canonical string directly, or MAY encode that canonical string as a deterministic identifier such as a UUID version 5 or hash. Consumers MUST treat the `testId` value as opaque. +Consumers MUST NOT assume that `testId` values generated by different producers are globally comparable unless the producers document a shared generation scheme. ### 9.3. `executionId` @@ -890,7 +898,7 @@ An execution represents a single reported outcome for a test case in this docume **Requirements:** `executionId` is OPTIONAL. -If present, it MUST be a string. +If present, it MUST be a non-empty string. UUID is RECOMMENDED. `executionId` SHOULD be unique across executions and SHOULD NOT be reused across runs. @@ -1347,7 +1355,7 @@ When a test run is distributed across multiple shards, workers, or machines, `sh **Requirements:** `shardId` is OPTIONAL. -If present, it MUST be a string. +If present, it MUST be a non-empty string. `shardId` values SHOULD be unique within documents sharing the same `runId`. The value MAY be human-readable (e.g. `"shard-1"`, `"worker-3"`). @@ -1402,7 +1410,7 @@ A unique identifier for this individual retry attempt. **Requirements:** `attemptId` is OPTIONAL. -If present, it MUST be a string. +If present, it MUST be a non-empty string. UUID is RECOMMENDED. `attemptId` SHOULD be unique across attempts. @@ -1568,7 +1576,7 @@ A unique identifier for this attachment reference instance. **Requirements:** `attachmentId` is OPTIONAL. -If present, it MUST be a string. +If present, it MUST be a non-empty string. UUID is RECOMMENDED. `attachmentId` SHOULD be unique within the containing parent object. @@ -1965,7 +1973,7 @@ Producers SHOULD: - emit a new CTRF document when performing post-processing after initial report generation and, when `reportId` is used, assign a different `reportId` value rather than modifying an emitted CTRF document - assign identity fields (`testId`, `executionId`, `attemptId`, `attachmentId`) when traceability is meaningful for their workflow - avoid duplicate identity values when duplicates would make correlation ambiguous -- prefer `testId` over the deprecated `id` field for new implementations +- prefer `testId` over the legacy `id` field for new implementations --- @@ -2039,11 +2047,13 @@ to this specification. "reportId": { "description": "Unique identifier for this CTRF document instance (UUID). Identifies the serialized artifact", "type": "string", + "minLength": 1, "format": "uuid" }, "runId": { "description": "Identifier for the logical test run. Shared across shards/documents belonging to the same run", - "type": "string" + "type": "string", + "minLength": 1 }, "timestamp": { "description": "Report generation time (RFC 3339 / ISO 8601)", @@ -2160,15 +2170,18 @@ to this specification. "id": { "description": "Legacy identifier for the test case (UUID). New producers should prefer testId", "type": "string", + "minLength": 1, "format": "uuid" }, "testId": { "description": "Stable identifier for the logical test case. Should be derived from stable test attributes when generated by a producer", - "type": "string" + "type": "string", + "minLength": 1 }, "executionId": { "description": "Unique identifier for this specific execution of the test case within a run. UUID recommended", - "type": "string" + "type": "string", + "minLength": 1 }, "name": { "description": "Name or title of the test case", @@ -2291,7 +2304,8 @@ to this specification. }, "attemptId": { "description": "Unique identifier for this individual retry attempt. UUID recommended", - "type": "string" + "type": "string", + "minLength": 1 }, "status": { "description": "Outcome of this attempt", @@ -2355,7 +2369,8 @@ to this specification. "properties": { "attachmentId": { "description": "Unique identifier for this attachment reference instance. UUID recommended", - "type": "string" + "type": "string", + "minLength": 1 }, "name": { "description": "Display name of the attachment", @@ -2429,7 +2444,8 @@ to this specification. "properties": { "attachmentId": { "description": "Unique identifier for this attachment reference instance. UUID recommended", - "type": "string" + "type": "string", + "minLength": 1 }, "name": { "description": "Display name of the attachment", @@ -2594,7 +2610,8 @@ to this specification. }, "shardId": { "description": "Shard or partition label identifying which part of a distributed run produced this document", - "type": "string" + "type": "string", + "minLength": 1 }, "healthy": { "description": "Indicates if the run is considered healthy", @@ -2661,6 +2678,7 @@ to this specification. "reportId": { "description": "UUID of the baseline report", "type": "string", + "minLength": 1, "format": "uuid" }, "timestamp": { diff --git a/tests/normative/string-constraints.test.json b/tests/normative/string-constraints.test.json index f52540a..b8da602 100644 --- a/tests/normative/string-constraints.test.json +++ b/tests/normative/string-constraints.test.json @@ -15,6 +15,34 @@ } } }, + { + "description": "Document with empty reportId violating minLength constraint", + "valid": false, + "data": { + "reportFormat": "CTRF", + "specVersion": "1.0.0", + "reportId": "", + "results": { + "tool": { "name": "runner" }, + "summary": { "tests": 1, "passed": 1, "failed": 0, "pending": 0, "skipped": 0, "other": 0, "start": 1609459200000, "stop": 1609459201000 }, + "tests": [{ "name": "test", "status": "passed", "duration": 100 }] + } + } + }, + { + "description": "Document with empty runId violating minLength constraint", + "valid": false, + "data": { + "reportFormat": "CTRF", + "specVersion": "1.0.0", + "runId": "", + "results": { + "tool": { "name": "runner" }, + "summary": { "tests": 1, "passed": 1, "failed": 0, "pending": 0, "skipped": 0, "other": 0, "start": 1609459200000, "stop": 1609459201000 }, + "tests": [{ "name": "test", "status": "passed", "duration": 100 }] + } + } + }, { "description": "Tool with name violating minLength constraint", "valid": false, @@ -41,6 +69,45 @@ } } }, + { + "description": "Test with empty legacy id violating minLength constraint", + "valid": false, + "data": { + "reportFormat": "CTRF", + "specVersion": "1.0.0", + "results": { + "tool": { "name": "runner" }, + "summary": { "tests": 1, "passed": 1, "failed": 0, "pending": 0, "skipped": 0, "other": 0, "start": 1609459200000, "stop": 1609459201000 }, + "tests": [{ "id": "", "name": "test", "status": "passed", "duration": 100 }] + } + } + }, + { + "description": "Test with empty testId violating minLength constraint", + "valid": false, + "data": { + "reportFormat": "CTRF", + "specVersion": "1.0.0", + "results": { + "tool": { "name": "runner" }, + "summary": { "tests": 1, "passed": 1, "failed": 0, "pending": 0, "skipped": 0, "other": 0, "start": 1609459200000, "stop": 1609459201000 }, + "tests": [{ "testId": "", "name": "test", "status": "passed", "duration": 100 }] + } + } + }, + { + "description": "Test with empty executionId violating minLength constraint", + "valid": false, + "data": { + "reportFormat": "CTRF", + "specVersion": "1.0.0", + "results": { + "tool": { "name": "runner" }, + "summary": { "tests": 1, "passed": 1, "failed": 0, "pending": 0, "skipped": 0, "other": 0, "start": 1609459200000, "stop": 1609459201000 }, + "tests": [{ "executionId": "", "name": "test", "status": "passed", "duration": 100 }] + } + } + }, { "description": "Test with suite violating minItems constraint", "valid": false, @@ -79,6 +146,46 @@ "tests": [{ "name": "test", "status": "passed", "duration": 100, "retryAttempts": [{ "attempt": -1, "status": "failed" }] }] } } + }, + { + "description": "RetryAttempt with empty attemptId violating minLength constraint", + "valid": false, + "data": { + "reportFormat": "CTRF", + "specVersion": "1.0.0", + "results": { + "tool": { "name": "runner" }, + "summary": { "tests": 1, "passed": 1, "failed": 0, "pending": 0, "skipped": 0, "other": 0, "start": 1609459200000, "stop": 1609459201000 }, + "tests": [{ "name": "test", "status": "passed", "duration": 100, "retryAttempts": [{ "attempt": 1, "attemptId": "", "status": "failed" }] }] + } + } + }, + { + "description": "Attachment with empty attachmentId violating minLength constraint", + "valid": false, + "data": { + "reportFormat": "CTRF", + "specVersion": "1.0.0", + "results": { + "tool": { "name": "runner" }, + "summary": { "tests": 1, "passed": 1, "failed": 0, "pending": 0, "skipped": 0, "other": 0, "start": 1609459200000, "stop": 1609459201000 }, + "tests": [{ "name": "test", "status": "passed", "duration": 100, "attachments": [{ "attachmentId": "", "name": "file", "contentType": "text/plain", "path": "/path" }] }] + } + } + }, + { + "description": "Environment with empty shardId violating minLength constraint", + "valid": false, + "data": { + "reportFormat": "CTRF", + "specVersion": "1.0.0", + "results": { + "tool": { "name": "runner" }, + "summary": { "tests": 1, "passed": 1, "failed": 0, "pending": 0, "skipped": 0, "other": 0, "start": 1609459200000, "stop": 1609459201000 }, + "tests": [{ "name": "test", "status": "passed", "duration": 100 }], + "environment": { "shardId": "" } + } + } } ] } diff --git a/tests/normative/valid-documents.test.json b/tests/normative/valid-documents.test.json index 792cd7a..0598df8 100644 --- a/tests/normative/valid-documents.test.json +++ b/tests/normative/valid-documents.test.json @@ -26,6 +26,23 @@ "description": "Document with retries", "valid": true, "dataPath": "../../examples/with-retries.json" + }, + { + "description": "Document with same testId and different executionId values", + "valid": true, + "data": { + "reportFormat": "CTRF", + "specVersion": "1.0.0", + "runId": "run-1", + "results": { + "tool": { "name": "runner" }, + "summary": { "tests": 2, "passed": 2, "failed": 0, "pending": 0, "skipped": 0, "other": 0, "start": 1609459200000, "stop": 1609459201000 }, + "tests": [ + { "name": "test", "testId": "tests/example.test.ts::suite::test", "executionId": "execution-1", "status": "passed", "duration": 100 }, + { "name": "test", "testId": "tests/example.test.ts::suite::test", "executionId": "execution-2", "status": "passed", "duration": 100 } + ] + } + } } ] } From f87f841593772a7eedd3c2801daab3897e25cafb Mon Sep 17 00:00:00 2001 From: Matthew Thomas <25988124+Ma11hewThomas@users.noreply.github.com> Date: Sun, 26 Jul 2026 18:31:18 +0100 Subject: [PATCH 7/8] Format identity schema updates --- schema/ctrf.schema.json | 12 ++++++------ spec/ctrf.md | 12 ++++++------ 2 files changed, 12 insertions(+), 12 deletions(-) diff --git a/schema/ctrf.schema.json b/schema/ctrf.schema.json index 6184f37..2cee012 100644 --- a/schema/ctrf.schema.json +++ b/schema/ctrf.schema.json @@ -17,8 +17,8 @@ "reportId": { "description": "Unique identifier for this CTRF document instance (UUID). Identifies the serialized artifact", "type": "string", - "minLength": 1, - "format": "uuid" + "format": "uuid", + "minLength": 1 }, "runId": { "description": "Identifier for the logical test run. Shared across shards/documents belonging to the same run", @@ -140,8 +140,8 @@ "id": { "description": "Legacy identifier for the test case (UUID). New producers should prefer testId", "type": "string", - "minLength": 1, - "format": "uuid" + "format": "uuid", + "minLength": 1 }, "testId": { "description": "Stable identifier for the logical test case. Should be derived from stable test attributes when generated by a producer", @@ -648,8 +648,8 @@ "reportId": { "description": "UUID of the baseline report", "type": "string", - "minLength": 1, - "format": "uuid" + "format": "uuid", + "minLength": 1 }, "timestamp": { "description": "Generation time of the baseline report", diff --git a/spec/ctrf.md b/spec/ctrf.md index ca18415..850818d 100644 --- a/spec/ctrf.md +++ b/spec/ctrf.md @@ -2047,8 +2047,8 @@ to this specification. "reportId": { "description": "Unique identifier for this CTRF document instance (UUID). Identifies the serialized artifact", "type": "string", - "minLength": 1, - "format": "uuid" + "format": "uuid", + "minLength": 1 }, "runId": { "description": "Identifier for the logical test run. Shared across shards/documents belonging to the same run", @@ -2170,8 +2170,8 @@ to this specification. "id": { "description": "Legacy identifier for the test case (UUID). New producers should prefer testId", "type": "string", - "minLength": 1, - "format": "uuid" + "format": "uuid", + "minLength": 1 }, "testId": { "description": "Stable identifier for the logical test case. Should be derived from stable test attributes when generated by a producer", @@ -2678,8 +2678,8 @@ to this specification. "reportId": { "description": "UUID of the baseline report", "type": "string", - "minLength": 1, - "format": "uuid" + "format": "uuid", + "minLength": 1 }, "timestamp": { "description": "Generation time of the baseline report", From d63d6c21f27c6f924d608629543631a2997ce66f Mon Sep 17 00:00:00 2001 From: Matthew Thomas <25988124+Ma11hewThomas@users.noreply.github.com> Date: Sun, 26 Jul 2026 18:51:53 +0100 Subject: [PATCH 8/8] Cover empty baseline reportId --- tests/normative/string-constraints.test.json | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/tests/normative/string-constraints.test.json b/tests/normative/string-constraints.test.json index b8da602..d37e206 100644 --- a/tests/normative/string-constraints.test.json +++ b/tests/normative/string-constraints.test.json @@ -186,6 +186,20 @@ "environment": { "shardId": "" } } } + }, + { + "description": "Baseline with empty reportId violating minLength constraint", + "valid": false, + "data": { + "reportFormat": "CTRF", + "specVersion": "1.0.0", + "results": { + "tool": { "name": "runner" }, + "summary": { "tests": 1, "passed": 1, "failed": 0, "pending": 0, "skipped": 0, "other": 0, "start": 1609459200000, "stop": 1609459201000 }, + "tests": [{ "name": "test", "status": "passed", "duration": 100 }] + }, + "baseline": { "reportId": "" } + } } ] }