diff --git a/codes/codes-workload-config.h b/codes/codes-workload-config.h new file mode 100644 index 00000000..0b014878 --- /dev/null +++ b/codes/codes-workload-config.h @@ -0,0 +1,100 @@ +/* + * Copyright (C) 2013 University of Chicago. + * See COPYRIGHT notice in top-level directory. + * + */ + +#ifndef CODES_WORKLOAD_CONFIG_H +#define CODES_WORKLOAD_CONFIG_H + +/** + * @file codes-workload-config.h + * + * Apply synthetic-workload parameters from a loaded config to a model main's + * option globals, with command-line precedence. + * + * The YAML front-end compiles a `workload:` shortcut or a single all-nodes + * `jobs:` entry into a WORKLOAD section (see the config compiler). A synthetic + * main calls the helpers below once, after configuration_load()/ + * codes_mapping_setup(), passing each of the ROSS options it registered + * (`traffic`, `num_messages`, `arrival_time`, `payload_size`) together with that + * option's compiled-in default. For each knob the precedence is: + * + * command line > config (WORKLOAD section) > the model's built-in default + * + * Command-line detection follows the same pattern codes_mapping uses for the + * simulation end time: ROSS records no "was this option set" flag, so the helper + * compares the current global against the registered default. An unchanged value + * means the command line did not set it, so a configured value may take over; any + * other value is treated as a command-line override and left untouched. The edge + * case is the same as end_time's: passing a value equal to the default on the + * command line is indistinguishable from omitting it, so the config wins there. + * + * A legacy `.conf` run carries no WORKLOAD section, so every apply is a no-op and + * the model keeps its command-line/default behavior exactly as before. + */ + +#ifdef __cplusplus +extern "C" { +#endif + +/** + * Maps a friendly traffic-pattern name (as written in a YAML `workload:` block) + * to the value of a model main's own traffic enum. Each main supplies its own + * table because the pattern set and the enum values differ per model (only + * "uniform" is universally 1). + */ +struct codes_workload_traffic_name { + const char* name; /**< friendly name, e.g. "uniform" (NULL terminates a table) */ + int value; /**< the main's traffic enum value for that pattern */ +}; + +/** + * True (non-zero) if the loaded config carries a WORKLOAD section -- i.e. a YAML + * `workload:`/`jobs:` config. Legacy `.conf` configs have none. + */ +int codes_workload_config_present(void); + +/** + * Abort (via tw_error) if the loaded config carries a JOBS section -- an explicit + * multi-job placement (see the config compiler). The synthetic mains generate a + * single global traffic pattern and cannot yet execute per-job placement, so + * running such a config would silently ignore it. The config still compiles and + * validates; this makes a main that cannot honor it fail loudly with guidance + * rather than misrun. A no-op for a single-workload (WORKLOAD-section) or legacy + * config. @p model_name names the model in the diagnostic. + */ +void codes_workload_config_check_unsupported_jobs(const char* model_name); + +/** + * Apply an integer WORKLOAD key to @p val unless the command line already set it. + * + * @param key the WORKLOAD key name (e.g. "num_messages", "payload_size"). + * @param val the option global; overwritten only when it still equals + * @p cli_default and the config provides @p key. + * @param cli_default the option's registered default (the command-line sentinel). + */ +void codes_workload_config_apply_int(const char* key, int* val, int cli_default); + +/** + * Apply a floating-point WORKLOAD key to @p val unless the command line already + * set it. Semantics match codes_workload_config_apply_int (used for + * `arrival_time`, resolved to nanoseconds by the compiler). + */ +void codes_workload_config_apply_double(const char* key, double* val, double cli_default); + +/** + * Apply WORKLOAD/traffic (a friendly pattern name) to @p val unless the command + * line already set it, mapping the name through @p names (a table terminated by a + * `{NULL, 0}` entry). A no-op if @p val already differs from @p cli_default or no + * traffic is configured. Aborts via tw_error if the configured name is not in + * @p names, naming the offending value. + */ +void codes_workload_config_apply_traffic(int* val, int cli_default, + const struct codes_workload_traffic_name* names); + +#ifdef __cplusplus +} +#endif + +#endif /* CODES_WORKLOAD_CONFIG_H */ diff --git a/codes/configfile.h b/codes/configfile.h index d6691683..629ceb1e 100644 --- a/codes/configfile.h +++ b/codes/configfile.h @@ -8,6 +8,7 @@ #define SRC_COMMON_MODELCONFIG_CONFIGFILE_H #include /* size_t */ +#include /* FILE (cf_equal_report) */ #include #ifdef __cplusplus @@ -94,9 +95,34 @@ struct ConfigVTable { * */ int cf_dump(struct ConfigVTable* cf, SectionHandle h, char** err); -/* Compare two config trees: return true if equal, false if not */ +/** + * Compare two config trees for equality. + * + * Equality is structural and **order-insensitive**: the two trees are equal when + * they hold the same named sections and keys with the same values, regardless of + * the order entries were written -- because the config store keys every lookup by + * name (sections case-insensitively, keys case-sensitively) and never by + * position. The value *list* of a key is compared as an ordered tuple, so a + * list-valued key such as `modelnet_order` must match value-for-value in order. + * + * @return 1 if the trees are equal, 0 otherwise. + */ int cf_equal(struct ConfigVTable* h1, struct ConfigVTable* h2); +/** + * Like cf_equal(), but on inequality writes a human-readable description of every + * divergence -- each under its `section/key` path, saying whether an entry is + * missing from one side, is a key on one side and a section on the other, or has + * differing values -- to @p report. Pass a NULL @p report for the same result + * with no output (identical to cf_equal). Intended for tests and diagnostics, so + * a failed comparison points at the exact section/key that diverged rather than + * only reporting that the trees differ. + * + * @param report stream to write the divergence report to, or NULL for none. + * @return 1 if the trees are equal, 0 otherwise. + */ +int cf_equal_report(struct ConfigVTable* h1, struct ConfigVTable* h2, FILE* report); + static inline int cf_free(struct ConfigVTable* cf) { if (!cf) return 1; diff --git a/doc/dev/yaml-config.md b/doc/dev/yaml-config.md index 1be0864c..3ccd077c 100644 --- a/doc/dev/yaml-config.md +++ b/doc/dev/yaml-config.md @@ -24,13 +24,15 @@ enable and no configure-time option to set. ## Shape of a config -A config has up to five top-level blocks: +A config has up to seven top-level blocks: ```yaml schema_version: 1 # required; this build understands version 1 components: # named component configs referenced by the topology topology: # flat network, parametric fabric, or explicit LP groups +jobs: # explicit multi-job workload placement (or an inline workload: on a component) sections: # config a model reads directly, carried through verbatim +simulation: # run-level settings (end time, event-pool factor) include: # other config files to reuse (base; this file overrides) ``` @@ -40,15 +42,24 @@ include: # other config files to reuse (base; this file overrides) - **`components`** is a map of name → component. A **component** pairs a model (`model:`) with its parameters; a flat-network component also names the NIC model it runs over (`network:`). Components are referenced by name from the - topology. The key `type:` is reserved for a future schema version and is - rejected rather than passed through as a model param. + topology. The compute component may carry an inline `workload:` — the + single-workload shortcut, see "Workloads and jobs" below. The key `type:` is + reserved for a future schema version and is rejected rather than passed through + as a model param. - **`topology`** selects the layout, via one of three `format`s: `flat` is an all-to-all point-to-point network described by a component and a node count; `parametric` is an HPC fabric described by shape parameters; `groups` lays the LP groups out directly (the escape hatch for configs that aren't a single network). Each is documented in its own section below. +- **`jobs`** places workloads on the topology's nodes — what the nodes run, as + opposed to what they are. The common single-workload case is instead an inline + `workload:` on the compute component; both are covered in "Workloads and jobs" + below. - **`sections`** carries config a model reads directly by name (DIRECTOR, NETWORK_SURROGATE, resource, …) through verbatim — see `sections:` below. +- **`simulation`** carries run-level settings — the simulation end time and the + ROSS per-PE event-pool factor — that belong to the run rather than to the + topology or a model. See `simulation:` below. - **`include`** reuses other config files — see `include:` below. Validation is strict: unknown top-level keys, unknown topology keys, a block a @@ -70,6 +81,117 @@ The compiler walks the friendly form and emits the `LPGROUPS` (LP layout) and - any scalar key the compiler doesn't special-case is **passed through verbatim** to `PARAMS`, so advanced model knobs need no compiler change. +The compiler derives `modelnet_order` from the fabric/network model, so it must +**not** be set by hand on a flat component or a parametric fabric — doing so would +otherwise be silently ignored (the compiler-derived value wins), so it is rejected +as a config error instead. (The explicit-groups form derives nothing, so there you +write `modelnet_order` yourself under `params:`.) + +--- + +# Units and dimensioned values + +A parameter that carries a physical dimension — a latency, a size, a bandwidth — +may be written **either** as a bare number in the model's internal unit **or** as +a unit-bearing string that the compiler converts to that unit before emitting it. + +```yaml + packet_size: 2KiB # -> 2048 (bytes) + router_delay: 1.5us # -> 1500 (nanoseconds) + cn_bandwidth: 100Gbps # -> converted to the model's bandwidth unit +``` + +## Bare numbers: the default unit + +A bare number keeps a documented default unit: + +| Quantity | Bare number means | +|----------|-------------------| +| time / latency | **nanoseconds** | +| size | **bytes** | +| count (num_routers, num_vcs, …) | **unitless** | +| bandwidth | **the model's internal unit** — see the table below | + +For time and size these defaults are exactly the units every model already reads, +so a bare number and an explicit `ns`/`B` suffix produce the identical value — and +every existing config keeps compiling unchanged. **Bandwidth is the exception:** +CODES models do not agree on a bandwidth unit (some read GiB/s, one reads bytes/ns, +simplenet reads MiB/s), so there is no safe universal default. A bare bandwidth +number is passed through as the model's internal unit. **Writing bandwidth with an +explicit unit is strongly recommended** — it says what you mean regardless of which +model reads it. + +## Accepted unit suffixes + +The suffix is matched exactly and is **case-sensitive** — the bit/byte distinction +rides on the case of `b`/`B` (`Gbps` is gigabit/s, `GBps` is gigabyte/s). + +| Quantity | Suffixes | +|----------|----------| +| time | `ns`, `us` (microseconds), `ms`, `s` | +| size | `B`, `KiB`, `MiB`, `GiB` (binary, 1024-based); `KB`, `MB`, `GB` (decimal, 1000-based) | +| bandwidth — bit rates | `bps`, `Kbps`, `Mbps`, `Gbps` (decimal, divided by 8 to bytes) | +| bandwidth — byte rates | `Bps`, `KBps`, `MBps`, `GBps` (decimal); `KiBps`, `MiBps`, `GiBps` (binary) | + +Conversions are exact whenever the result is a whole number (`2KiB` → `2048`, +`1.5us` → `1500`); otherwise the emitted value is a plain decimal (never +scientific notation) that round-trips to the same double the model would compute. + +**Rejections** (each a config error naming the parameter): + +- an **unknown suffix** or trailing junk on a dimensioned parameter (`packet_size: + 512qux`); +- a unit of the **wrong quantity** (`packet_size: 5ms` — a time on a size); +- a **negative** dimensioned value (`cn_bandwidth: -1Gbps`); +- a unit on a parameter the compiler **cannot classify** (a plain count or an + opaque pass-through knob, e.g. `num_vcs: 4KiB`). The model would read such a + value with `atof`/`strtol` and silently keep only the leading number, so the + front-end rejects it: drop the unit and write the model's internal unit, or use + a recognized dimensioned parameter. (A clearly non-numeric value — a routing + name, a file path — still passes through untouched.) + +Units are applied to the friendly **component** and **fabric** parameters (flat +and parametric topologies). The escape-hatch `format: groups` `params:` and the +verbatim `sections:` blocks are passed through unchanged, so write internal-unit +numbers there. + +## Bandwidth internal units per model + +A bare bandwidth number — and any value you convert — lands in the unit the model +actually reads. These come from each model's own byte-time arithmetic: + +| Model(s) | Bandwidth parameter(s) | Internal unit (what a bare number means) | +|----------|------------------------|------------------------------------------| +| dragonfly, dragonfly-dally, dragonfly-plus, dragonfly-custom, slimfly | `local_bandwidth`, `global_bandwidth`, `cn_bandwidth` | **GiB/s** (1024³ bytes/s) | +| torus | `link_bandwidth` | **GiB/s** | +| express-mesh | `link_bandwidth`, `cn_bandwidth` | **GiB/s** | +| fattree | `link_bandwidth`, `cn_bandwidth` | **bytes/ns** (= GB/s, decimal 10⁹) — **not** GiB/s | +| simplenet | `net_bw_mbps` | **MiB/s** (1024² bytes/s) — despite the `mbps` name | +| simplep2p | `net_bw_mbps_file` (per-pair matrix) | MiB/s, read from the referenced file (not a scalar, not converted) | +| loggp | rates come from `net_config_file` | not a scalar parameter | + +The same string means different physical rates across models: `link_bandwidth: +1GBps` is `1` in fattree (bytes/ns) but `≈0.931` in torus (GiB/s). Because a bare +number is a raw pass-through, `link_bandwidth: 12.5` likewise means 12.5 GiB/s in +torus and 12.5 bytes/ns in fattree — another reason to prefer explicit bandwidth +units. + +## Time and size internal units + +Every size parameter (`packet_size`, `chunk_size`, `message_size`, `vc_size` and +the `*_vc_size` variants, `buffer_size`, `credit_size`) is read in **bytes**, and +the common latency knobs (`router_delay`, `soft_delay`, `net_startup_ns`) in +**nanoseconds** — so a bare number needs no suffix. The one exception is the +dragonfly QoS statistics window, read in **microseconds**: + +| Model(s) | Parameter(s) | Internal unit | +|----------|--------------|---------------| +| dragonfly-dally, dragonfly-custom | `counting_start`, `counting_interval` | **microseconds** | +| dragonfly-plus | `counting_start`, `counting_interval`, `counting_end` | **microseconds** | + +A bare number there means microseconds (`counting_start: 100` is 100 µs); an +explicit time unit is converted accordingly (`counting_start: 5ms` → `5000`). + --- # Flat networks @@ -608,7 +730,10 @@ topology: Each group names its `repetitions` and, under `lps`, the LP types with their per-repetition counts — a direct, validated transcription of a `.conf` `LPGROUPS`. There is no network to derive from, so `modelnet_order` and any other -knobs come from whatever you put in `params`. `params:` follows the same +knobs come from whatever you put in `params`. `modelnet_order` is **required** when +your layout includes model-net LP types (e.g. `modelnet_dragonfly`) and lists the +methods present; it is simply omitted for configs with no model-net models at all +(a pure storage cluster or mapping test). `params:` follows the same scalar/list/nested-map rules as a `sections:` block (below): a scalar becomes a single-value key, a list a multi-value key, and a nested map a subsection. Group and LP-type names are free-form (they match what each model registers). @@ -630,6 +755,131 @@ exactly that. --- +# Workloads and jobs: what a node runs + +What a node **runs** is separate from what a node **is**. A topology gives you +the node models and the network; a workload says what traffic those nodes +generate. There are two spellings, and a config uses **one or the other, never +both**. + +## The single-workload shortcut: inline `workload:` + +The common case — every compute node runs the same workload — is an inline +`workload:` block on the topology's compute component: + +```yaml +components: + compute_host: + model: nw-lp + network: { ... } # (flat topologies) or omitted for a parametric fabric + workload: + type: synthetic + traffic: uniform # a pattern name the synthetic model accepts + num_messages: 30 # positive integer + payload_size: "2KiB" # size-valued: units apply (resolves to bytes) + arrival_time: "1us" # time-valued: units apply (resolves to nanoseconds) +``` + +It desugars to a single job on **all** of that component's nodes. The compiler +lowers it to a `WORKLOAD` section carrying the resolved params, which shows up in +the resolved-config dump like anything else. A `workload:` may live **only** on +the component the topology actually runs (the flat `component:` or the parametric +`hosts.component`); putting one on any other component, or on an explicit-groups +topology, is a config error. + +## Explicit multi-job: `jobs:` + +To place several workloads on disjoint sets of nodes, use a top-level `jobs:` +list instead: + +```yaml +jobs: + - id: production + workload: { type: synthetic, traffic: uniform, num_messages: 30 } + ranks: 256 + placement: { policy: contiguous } + - id: background + workload: { type: synthetic, traffic: nearest_neighbor, arrival_time: "1us" } + ranks: 128 + placement: { nodes: [ 384, 385, 386, ... ] } # explicit node list — the escape hatch +``` + +Each job needs a unique non-empty `id`, exactly one `workload`, a positive +`ranks` count, and a `placement` that is **exactly one** of: + +- `{ policy: contiguous }` — the job is assigned the next contiguous range of + node slots (jobs pack in declared order from node 0); or +- `{ nodes: [ ... ] }` — an explicit list of node indices, whose length must + equal `ranks`. + +The compiler validates the whole allocation: every node index is in range +(`0 .. total_slots-1`), no node is booked by two jobs, and the total rank count +fits the topology. A single synthetic job placed contiguously on **every** node +compiles to exactly the `WORKLOAD` section the inline shortcut produces — the two +spellings are interchangeable for that case. Any richer placement compiles to a +`JOBS` section: a `num_jobs` marker plus one subsection per job carrying its +resolved workload, `ranks`, and the resolved `nodes` allocation. + +## Which workload types are wired + +Only `type: synthetic` is wired end to end. Other workload types (e.g. `dumpi` +trace replay) and a job-level `qos:` key are **recognized** — the block shape is +understood — but rejected with a clear "not yet configurable from this format" +error; keep the legacy `.conf` path for those. Unknown keys anywhere in a +`workload:` or `jobs:` block are rejected like everywhere else in the front-end. + +### Synthetic workload keys + +| key | value | notes | +| -------------- | ------------------------------ | --------------------------------------- | +| `type` | `synthetic` | required; the discriminator | +| `traffic` | a pattern name (`uniform`, …) | mapped to the model's own traffic enum | +| `num_messages` | positive integer | messages generated per terminal | +| `payload_size` | size-valued (`"2KiB"`, `2048`) | resolves to bytes | +| `arrival_time` | time-valued (`"1us"`, `1000`) | resolves to nanoseconds | + +`traffic` is left as a **verbatim name** the synthetic model maps to its own +enum, because the pattern set and the enum values differ per model (only +`uniform` is universally pattern 1). Each synthetic driver accepts the patterns +its C enum defines — e.g. the dragonfly driver takes `uniform`, `nearest_group`, +`nearest_neighbor`, `rand_perm`, `random_other_group`; fat-tree adds `bisection`; +slim-fly adds `worst_case`, `gather`, `scatter`, and the `nearest_neighbor_1d/2d/ +3d` variants. An unrecognized name aborts at run time with a diagnostic naming +the offending value. + +## Command-line precedence for synthetic params + +The synthetic drivers already expose `traffic`, `num_messages`, `payload_size`, +and `arrival_time` as ROSS command-line options. Configuring them here does not +take that away — the precedence is: + +``` +command line > config (workload:/jobs:) > the model's built-in default +``` + +A driver applies each config value only when the command line did not set that +option, so a parameter sweep can share one config and vary a single knob on the +command line. Detection works exactly like `end_time`'s (below): the option's +global is compared to its registered default, so a value equal to the default +passed on the command line is indistinguishable from omitting it (the config wins +there) — the same one edge case, per knob. A legacy `.conf` carries no `WORKLOAD` +section, so every apply is a no-op and `.conf` behavior is unchanged. + +## Multi-job execution status + +Multi-job placement **compiles and validates** today, but the synthetic drivers +do **not execute** it yet: they generate a single global traffic pattern with no +per-job destination dispatch. A driver that loads a multi-job `JOBS` config +therefore **aborts** with a clear message rather than silently running global +traffic and ignoring the placement. To run a multi-job workload today, use +`model-net-mpi-replay` (trace replay with an `alloc_file`), or collapse the +config to a single `workload:` (or a one-job `jobs:` entry on all nodes), which +does execute. The compiled `JOBS` section — job count and per-job node +allocation — is the inspectable artifact a future job-aware synthetic executor +would consume. + +--- + # Config a model reads directly: `sections:` Not every subsystem's config is topology the compiler derives. Many read their @@ -707,6 +957,102 @@ section or tier introduces. --- +# Run-level settings: `simulation:` + +A few settings belong to the **run** rather than to the topology or any one +model — how long to simulate, and how big to size ROSS's per-PE event pool. They +go in a top-level `simulation:` block: + +```yaml +simulation: + end_time: "100us" # optional: simulation end time (see CLI precedence below) + pe_mem_factor: 512 # optional: advanced ROSS event-pool factor (most leave unset) +``` + +Both keys are optional, and any other key in the block is rejected — the same +strict validation as everywhere else. The compiler resolves each to the `PARAMS` +key the simulator already reads (`PARAMS/end_time`, `PARAMS/pe_mem_factor`), so +they show up in the resolved-config dump like any other parameter, and a legacy +`.conf` may carry the same `PARAMS` keys directly with identical effect (there is +no compiler for a `.conf`, so a `.conf` user just writes them under `PARAMS`). +Setting one of these keys **both** in `simulation:` and as a pass-through model +parameter (on a component/fabric, or in an explicit-groups `params:`) is a config +error rather than a silent drop — put it in one place. + +## `end_time` + +The simulation end time — how far in virtual time the run advances before it +stops. It is **time-valued**: write a bare number (nanoseconds, the unit ROSS and +every model use internally) or a value with a time unit (`"100us"`, `"2ms"`); a +size or bandwidth unit, a non-positive value, or garbage is rejected. It must be +positive. + +**Command line wins.** ROSS's own `--end=` option always takes precedence: if +you pass `--end` on the command line, that value is used and the config's +`end_time` is ignored. Only when `--end` is *not* given does the config value +apply. If neither is set, behavior is exactly as before (ROSS's built-in +default). + +**The one edge case.** ROSS records only the *value* of `--end`, not whether it +was set, so the config front-end distinguishes "no `--end`" from "`--end` given" +by comparing against ROSS's compiled-in default (`100000.0` ns). The consequence: +passing `--end=100000` explicitly on the command line is **indistinguishable** +from omitting it, so a config `end_time` would still override that particular +value. Any other `--end` value is honored as the override it is. This only bites +the exact default; pick any other number and precedence is unambiguous. + +**A model that sets its own end time still wins.** `end_time` is applied inside +`codes_mapping_setup()`, the setup call every model makes between loading the +config and running. A model is free to hardcode `g_tw_ts_end` *after* that call; +if it does, its value stands and the config `end_time` has no effect. This is +deliberate: the config front-end does not rewrite model `main()`s, so a model +that insists on its own end time keeps it. Most models never set their own end +time and honor the config value. The synthetic-traffic drivers +(`model-net-synthetic`, `model-net-synthetic-dragonfly-all`) treat their built-in +5-day span as a **fallback**: they apply it only when neither the config nor +`--end` set an end time, so a config `end_time` (and a command-line `--end`) is +honored. (This also fixed a prior bug where those two drivers unconditionally +overwrote `g_tw_ts_end`, silently ignoring `--end`.) + +## `pe_mem_factor` + +An **advanced** knob that most users should leave unset. ROSS pre-allocates a +pool of event structures per PE; `pe_mem_factor` scales that pool — the per-PE +event count is `pe_mem_factor × (LPs mapped to that PE)`, with a default factor of +`256`. Raise it if a run aborts having run out of events ("Out of events in +GVT"), which happens when many events are in flight at once. It is a positive +integer. (ROSS's `--extramem` is the *additive* term of the same pool — a flat +number of extra events per PE — so the two compose: total ≈ `pe_mem_factor × +LPs + extramem`.) + +--- + +# Dumping the resolved config + +Any run can print the **fully-resolved** config tree — every compiler-derived and +defaulted value included — so a result is traceable to one complete +configuration. Set the environment variable `CODES_RESOLVED_CONFIG_DUMP`: + +```bash +# write the resolved tree to a file +CODES_RESOLVED_CONFIG_DUMP=resolved.conf ./model-net-synthetic --sync=1 -- my-network.yaml + +# or to stdout +CODES_RESOLVED_CONFIG_DUMP=- ./model-net-synthetic --sync=1 -- my-network.yaml +``` + +The value is a file path, or `-` (equivalently `stdout`) for standard output. The +dump is written by **rank 0 only**, in the legacy `.conf` text format, at config +load time. It is **opt-in and off by default**, so it never perturbs a normal run. + +This is about traceability of runs, not a YAML feature: it works for a legacy +`.conf` run exactly the same way (the dump is taken from the in-memory config tree, +which both formats produce). For a compiled YAML config it is the way to see what +the friendly form expanded to — the derived `LPGROUPS`/`PARAMS`, including +`modelnet_order` and the shape-derived LP counts. + +--- + # Reusing config across files: `include:` A config can pull in other files with a top-level `include:` — a filename, or a diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 0278e0ec..40deea0a 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -43,6 +43,7 @@ list(APPEND SRCS workload/methods/codes-iomock-wrkld.c util/codes_mapping.c + util/codes-workload-config.c util/lp-type-lookup.c util/lp-io.c util/lp-msg.c @@ -101,6 +102,8 @@ list(APPEND SRCS # front-end is core, not an optional feature. modelconfig/config_compiler.h modelconfig/config_compiler.cxx + modelconfig/unit_convert.h + modelconfig/unit_convert.cxx modelconfig/config_emitter.h modelconfig/config_emitter.cxx modelconfig/yaml_configfile.h diff --git a/src/modelconfig/config_compiler.cxx b/src/modelconfig/config_compiler.cxx index d6f27171..177c31fc 100644 --- a/src/modelconfig/config_compiler.cxx +++ b/src/modelconfig/config_compiler.cxx @@ -6,6 +6,8 @@ #include "config_compiler.h" +#include "unit_convert.h" + #include #include @@ -98,13 +100,36 @@ long parse_int_strict(const std::string& s, const char* what) { using kv_list = std::vector>; +/* A parsed, unit-resolved workload description: what a node runs, as opposed to + * what a node is. Attached to a component (the inline `workload:` shortcut) or to + * a job (`jobs:`). Only the `synthetic` type is wired end-to-end here; other + * types are recognized (the block shape is understood) but rejected with a clear + * diagnostic, so a user keeps the legacy path for them. Size/time params are + * resolved to canonical units at parse time (payload_size -> bytes, arrival_time + * -> nanoseconds), since those conversions are model-independent; `traffic` + * stays a verbatim pattern name the model main maps to its own enum. */ +struct workload_block { + bool present = false; + std::string type; /* discriminator: "synthetic" is the only wired type */ + bool has_traffic = false; + std::string traffic; /* verbatim pattern name (uniform, nearest_neighbor, ...) */ + bool has_num_messages = false; + std::string num_messages; /* positive integer, verbatim */ + bool has_payload_size = false; + std::string payload_size; /* resolved to bytes */ + bool has_arrival_time = false; + std::string arrival_time; /* resolved to nanoseconds */ +}; + /* A custom component: a model paired with configured parameters. */ struct component { - std::string key; /* the components: key referenced by a topology */ - std::string model; /* ComponentModel name (nw-lp, ...) */ - std::string network; /* enumerated flat models: the NIC model a compute node + std::string key; /* the components: key referenced by a topology */ + std::string model; /* ComponentModel name (nw-lp, ...) */ + std::string network; /* enumerated flat models: the NIC model a compute node runs its workload over (added with the flat path) */ - kv_list params; /* scalar model params, raw text, in source order */ + kv_list params; /* scalar model params, raw text, in source order */ + workload_block workload; /* inline `workload:` shortcut (present only on the + topology's compute component) */ }; /* A per-link-class parameter block (e.g. dragonfly local/global/cn). */ @@ -126,6 +151,19 @@ struct fabric { std::string hosts_component; /* hosts.component: the per-terminal workload */ }; +/* One entry of the top-level `jobs:` block: a workload placed on a set of the + * topology's node slots. `ranks` is the job's size; placement is either the + * `contiguous` policy (a packed range assigned when the config is compiled) or + * an explicit node list (the escape hatch). Node indices are resolved and + * validated against the topology's total slot count at compile time. */ +struct job { + std::string id; + workload_block wl; + long ranks = 0; + bool contiguous = false; /* placement: { policy: contiguous } */ + std::vector nodes; /* placement: { nodes: [...] }; empty for contiguous */ +}; + struct friendly_config { std::vector components; bool parametric = false; @@ -135,6 +173,12 @@ struct friendly_config { std::string flat_component; /* the component every node runs */ long node_count = 0; /* number of nodes = repetitions */ + /* top-level `jobs:` block (multi-job workload placement). Mutually exclusive + * with an inline component `workload:` (a config uses one form or the other). + * Replaced wholesale by a later document that restates jobs:. */ + std::vector jobs; + bool has_jobs = false; + /* verbatim `sections:` blocks -- config a model reads directly (DIRECTOR, * surrogate, storage, ...), passed straight through to the compiled output. * Emitted after the topology sections. */ @@ -147,6 +191,23 @@ struct friendly_config { compiled_section explicit_lpgroups{"LPGROUPS", {}, {}}; compiled_section explicit_params{"PARAMS", {}, {}}; + /* run-level `simulation:` settings, already resolved to the PARAMS keys + * codes_mapping reads (end_time in nanoseconds, pe_mem_factor). Empty unless + * a simulation: block appeared; a later document overrides an earlier one key + * by key. Appended to PARAMS once the topology is compiled. */ + kv_list simulation; + + /* Record a resolved simulation setting, overriding any earlier value for the + * same key (last document wins, like components and sections). */ + void set_simulation(const std::string& name, std::string value) { + for (auto& kv : simulation) + if (kv.first == name) { + kv.second = std::move(value); + return; + } + simulation.emplace_back(name, std::move(value)); + } + const component* find_component(const std::string& k) const { for (const component& c : components) if (c.key == k) @@ -167,6 +228,121 @@ struct layout { long routers_per_rep; /* router/switch LP count per repetition */ }; +/* ------------------------------------------------------------------------- + * Per-parameter unit metadata. + * + * A dimensioned PARAMS key carries its quantity and the model's internal unit, + * expressed as how many canonical base units make up one internal unit (time + * base = ns, size base = bytes, bandwidth base = bytes/second). A unit-bearing + * value is normalized to the base unit (unit_convert) then divided by this scale + * to get the number the model reads; a bare number is emitted verbatim, so it + * already means the internal unit. See add_user_param. + * ---------------------------------------------------------------------- */ +struct param_unit { + const char* name; /* PARAMS key name */ + quantity kind; /* time / size / bandwidth */ + double internal_scale; /* canonical base units per one internal unit */ +}; + +/* Bandwidth internal-unit scales (bytes/second in one internal unit). CODES + * models do NOT agree on a bandwidth unit -- these come straight from each + * model's own byte-time arithmetic (see doc/dev/yaml-config.md). */ +constexpr double BW_GIB_S = 1024.0 * 1024.0 * 1024.0; /* GiB/s: bytes_to_ns() models */ +constexpr double BW_MIB_S = 1024.0 * 1024.0; /* MiB/s: simplenet net_bw_mbps */ +constexpr double BW_BYTES_NS = 1000.0 * 1000.0 * 1000.0; /* bytes/ns (= GB/s): fattree */ + +/* Time internal-unit scales (ns in one internal unit). */ +constexpr double T_NS = 1.0; +constexpr double T_US = 1000.0; /* the dragonfly QoS counting_* windows are read in us */ + +constexpr double SZ_BYTES = 1.0; /* every size param is read in bytes */ + +/* Dimensioned keys whose unit is the same in every model that reads them: byte + * counts and nanosecond delays. Looked up for every model in addition to its own + * table below, so they need not be repeated per model. */ +const param_unit common_params[] = { + {"packet_size", quantity::size, SZ_BYTES}, {"chunk_size", quantity::size, SZ_BYTES}, + {"message_size", quantity::size, SZ_BYTES}, {"credit_size", quantity::size, SZ_BYTES}, + {"buffer_size", quantity::size, SZ_BYTES}, {"vc_size", quantity::size, SZ_BYTES}, + {"cn_vc_size", quantity::size, SZ_BYTES}, {"local_vc_size", quantity::size, SZ_BYTES}, + {"global_vc_size", quantity::size, SZ_BYTES}, {"router_delay", quantity::time, T_NS}, + {"soft_delay", quantity::time, T_NS}, {"net_startup_ns", quantity::time, T_NS}, +}; + +/* Per-model dimensioned keys: the bandwidths (whose unit varies by model) and + * the microsecond QoS counting windows. Sizes and ns delays come from + * common_params above. */ +const param_unit dragonfly_units[] = { + {"local_bandwidth", quantity::bandwidth, BW_GIB_S}, + {"global_bandwidth", quantity::bandwidth, BW_GIB_S}, + {"cn_bandwidth", quantity::bandwidth, BW_GIB_S}, +}; +const param_unit dragonfly_dally_units[] = { + {"local_bandwidth", quantity::bandwidth, BW_GIB_S}, + {"global_bandwidth", quantity::bandwidth, BW_GIB_S}, + {"cn_bandwidth", quantity::bandwidth, BW_GIB_S}, + {"counting_start", quantity::time, T_US}, + {"counting_interval", quantity::time, T_US}, +}; +const param_unit dragonfly_plus_units[] = { + {"local_bandwidth", quantity::bandwidth, BW_GIB_S}, + {"global_bandwidth", quantity::bandwidth, BW_GIB_S}, + {"cn_bandwidth", quantity::bandwidth, BW_GIB_S}, + {"counting_start", quantity::time, T_US}, + {"counting_interval", quantity::time, T_US}, + {"counting_end", quantity::time, T_US}, +}; +const param_unit dragonfly_custom_units[] = { + {"local_bandwidth", quantity::bandwidth, BW_GIB_S}, + {"global_bandwidth", quantity::bandwidth, BW_GIB_S}, + {"cn_bandwidth", quantity::bandwidth, BW_GIB_S}, + {"counting_start", quantity::time, T_US}, + {"counting_interval", quantity::time, T_US}, +}; +const param_unit slimfly_units[] = { + {"local_bandwidth", quantity::bandwidth, BW_GIB_S}, + {"global_bandwidth", quantity::bandwidth, BW_GIB_S}, + {"cn_bandwidth", quantity::bandwidth, BW_GIB_S}, +}; +const param_unit torus_units[] = { + {"link_bandwidth", quantity::bandwidth, BW_GIB_S}, +}; +const param_unit express_mesh_units[] = { + {"link_bandwidth", quantity::bandwidth, BW_GIB_S}, + {"cn_bandwidth", quantity::bandwidth, BW_GIB_S}, +}; +/* fattree reads bandwidth as bytes/ns (1/bandwidth ns per byte), NOT the GiB/s + * convention the bytes_to_ns() models use -- a value means a different rate here. */ +const param_unit fattree_units[] = { + {"link_bandwidth", quantity::bandwidth, BW_BYTES_NS}, + {"cn_bandwidth", quantity::bandwidth, BW_BYTES_NS}, +}; + +/* simplenet's sole scalar rate; despite the "mbps" spelling the model reads it in + * MiB/s (rate_to_ns divides bytes by 1024*1024). simplep2p/loggp take their rates + * from files, so they have no scalar bandwidth key here. */ +const param_unit simplenet_units[] = { + {"net_bw_mbps", quantity::bandwidth, BW_MIB_S}, +}; + +/* count of a param_unit table declared as a C array */ +template constexpr size_t array_len(const T (&)[N]) { + return N; +} + +/* Find the unit metadata for a PARAMS key: the model's own table first, then the + * shared common table; nullptr if the key is not a known dimensioned param. */ +const param_unit* find_param_unit(const param_unit* model_units, size_t n_model, + const std::string& key) { + for (size_t i = 0; i < n_model; ++i) + if (key == model_units[i].name) + return &model_units[i]; + for (const param_unit& p : common_params) + if (key == p.name) + return &p; + return nullptr; +} + struct fabric_model { const char* name; /* friendly name used in fabric.model */ const char* terminal_lp; /* LPGROUPS lp-type name for the NIC/terminal */ @@ -175,6 +351,8 @@ struct fabric_model { const char* router_method; /* modelnet_order method for the router, or nullptr if the router is not a separate model-net method */ layout (*derive)(const kv_list& shape); /* shape -> LP layout */ + const param_unit* units; /* model's dimensioned-param unit table */ + size_t n_units; /* entries in `units` */ }; /* Look up a shape value by name, throwing if absent. The value is parsed @@ -371,19 +549,24 @@ layout derive_dragonfly_custom(const kv_list& shape) { const fabric_model fabric_models[] = { {"dragonfly", "modelnet_dragonfly", "modelnet_dragonfly_router", "dragonfly", - "dragonfly_router", derive_dragonfly}, + "dragonfly_router", derive_dragonfly, dragonfly_units, array_len(dragonfly_units)}, {"dragonfly-dally", "modelnet_dragonfly_dally", "modelnet_dragonfly_dally_router", - "dragonfly_dally", "dragonfly_dally_router", derive_dragonfly_dally}, - {"fattree", "modelnet_fattree", "fattree_switch", "fattree", nullptr, derive_fattree}, - {"torus", "modelnet_torus", nullptr, "torus", nullptr, derive_torus}, + "dragonfly_dally", "dragonfly_dally_router", derive_dragonfly_dally, dragonfly_dally_units, + array_len(dragonfly_dally_units)}, + {"fattree", "modelnet_fattree", "fattree_switch", "fattree", nullptr, derive_fattree, + fattree_units, array_len(fattree_units)}, + {"torus", "modelnet_torus", nullptr, "torus", nullptr, derive_torus, torus_units, + array_len(torus_units)}, {"express-mesh", "modelnet_express_mesh", "modelnet_express_mesh_router", "express_mesh", - "express_mesh_router", derive_express_mesh}, + "express_mesh_router", derive_express_mesh, express_mesh_units, array_len(express_mesh_units)}, {"slimfly", "modelnet_slimfly", "modelnet_slimfly_router", "slimfly", "slimfly_router", - derive_slimfly}, + derive_slimfly, slimfly_units, array_len(slimfly_units)}, {"dragonfly-plus", "modelnet_dragonfly_plus", "modelnet_dragonfly_plus_router", - "dragonfly_plus", "dragonfly_plus_router", derive_dragonfly_plus}, + "dragonfly_plus", "dragonfly_plus_router", derive_dragonfly_plus, dragonfly_plus_units, + array_len(dragonfly_plus_units)}, {"dragonfly-custom", "modelnet_dragonfly_custom", "modelnet_dragonfly_custom_router", - "dragonfly_custom", "dragonfly_custom_router", derive_dragonfly_custom}, + "dragonfly_custom", "dragonfly_custom_router", derive_dragonfly_custom, dragonfly_custom_units, + array_len(dragonfly_custom_units)}, }; const fabric_model* find_fabric_model(const std::string& name) { @@ -397,15 +580,19 @@ const fabric_model* find_fabric_model(const std::string& name) { * Maps a friendly network name to the LPGROUPS lp-type name and the * modelnet_order method the model registers. */ struct network_model { - const char* name; /* friendly name used in a component's network: field */ - const char* nic_lp; /* LPGROUPS lp-type name for the NIC */ - const char* method; /* modelnet_order method name */ + const char* name; /* friendly name used in a component's network: field */ + const char* nic_lp; /* LPGROUPS lp-type name for the NIC */ + const char* method; /* modelnet_order method name */ + const param_unit* units; /* model's dimensioned-param unit table */ + size_t n_units; /* entries in `units` */ }; const network_model network_models[] = { - {"simplenet", "modelnet_simplenet", "simplenet"}, - {"simplep2p", "modelnet_simplep2p", "simplep2p"}, - {"loggp", "modelnet_loggp", "loggp"}, + {"simplenet", "modelnet_simplenet", "simplenet", simplenet_units, array_len(simplenet_units)}, + /* simplep2p and loggp read their rates from files (paths pass through as + * non-numeric values), so they declare no scalar dimensioned param here. */ + {"simplep2p", "modelnet_simplep2p", "simplep2p", nullptr, 0}, + {"loggp", "modelnet_loggp", "loggp", nullptr, 0}, }; const network_model* find_network_model(const std::string& name) { @@ -420,6 +607,11 @@ const network_model* find_network_model(const std::string& name) { * unconsumed keys are errors, not silent drops). * ---------------------------------------------------------------------- */ +/* Parse + validate a `workload:` map (inline on a component, or a job's), unit- + * resolving size/time params. `where` names the owner in diagnostics. Defined + * with the other unit-resolving helpers below. */ +workload_block parse_workload(ryml::ConstNodeRef node, const std::string& where); + void parse_components(ryml::ConstNodeRef root, friendly_config& cfg) { if (!has(root, "components")) return; @@ -443,14 +635,16 @@ void parse_components(ryml::ConstNodeRef root, friendly_config& cfg) { "\": key \"type\" is reserved for a future schema version and " "is not accepted yet; remove it (the model is inferred from " "\"model:\")"); + else if (k == "workload") + c.workload = parse_workload(f, "component \"" + c.key + "\""); else if (f.is_keyval()) c.params.emplace_back(k, scalar(f)); else throw config_error("config error: component \"" + c.key + "\": unexpected block \"" + k + - "\"; a component takes a model, an optional network, and scalar " - "params (per-node data, edges and inline workloads are not " - "supported)"); + "\"; a component takes a model, an optional network, scalar " + "params, and an optional inline workload (per-node data and " + "edges are not supported)"); } /* include-merge: a later document's component overrides an earlier one of * the same name (within one document, keys are already unique). */ @@ -743,6 +937,272 @@ void parse_sections(ryml::ConstNodeRef root, friendly_config& cfg) { } } +/* ------------------------------------------------------------------------- + * Run-level settings (`simulation:`) + * + * The top-level `simulation:` block carries settings that belong to the run + * itself rather than to the topology or a model: the simulation end time and the + * ROSS per-PE event-pool factor. Each is resolved to the PARAMS key codes_mapping + * already reads (end_time in nanoseconds, pe_mem_factor), so no model or + * downstream reader changes. A legacy `.conf` user writes those PARAMS keys + * directly, exactly as before. + * ---------------------------------------------------------------------- */ + +/* forward-declared: defined with the other emit-phase helpers below. */ +const char* quantity_name(quantity q); + +/* Resolve a time-valued setting to nanoseconds (the unit every model uses). + * A bare number is already nanoseconds; a unit-bearing value must be a time and + * converts to ns. The value must be strictly positive. `what` names the setting + * in diagnostics. */ +std::string resolve_time_ns(const char* what, const std::string& raw) { + classified_value cv = classify_value(raw); + switch (cv.form) { + case value_form::bare_number: + if (cv.number <= 0.0) + throw config_error(std::string("config error: ") + what + " must be positive, got \"" + + raw + "\""); + return raw; /* already nanoseconds */ + case value_form::with_unit: + if (cv.kind != quantity::time) + throw config_error(std::string("config error: ") + what + + " takes a time value, but \"" + raw + "\" is a " + + quantity_name(cv.kind) + " value"); + if (cv.canonical <= 0.0) + throw config_error(std::string("config error: ") + what + " must be positive, got \"" + + raw + "\""); + return format_number(cv.canonical); /* canonical time base is ns */ + case value_form::plain: + case value_form::unknown_suffix: + break; + } + throw config_error(std::string("config error: ") + what + + " must be a time value: a bare number of nanoseconds or a value with a " + "time unit (ns/us/ms/s), got \"" + + raw + "\""); +} + +/* Resolve a size-valued setting to bytes (the unit every model's byte counter + * reads). A bare number is already bytes; a unit-bearing value must be a size -- + * a time or bandwidth unit is rejected -- and converts to bytes. The value must + * be strictly positive. `what` names the setting in diagnostics. */ +std::string resolve_size_bytes(const char* what, const std::string& raw) { + classified_value cv = classify_value(raw); + switch (cv.form) { + case value_form::bare_number: + if (cv.number <= 0.0) + throw config_error(std::string("config error: ") + what + " must be positive, got \"" + + raw + "\""); + return raw; /* already bytes */ + case value_form::with_unit: + if (cv.kind != quantity::size) + throw config_error(std::string("config error: ") + what + + " takes a size value, but \"" + raw + "\" is a " + + quantity_name(cv.kind) + " value"); + if (cv.canonical <= 0.0) + throw config_error(std::string("config error: ") + what + " must be positive, got \"" + + raw + "\""); + return format_number(cv.canonical); /* canonical size base is bytes */ + case value_form::plain: + case value_form::unknown_suffix: + break; + } + throw config_error(std::string("config error: ") + what + + " must be a size value: a bare number of bytes or a value with a size unit " + "(B/KiB/MiB/GiB), got \"" + + raw + "\""); +} + +/* Parse + validate a `workload:` map, unit-resolving its size/time params. Only + * the `synthetic` type is wired: the type is read first, and any other type is + * rejected up front with a clear "not yet configurable" diagnostic (so a + * dumpi/trace workload does not trip over a per-key error for a key this path + * doesn't model). `traffic` is kept verbatim -- the pattern-name -> enum meaning + * is model-specific, so the model main validates it. Unknown keys are rejected + * like everywhere else in the front-end. `where` names the owner in diagnostics + * (a component or a job). */ +workload_block parse_workload(ryml::ConstNodeRef node, const std::string& where) { + if (!node.is_map()) + throw config_error("config error: " + where + + ": \"workload\" must be a map (type, and type-specific settings)"); + if (!has(node, "type")) + throw config_error("config error: " + where + ": workload needs a \"type\" (synthetic)"); + workload_block wl; + wl.present = true; + wl.type = scalar(node["type"]); + if (wl.type != "synthetic") + throw config_error("config error: " + where + ": workload type \"" + wl.type + + "\" is not yet configurable from this format; only \"synthetic\" is " + "wired -- use the legacy .conf path for other workload types"); + for (ryml::ConstNodeRef c : node.children()) { + std::string k = key_of(c); + if (k == "type") { + continue; + } else if (k == "traffic") { + wl.has_traffic = true; + wl.traffic = scalar(c); + if (wl.traffic.empty()) + throw config_error("config error: " + where + + ": workload.traffic must be a non-empty pattern name"); + } else if (k == "num_messages") { + std::string what = where + " workload.num_messages"; + long v = parse_int_strict(scalar(c), what.c_str()); + if (v <= 0) + throw config_error("config error: " + where + + ": workload.num_messages must be a positive integer, got \"" + + scalar(c) + "\""); + wl.has_num_messages = true; + wl.num_messages = std::to_string(v); + } else if (k == "payload_size") { + std::string what = where + " workload.payload_size"; + wl.has_payload_size = true; + wl.payload_size = resolve_size_bytes(what.c_str(), scalar(c)); + } else if (k == "arrival_time") { + std::string what = where + " workload.arrival_time"; + wl.has_arrival_time = true; + wl.arrival_time = resolve_time_ns(what.c_str(), scalar(c)); + } else { + throw config_error("config error: " + where + ": workload: unexpected key \"" + k + + "\" (supported: type, traffic, num_messages, payload_size, " + "arrival_time)"); + } + } + return wl; +} + +/* Parse + validate the top-level `jobs:` block: a sequence of jobs, each a + * workload placed on a set of node slots. Structural validation happens here + * (unique non-empty ids, one workload, positive ranks, exactly one placement + * form); placement is resolved against the topology's slot count at compile time + * (compile_jobs), which is the first point the count is known. */ +void parse_jobs(ryml::ConstNodeRef root, friendly_config& cfg) { + if (!has(root, "jobs")) + return; + ryml::ConstNodeRef jobs = root["jobs"]; + if (!jobs.is_seq()) + throw config_error("config error: \"jobs\" must be a list of job blocks"); + /* a later document restating jobs: replaces the earlier list wholesale. */ + cfg.jobs.clear(); + cfg.has_jobs = true; + for (ryml::ConstNodeRef jnode : jobs.children()) { + if (!jnode.is_map()) + throw config_error("config error: jobs: each entry must be a block with id, workload, " + "ranks, and placement"); + job j; + bool has_ranks = false, has_placement = false; + for (ryml::ConstNodeRef f : jnode.children()) { + std::string k = key_of(f); + if (k == "id") { + j.id = scalar(f); + } else if (k == "workload") { + std::string where = + j.id.empty() ? std::string("jobs: entry") : ("job \"" + j.id + "\""); + j.wl = parse_workload(f, where); + } else if (k == "ranks") { + std::string where = j.id.empty() ? std::string("jobs: entry ranks") + : ("job \"" + j.id + "\" ranks"); + j.ranks = parse_int_strict(scalar(f), where.c_str()); + has_ranks = true; + } else if (k == "placement") { + has_placement = true; + if (!f.is_map()) + throw config_error("config error: job placement must be a block: either " + "{ policy: contiguous } or { nodes: [ ... ] }"); + bool has_policy = has(f, "policy"), has_nodes = has(f, "nodes"); + if (has_policy == has_nodes) + throw config_error("config error: job placement must set exactly one of " + "\"policy\" or \"nodes\""); + for (ryml::ConstNodeRef p : f.children()) { + std::string pk = key_of(p); + if (pk != "policy" && pk != "nodes") + throw config_error("config error: job placement: unexpected key \"" + pk + + "\" (only policy or nodes)"); + } + if (has_policy) { + std::string policy = scalar(f["policy"]); + if (policy != "contiguous") + throw config_error("config error: job placement policy \"" + policy + + "\" is not supported (only \"contiguous\")"); + j.contiguous = true; + } else { + ryml::ConstNodeRef nodes = f["nodes"]; + if (!nodes.is_seq() || nodes.num_children() == 0) + throw config_error("config error: job placement nodes must be a non-empty " + "list of node indices"); + for (ryml::ConstNodeRef n : nodes.children()) { + long idx = parse_int_strict(scalar(n), "job placement node index"); + if (idx < 0) + throw config_error("config error: job placement node index must be " + "non-negative, got \"" + + scalar(n) + "\""); + j.nodes.push_back(idx); + } + } + } else if (k == "qos") { + throw config_error("config error: job \"qos\" is recognized but not yet " + "configurable from this format; remove it (use the legacy path " + "for QoS)"); + } else { + throw config_error("config error: jobs: unexpected key \"" + k + + "\" (supported: id, workload, ranks, placement)"); + } + } + if (j.id.empty()) + throw config_error("config error: every job needs a non-empty \"id\""); + for (const job& e : cfg.jobs) + if (e.id == j.id) + throw config_error("config error: duplicate job id \"" + j.id + "\""); + if (!j.wl.present) + throw config_error("config error: job \"" + j.id + "\" needs a \"workload\""); + if (!has_ranks) + throw config_error("config error: job \"" + j.id + "\" needs a \"ranks\" count"); + if (j.ranks <= 0) + throw config_error("config error: job \"" + j.id + + "\" ranks must be a positive integer"); + if (!has_placement) + throw config_error("config error: job \"" + j.id + + "\" needs a \"placement\" (contiguous policy or a node list)"); + if (!j.contiguous && static_cast(j.nodes.size()) != j.ranks) + throw config_error("config error: job \"" + j.id + "\" ranks (" + + std::to_string(j.ranks) + + ") does not match its placement node " + "count (" + + std::to_string(j.nodes.size()) + ")"); + cfg.jobs.push_back(std::move(j)); + } +} + +/* Parse the top-level `simulation:` block, validating and resolving each setting + * into cfg.simulation (emitted into PARAMS after the topology compiles). Unknown + * keys are rejected, like everywhere else in the front-end. */ +void parse_simulation(ryml::ConstNodeRef root, friendly_config& cfg) { + if (!has(root, "simulation")) + return; + ryml::ConstNodeRef sim = root["simulation"]; + if (!sim.is_map()) + throw config_error("config error: \"simulation\" must be a map of run-level settings " + "(end_time, pe_mem_factor)"); + for (ryml::ConstNodeRef c : sim.children()) { + std::string k = key_of(c); + if (!c.is_keyval()) + throw config_error("config error: simulation: \"" + k + "\" must be a scalar value"); + if (k == "end_time") { + cfg.set_simulation("end_time", resolve_time_ns("simulation.end_time", scalar(c))); + } else if (k == "pe_mem_factor") { + std::string raw = scalar(c); + long v = parse_int_strict(raw, "simulation.pe_mem_factor"); + if (v <= 0) + throw config_error("config error: simulation.pe_mem_factor must be a positive " + "integer, got \"" + + raw + "\""); + cfg.set_simulation("pe_mem_factor", std::to_string(v)); + } else { + throw config_error("config error: simulation: unexpected key \"" + k + + "\" (supported: end_time, pe_mem_factor)"); + } + } +} + /* Parse one document with our throwing error handler installed (so ryml's own * parse errors route through config_error, exactly like our validation errors), * check it is a top-level map, and hand its root to `fn`. The parser and tree @@ -769,7 +1229,7 @@ void validate_toplevel_keys(ryml::ConstNodeRef root, bool is_base) { for (ryml::ConstNodeRef c : root.children()) { std::string k = key_of(c); if (k != "schema_version" && k != "components" && k != "topology" && k != "sections" && - k != "include") + k != "simulation" && k != "jobs" && k != "include") throw config_error("config error: unexpected top-level key \"" + k + "\""); if (k == "include" && is_base) throw config_error("config error: an included file cannot itself use \"include\" " @@ -820,14 +1280,131 @@ void merge_document(ryml::ConstNodeRef root, friendly_config& cfg) { parse_topology(root, cfg); } parse_sections(root, cfg); + parse_simulation(root, cfg); + parse_jobs(root, cfg); } /* ------------------------------------------------------------------------- * Compile: friendly IR -> compiled_config * ---------------------------------------------------------------------- */ -/* Compile a parametric fabric into LPGROUPS + PARAMS. */ -void compile_fabric(const friendly_config& cfg, compiled_config& out) { +/* PARAMS keys the compiler derives from the topology itself, rather than passing + * through from the user. Currently just modelnet_order (computed from the fabric/ + * network model registry); the array leaves room for more without touching the + * call sites. */ +const char* const derived_params_keys[] = {"modelnet_order"}; + +bool is_derived_param(const std::string& key) { + for (const char* k : derived_params_keys) + if (key == k) + return true; + return false; +} + +/* A model's dimensioned-param unit table, threaded from the model registry into + * PARAMS emission so unit conversion is per-model (link_bandwidth is GiB/s in + * torus but bytes/ns in fattree, so the same key resolves differently). */ +struct model_units { + const param_unit* tbl; + size_t n; +}; + +const char* quantity_name(quantity q) { + switch (q) { + case quantity::time: + return "time"; + case quantity::size: + return "size"; + case quantity::bandwidth: + return "bandwidth"; + } + return "value"; +} + +/* Resolve one raw PARAMS value for `key` against the model's unit table: + * + * - a value with a recognized unit is converted to the model's internal unit + * for that dimensioned key (rejecting a unit of the wrong quantity, e.g. a + * time on a size key, and a negative magnitude); + * - a bare number is emitted verbatim, so it already means the internal unit + * (identity for ns/bytes keys, pass-through for bandwidth, which has no safe + * universal default); + * - a non-numeric string (a name, path, enum) passes through untouched; + * - a unit-suffixed value on a knob the model can't classify is a trap -- the + * model would atof() it and silently read the bare number -- so it is + * rejected with a diagnostic naming the key. + * + * Trailing garbage after a number (an unrecognized suffix) is rejected on a + * dimensioned key but passed through on an unclassified one (a value like a + * "4,2,2" dim_length or a "5.dat" filename is not ours to reject). */ +std::string resolve_param_value(const model_units& units, const std::string& key, + const std::string& raw) { + const param_unit* pu = find_param_unit(units.tbl, units.n, key); + classified_value cv = classify_value(raw); + + if (pu) { + switch (cv.form) { + case value_form::plain: + case value_form::bare_number: + return raw; /* verbatim: already the model's internal unit */ + case value_form::with_unit: + if (cv.kind != pu->kind) + throw config_error("config error: parameter \"" + key + "\" takes a " + + quantity_name(pu->kind) + " value, but \"" + raw + "\" is a " + + quantity_name(cv.kind) + " value"); + if (cv.canonical < 0.0) + throw config_error("config error: parameter \"" + key + "\" value \"" + raw + + "\" must not be negative"); + return format_number(cv.canonical / pu->internal_scale); + case value_form::unknown_suffix: + throw config_error("config error: parameter \"" + key + "\" value \"" + raw + + "\" has an unrecognized unit; use ns/us/ms/s (time), " + "B/KiB/MiB/GiB (size), or a bit/byte rate like Gbps/GiBps " + "(bandwidth), or write a bare number in the model's internal unit"); + } + } else if (cv.form == value_form::with_unit) { + throw config_error( + "config error: parameter \"" + key + + "\" is a pass-through model knob that is not " + "unit-aware, so the unit-bearing value \"" + + raw + + "\" would be read as its bare number by the model; drop the unit and write the value " + "in the model's internal unit, or use a recognized dimensioned parameter"); + } + return raw; +} + +/* Append a user-supplied key to PARAMS, resolving any unit-bearing values + * against `units` (see resolve_param_value) and refusing to let it shadow a key + * the compiler derives itself. compile_fabric/compile_flat emit the derived keys + * (modelnet_order) first, then append the user's fabric/component params; because + * the config store returns the FIRST match for a name, a user key of the same + * name would land after the derived one and be silently ignored. The front-end + * never silently drops a key, so reject the collision here with a diagnostic. (An + * explicit-groups config derives no PARAMS at all and writes modelnet_order + * itself, so that path builds PARAMS directly and never goes through here.) */ +void add_user_param(compiled_section& params, const model_units& units, const std::string& key, + const std::vector& values) { + if (is_derived_param(key)) + throw config_error("config error: \"" + key + + "\" is derived by the compiler from the topology and cannot be set as a " + "model parameter; remove it (the compiler emits it for you)"); + std::vector resolved; + resolved.reserve(values.size()); + for (const std::string& v : values) + resolved.push_back(resolve_param_value(units, key, v)); + params.add_key(key, std::move(resolved)); +} + +void add_user_param(compiled_section& params, const model_units& units, const std::string& key, + const std::string& value) { + add_user_param(params, units, key, std::vector{value}); +} + +/* Compile a parametric fabric into LPGROUPS + PARAMS. Reports the total number + * of per-terminal workload slots (repetitions * terminals per rep) in + * total_slots, for workload/jobs placement validation. */ +void compile_fabric(const friendly_config& cfg, compiled_config& out, long& total_slots) { const fabric& fab = cfg.fab; const fabric_model* model = find_fabric_model(fab.model); @@ -853,6 +1430,7 @@ void compile_fabric(const friendly_config& cfg, compiled_config& out) { "component; a parametric fabric defines the network itself"); layout lay = model->derive(fab.shape); + total_slots = lay.repetitions * lay.terminals_per_rep; /* A backstop over every model's derivation: a shape that produces a * degenerate layout (e.g. num_groups: 0) must not reach codes_mapping. Each @@ -889,51 +1467,58 @@ void compile_fabric(const friendly_config& cfg, compiled_config& out) { else params.add_key("modelnet_order", std::vector{model->term_method}); + /* dimensioned params are resolved against this model's unit table (see + * resolve_param_value): a unit-bearing value converts to the model's internal + * unit, a bare number passes through. */ + const model_units units{model->units, model->n_units}; + /* shape parameters pass straight through (num_routers etc.). */ for (const auto& kv : fab.shape) - params.add_key(kv.first, kv.second); + add_user_param(params, units, kv.first, kv.second); /* per-link-class params become _ (local_bandwidth, ...). */ for (const link_class& cls : fab.links) for (const auto& kv : cls.params) - params.add_key(cls.name + "_" + kv.first, kv.second); + add_user_param(params, units, cls.name + "_" + kv.first, kv.second); /* routing.algorithm -> "routing"; any other routing.* passes through. */ for (const auto& kv : fab.routing) - params.add_key(kv.first == "algorithm" ? std::string("routing") : kv.first, kv.second); + add_user_param(params, units, kv.first == "algorithm" ? std::string("routing") : kv.first, + kv.second); /* connections.{intra,inter} -> the file-enumerated model's connection-file * keys; paths pass through verbatim (the model reads them relative to the * working directory). */ for (const auto& kv : fab.connections) { if (kv.first == "intra") - params.add_key("intra-group-connections", kv.second); + add_user_param(params, units, "intra-group-connections", kv.second); else if (kv.first == "inter") - params.add_key("inter-group-connections", kv.second); + add_user_param(params, units, "inter-group-connections", kv.second); else - params.add_key(kv.first, kv.second); + add_user_param(params, units, kv.first, kv.second); } /* remaining scalar fabric keys (packet_size, chunk_size, parity pass-through * knobs) map to PARAMS verbatim. */ for (const auto& kv : fab.extra) - params.add_key(kv.first, kv.second); + add_user_param(params, units, kv.first, kv.second); /* list-valued fabric keys (slimfly's generator_set_X / _X_prime) emit as * multi-value PARAMS, e.g. generator_set_X=("1","4"). */ for (const auto& kv : fab.extra_lists) - params.add_key(kv.first, kv.second); + add_user_param(params, units, kv.first, kv.second); /* the workload component's own params (if any) also land in PARAMS. */ for (const auto& kv : host->params) - params.add_key(kv.first, kv.second); + add_user_param(params, units, kv.first, kv.second); } /* Compile a flat (enumerated) network into LPGROUPS + PARAMS: `node_count` * peer compute nodes, each one repetition running the component's workload LP * over its NIC LP. simplep2p's link table stays referenced by path in the * component params; the friendly form supplies only the node count. */ -void compile_flat(const friendly_config& cfg, compiled_config& out) { +void compile_flat(const friendly_config& cfg, compiled_config& out, long& total_slots) { + total_slots = cfg.node_count; const component* comp = cfg.find_component(cfg.flat_component); if (!comp) throw config_error("config error: topology.component \"" + cfg.flat_component + @@ -964,8 +1549,168 @@ void compile_flat(const friendly_config& cfg, compiled_config& out) { * straight through. --- */ compiled_section& params = out.add_section("PARAMS"); params.add_key("modelnet_order", std::vector{net->method}); + const model_units units{net->units, net->n_units}; for (const auto& kv : comp->params) - params.add_key(kv.first, kv.second); + add_user_param(params, units, kv.first, kv.second); +} + +/* Append the resolved `simulation:` settings to PARAMS -- the section + * codes_mapping reads (PARAMS/end_time, PARAMS/pe_mem_factor). A model parameter + * of the same name (set on a component or fabric, or in an explicit-groups + * params: block) lands earlier in PARAMS and would win the config store's + * first-match lookup, silently shadowing the simulation setting; the front-end + * never drops a key silently, so the collision is rejected instead. Setting the + * key in only one place resolves it (a pass-through param alone, or the + * simulation: block alone, are both fine). */ +void apply_simulation(const friendly_config& cfg, compiled_config& out) { + if (cfg.simulation.empty()) + return; + compiled_section* params = nullptr; + for (compiled_section& s : out.sections) + if (s.name == "PARAMS") { + params = &s; + break; + } + /* every topology form emits a PARAMS section, so this is defensive. */ + if (!params) + params = &out.add_section("PARAMS"); + for (const auto& kv : cfg.simulation) { + for (const compiled_key& existing : params->keys) + if (existing.name == kv.first) + throw config_error("config error: \"" + kv.first + + "\" is set both in the simulation: block and as a model " + "parameter; set it in only one place"); + params->add_key(kv.first, kv.second); + } +} + +/* ------------------------------------------------------------------------- + * Workloads and jobs (`workload:` shortcut / `jobs:` block) + * + * What a node runs is separate from what a node is. The inline `workload:` on + * the topology's compute component is the common single-workload case; the + * top-level `jobs:` block is the explicit multi-job form. Both are mutually + * exclusive and only synthetic workloads are wired here. A synthetic workload's + * resolved params land in a dedicated WORKLOAD section (or a JOBS section, one + * subsection per job, for a richer placement) that a model main reads with the + * codes-workload-config helper -- config beats the model default, and the CLI + * beats config. The section is inert for a legacy .conf, which emits none. + * ---------------------------------------------------------------------- */ + +/* Emit a workload's resolved params as keys on `sec`, in a fixed order that is + * independent of the source key order, so equivalent configs (an inline + * shortcut and a single all-nodes job) compile to an identical tree. */ +void add_workload_keys(compiled_section& sec, const workload_block& wl) { + sec.add_key("type", wl.type); + if (wl.has_traffic) + sec.add_key("traffic", wl.traffic); + if (wl.has_num_messages) + sec.add_key("num_messages", wl.num_messages); + if (wl.has_payload_size) + sec.add_key("payload_size", wl.payload_size); + if (wl.has_arrival_time) + sec.add_key("arrival_time", wl.arrival_time); +} + +/* Emit a top-level WORKLOAD section for a single synthetic workload. The inline + * shortcut and a single all-nodes synthetic job both lower to exactly this. */ +void emit_workload_section(compiled_config& out, const workload_block& wl) { + compiled_section& sec = out.add_section("WORKLOAD"); + add_workload_keys(sec, wl); +} + +/* Compile the inline `workload:` shortcut or the `jobs:` block into the tree, + * resolving and validating placement against the topology's slot count. The two + * forms are mutually exclusive. `host_key` is the topology's compute component + * (a workload: may live only there). A single synthetic job that covers every + * slot contiguously lowers to the same WORKLOAD section as the inline shortcut, + * so the two spellings are tree-equal; any richer placement emits a JOBS section + * carrying each job's resolved workload and node allocation. */ +void compile_workload_jobs(const friendly_config& cfg, compiled_config& out, long total_slots, + const std::string& host_key) { + /* An inline workload: is meaningful only on the compute component; reject it + * anywhere else rather than silently ignoring it. */ + const workload_block* inline_wl = nullptr; + for (const component& c : cfg.components) { + if (!c.workload.present) + continue; + if (c.key != host_key) + throw config_error("config error: component \"" + c.key + + "\" carries a workload: but is not the topology's compute node; put " + "the workload on \"" + + host_key + + "\" (the component the topology runs) or use a jobs: block"); + inline_wl = &c.workload; + } + + if (inline_wl && cfg.has_jobs) + throw config_error("config error: a component sets an inline workload: and the config also " + "has a jobs: block; use one form or the other, not both"); + + if (inline_wl) { + emit_workload_section(out, *inline_wl); + return; + } + if (!cfg.has_jobs) + return; + + /* Resolve + validate placement across all jobs against the slot count: a + * contiguous job packs the next range, an explicit job takes its listed + * nodes; every slot is booked at most once and none runs past the end. */ + std::vector booked(static_cast(total_slots), 0); + long contig_cursor = 0; + long total_ranks = 0; + std::vector> alloc(cfg.jobs.size()); + for (size_t ji = 0; ji < cfg.jobs.size(); ++ji) { + const job& j = cfg.jobs[ji]; + total_ranks += j.ranks; + std::vector& nodes = alloc[ji]; + if (j.contiguous) { + for (long r = 0; r < j.ranks; ++r) + nodes.push_back(contig_cursor + r); + contig_cursor += j.ranks; + } else { + nodes = j.nodes; + } + for (long idx : nodes) { + if (idx >= total_slots) + throw config_error("config error: job \"" + j.id + "\" places a rank on node " + + std::to_string(idx) + ", but the topology has only " + + std::to_string(total_slots) + " node slots (0.." + + std::to_string(total_slots - 1) + ")"); + if (booked[static_cast(idx)]) + throw config_error("config error: job \"" + j.id + "\" double-books node " + + std::to_string(idx) + " (already assigned to another job)"); + booked[static_cast(idx)] = 1; + } + } + if (total_ranks > total_slots) + throw config_error("config error: jobs request " + std::to_string(total_ranks) + + " ranks but the topology has only " + std::to_string(total_slots) + + " node slots"); + + /* A single all-nodes synthetic job desugars to the inline-shortcut tree. */ + if (cfg.jobs.size() == 1 && cfg.jobs[0].contiguous && cfg.jobs[0].ranks == total_slots) { + emit_workload_section(out, cfg.jobs[0].wl); + return; + } + + /* Otherwise emit a JOBS section: a job count (a marker a reader keys on to + * detect the multi-job form) plus one subsection per job with its resolved + * workload params, rank count, and the node allocation it was placed on. */ + compiled_section& jobs_sec = out.add_section("JOBS"); + jobs_sec.add_key("num_jobs", std::to_string(cfg.jobs.size())); + for (size_t ji = 0; ji < cfg.jobs.size(); ++ji) { + const job& j = cfg.jobs[ji]; + compiled_section& js = jobs_sec.add_subsection(j.id); + add_workload_keys(js, j.wl); + js.add_key("ranks", std::to_string(j.ranks)); + std::vector nodev; + nodev.reserve(alloc[ji].size()); + for (long idx : alloc[ji]) + nodev.push_back(std::to_string(idx)); + js.add_key("nodes", std::move(nodev)); + } } } // namespace @@ -1013,14 +1758,36 @@ compiled_config compile(std::string_view main_doc, const std::vector a WORKLOAD (or JOBS) section following PARAMS. */ + if (!cfg.explicit_groups) + compile_workload_jobs(cfg, out, total_slots, host_key); /* verbatim `sections:` blocks follow the compiler-derived topology sections. */ for (compiled_section& s : cfg.passthrough) diff --git a/src/modelconfig/config_compiler.h b/src/modelconfig/config_compiler.h index f76d6439..a7fbf38a 100644 --- a/src/modelconfig/config_compiler.h +++ b/src/modelconfig/config_compiler.h @@ -52,8 +52,10 @@ struct compiled_key { /** One configuration section: named keys plus nested subsections. LPGROUPS * holds a MODELNET_GRP subsection; PARAMS is flat. Insertion order is - * preserved throughout, so a compiled config is byte-comparable to the `.conf` - * it replaces. */ + * preserved throughout, so emission is deterministic — but key order may + * differ from a hand-written `.conf` (the compiler emits derived keys like + * `modelnet_order` first). The config store looks everything up by name, so + * equivalence with a `.conf` is by name/value (see `cf_equal`), not bytes. */ struct compiled_section { std::string name; ///< section name (e.g. LPGROUPS, PARAMS) std::vector keys; ///< this section's keys, in insertion order diff --git a/src/modelconfig/configfile.c b/src/modelconfig/configfile.c index 19cb6123..3f09a047 100644 --- a/src/modelconfig/configfile.c +++ b/src/modelconfig/configfile.c @@ -18,147 +18,186 @@ #include #include "txt_configfile.h" -static int cf_equal_helper(struct ConfigVTable* h1, SectionHandle s1, struct ConfigVTable* h2, - SectionHandle s2) { - unsigned int sectionsize1; - unsigned int sectionsize2; - size_t count1; - size_t count2; - unsigned int i; - int ret = 1; - - cf_getSectionSize(h1, s1, §ionsize1); - cf_getSectionSize(h2, s2, §ionsize2); +/* Build "path/name" (or just "name" at the root) into out, for diagnostics. */ +static void cf_join_path(char* out, size_t outsz, const char* path, const char* name) { + if (path && path[0]) + snprintf(out, outsz, "%s/%s", path, name); + else + snprintf(out, outsz, "%s", name ? name : ""); +} - count1 = sectionsize1; - count2 = sectionsize2; +/* Does a section named `name` (case-insensitive, per the store) or a key named + * `name` (case-sensitive) exist directly under section `s`? Used to decide + * whether an entry seen only in the second tree is genuinely missing from the + * first (rather than already reported by the forward pass as a type mismatch). */ +static int cf_has_entry(struct ConfigVTable* h, SectionHandle s, const char* name) { + SectionHandle sec; + char** ptrs = NULL; + size_t n = 0; + if (cf_openSection(h, s, name, &sec) == 1) { + cf_closeSection(h, sec); + return 1; + } + if (cf_getMultiKey(h, s, name, &ptrs, &n) >= 0) { + for (size_t j = 0; j < n; ++j) + free(ptrs[j]); + free(ptrs); + return 1; + } + return 0; +} - if (count1 != count2) +/* Compare two value lists element-wise (a NULL value reads as ""). */ +static int cf_value_lists_equal(char** a, size_t na, char** b, size_t nb) { + if (na != nb) return 0; + for (size_t j = 0; j < na; ++j) + if (strcmp(a[j] ? a[j] : "", b[j] ? b[j] : "")) + return 0; + return 1; +} - SectionEntry entries1[sectionsize1]; - SectionEntry entries2[sectionsize2]; +static int cf_print_values(FILE* f, char** vals, size_t n) { + fputc('(', f); + for (size_t j = 0; j < n; ++j) + fprintf(f, "%s\"%s\"", j ? ", " : "", vals[j] ? vals[j] : ""); + return fputc(')', f); +} +/* + * Order-insensitive structural comparison of two config sections. + * + * The config store keys every lookup by name -- sections case-insensitively, + * keys case-sensitively -- and never by position, so two trees describe the same + * configuration when they hold the same named entries with the same values, + * regardless of the order those entries were written. Comparing a key by its full + * value *list* also subsumes the SE_KEY / SE_MULTIKEY split (which the store + * derives purely from the value count), and the *order of values within* a key is + * still significant and is checked -- so a list-valued key like modelnet_order is + * compared as an ordered tuple. + * + * `path` is the section path so far, for diagnostics. When `report` is non-NULL + * every divergence is written to it (all of them, not just the first) under its + * path, so a failing comparison names the exact section/key that diverged. When + * `report` is NULL the comparison short-circuits on the first divergence. + * Returns 1 if the two sections are equal, 0 otherwise. + */ +static int cf_section_equal(struct ConfigVTable* h1, SectionHandle s1, struct ConfigVTable* h2, + SectionHandle s2, const char* path, FILE* report) { + unsigned int size1 = 0; + unsigned int size2 = 0; + size_t count1; + size_t count2; + size_t i; + int eq = 1; + char child[1024]; + + cf_getSectionSize(h1, s1, &size1); + cf_getSectionSize(h2, s2, &size2); + count1 = size1; + count2 = size2; + + SectionEntry entries1[size1 ? size1 : 1]; + SectionEntry entries2[size2 ? size2 : 1]; cf_listSection(h1, s1, &entries1[0], &count1); cf_listSection(h2, s2, &entries2[0], &count2); + /* Forward pass: every entry in tree 1 must appear -- same kind, same value -- + * in tree 2. */ for (i = 0; i < count1; ++i) { - if (entries1[i].type == entries2[i].type) { - switch (entries1[i].type) { - case SE_SECTION: { - SectionHandle newsec1; - SectionHandle newsec2; - int corresponding = -1; - int j; - - /* find matching section in 2nd tree */ - j = 0; - while ((int)j < (int)count2) { - if (!strcmp(entries1[i].name, entries2[j].name)) { - corresponding = j; - break; - } - ++j; + const char* name = entries1[i].name; + cf_join_path(child, sizeof(child), path, name); + + if (entries1[i].type == SE_SECTION) { + SectionHandle c1; + SectionHandle c2; + if (cf_openSection(h2, s2, name, &c2) != 1) { + if (report) + fprintf(report, " %s: section only in first tree\n", child); + eq = 0; + } else { + cf_openSection(h1, s1, name, &c1); + if (!cf_section_equal(h1, c1, h2, c2, child, report)) + eq = 0; + cf_closeSection(h1, c1); + cf_closeSection(h2, c2); + } + } else { + /* key or multikey in tree 1 */ + char** p1 = NULL; + char** p2 = NULL; + size_t n1 = 0; + size_t n2 = 0; + size_t j; + int have2 = (cf_getMultiKey(h2, s2, name, &p2, &n2) >= 0); + cf_getMultiKey(h1, s1, name, &p1, &n1); /* present by construction */ + + if (!have2) { + SectionHandle sec2; + if (cf_openSection(h2, s2, name, &sec2) == 1) { + cf_closeSection(h2, sec2); + if (report) + fprintf(report, " %s: a key in the first tree, a section in the second\n", + child); + } else if (report) { + fprintf(report, " %s: key only in first tree\n", child); } - - if (corresponding < 0) { - /* missing section in 2nd tree */ - ret = 0; - break; + eq = 0; + } else if (!cf_value_lists_equal(p1, n1, p2, n2)) { + if (report) { + fprintf(report, " %s: values differ: ", child); + cf_print_values(report, p1, n1); + fprintf(report, " vs "); + cf_print_values(report, p2, n2); + fputc('\n', report); } - - if (entries1[i].type != entries2[j].type) { - ret = 0; /* name exists but is of different type */ - break; - } - - cf_openSection(h1, s1, entries1[i].name, &newsec1); - cf_openSection(h2, s2, entries2[j].name, &newsec2); - - ret = cf_equal_helper(h1, newsec1, h2, newsec2); - - cf_closeSection(h1, newsec1); - cf_closeSection(h2, newsec2); - - break; + eq = 0; } - case SE_KEY: { - char buf1[255]; - char buf2[255]; - buf1[0] = buf2[0] = 0; - if (cf_getKey(h1, s1, entries1[i].name, &buf1[0], sizeof(buf1)) <= 0) { - ret = 0; - break; - } - if (cf_getKey(h2, s2, entries1[i].name, &buf2[0], sizeof(buf2)) <= 0) { - ret = 0; - break; - } + for (j = 0; j < n1; ++j) + free(p1[j]); + for (j = 0; j < n2; ++j) + free(p2[j]); + free(p1); + free(p2); + } - if (strcmp(buf1, buf2)) { - ret = 0; /* strings not equal! */ - } + if (!eq && !report) + break; + } - break; - } - case SE_MULTIKEY: { - char** ptrs1; - size_t size1 = 0; - char** ptrs2; - size_t size2 = 0; - size_t j; - - do { - if (cf_getMultiKey(h1, s1, entries1[i].name, &ptrs1, &size1) <= 0) { - ret = 0; - break; - } - if (cf_getMultiKey(h2, s2, entries1[i].name, &ptrs2, &size2) <= 0) { - ret = 0; - break; - } - - if (size1 != size2) { - ret = 0; - break; - } - - for (j = 0; j < size1; ++j) { - if (strcmp(ptrs1[j], ptrs2[j])) { - ret = 0; - break; - } - } - } while (0); - - for (j = 0; j < size1; ++j) - free(ptrs1[j]); - for (j = 0; j < size2; ++j) - free(ptrs2[j]); - - free(ptrs1); - free(ptrs2); - } + /* Reverse pass: an entry present only in tree 2 (a type mismatch was already + * reported by the forward pass, so only flag names the first tree lacks + * entirely). */ + if (eq || report) { + for (i = 0; i < count2; ++i) { + const char* name = entries2[i].name; + if (!cf_has_entry(h1, s1, name)) { + cf_join_path(child, sizeof(child), path, name); + if (report) + fprintf(report, " %s: %s only in second tree\n", child, + entries2[i].type == SE_SECTION ? "section" : "key"); + eq = 0; + if (!report) + break; } - } else { - ret = 0; } - if (!ret) - break; } - /* cleanup */ - for (i = 0; i < count1; ++i) { + for (i = 0; i < count1; ++i) free((char*)entries1[i].name); + for (i = 0; i < count2; ++i) free((char*)entries2[i].name); - } - return ret; + return eq; +} + +int cf_equal_report(struct ConfigVTable* h1, struct ConfigVTable* h2, FILE* report) { + return cf_section_equal(h1, ROOT_SECTION, h2, ROOT_SECTION, "", report); } int cf_equal(struct ConfigVTable* h1, struct ConfigVTable* h2) { - return cf_equal_helper(h1, ROOT_SECTION, h2, ROOT_SECTION); + return cf_section_equal(h1, ROOT_SECTION, h2, ROOT_SECTION, "", NULL); } diff --git a/src/modelconfig/configuration.c b/src/modelconfig/configuration.c index 5e057a7e..b96dd305 100644 --- a/src/modelconfig/configuration.c +++ b/src/modelconfig/configuration.c @@ -38,6 +38,52 @@ ConfigHandle config; /* Global to hold LP configuration */ config_lpgroups_t lpconf; +/* Optional resolved-config dump. When the environment variable + * CODES_RESOLVED_CONFIG_DUMP is set, rank 0 writes the fully-resolved config tree + * -- every compiler-derived and defaulted value, in .conf text form -- so a run + * is traceable to one complete configuration. The value "-" (or "stdout") writes + * to stdout; any other value is a file path. It is opt-in and default-off, so + * existing tests and workflows (which diff lp-io output or grep stdout) are + * unaffected, and it works identically for a .conf or a compiled YAML config -- + * both are the same in-memory tree. A destination that can't be opened is a + * non-fatal warning; the run continues. */ +static void config_maybe_dump_resolved(ConfigHandle handle, MPI_Comm comm) { + const char* dest = getenv("CODES_RESOLVED_CONFIG_DUMP"); + if (!dest || !dest[0]) + return; + + int rank = 0; + MPI_Comm_rank(comm, &rank); + if (rank != 0) + return; + + int to_stdout = (strcmp(dest, "-") == 0 || strcmp(dest, "stdout") == 0); + char* err = NULL; + int rc; + + if (to_stdout) { + /* cf_dump writes the tree to stdout in .conf text form. */ + rc = cf_dump(handle, ROOT_SECTION, &err); + fflush(stdout); + } else { + FILE* out = fopen(dest, "w"); + if (!out) { + fprintf(stderr, "config warning: cannot open CODES_RESOLVED_CONFIG_DUMP=\"%s\": %s\n", + dest, strerror(errno)); + return; + } + /* the same writer cf_dump wraps, aimed at the chosen file. */ + rc = txtfile_writeConfig(handle, ROOT_SECTION, out, &err); + fclose(out); + } + + if (rc < 0) { + fprintf(stderr, "config warning: could not dump resolved config: %s\n", + err ? err : "(unknown error)"); + free(err); + } +} + int configuration_load(const char* filepath, MPI_Comm comm, ConfigHandle* handle) { MPI_File fh; MPI_Status status; @@ -140,6 +186,9 @@ int configuration_load(const char* filepath, MPI_Comm comm, ConfigHandle* handle rc = configuration_get_lpgroups(handle, "LPGROUPS", &lpconf); + /* Opt-in, default-off dump of the fully-resolved tree (rank 0 only). */ + config_maybe_dump_resolved(*handle, comm); + finalize: if (fh != MPI_FILE_NULL) MPI_File_close(&fh); diff --git a/src/modelconfig/unit_convert.cxx b/src/modelconfig/unit_convert.cxx new file mode 100644 index 00000000..0bf98e71 --- /dev/null +++ b/src/modelconfig/unit_convert.cxx @@ -0,0 +1,155 @@ +/* + * Copyright (C) 2013 University of Chicago. + * See COPYRIGHT notice in top-level directory. + * + */ + +#include "unit_convert.h" + +#include +#include +#include +#include +#include + +namespace codes { +namespace config { + +namespace { + +/* One accepted unit suffix: its exact spelling, the quantity it measures, and + * how many canonical base units make up one of it (time base = ns, size base = + * bytes, bandwidth base = bytes/second). Matching is exact and case-sensitive. */ +struct suffix_entry { + const char* suffix; + quantity kind; + double factor; +}; + +/* Powers used below, spelled out so the intent (binary IEC vs decimal SI) is + * obvious at each entry. */ +constexpr double KIB = 1024.0; +constexpr double MIB = 1024.0 * 1024.0; +constexpr double GIB = 1024.0 * 1024.0 * 1024.0; + +/* The full accepted suffix table. + * + * - time (base ns): ns, us (microseconds), ms, s. + * - size (base bytes): B, KiB/MiB/GiB (binary, 1024-based) and KB/MB/GB + * (decimal, 1000-based) -- both accepted; use the IEC forms to be exact. + * - bandwidth (base bytes/second): lowercase-b bit rates (bps, Kbps, Mbps, + * Gbps -- decimal, /8 to bytes) and uppercase-B byte rates in decimal (Bps, + * KBps, MBps, GBps) and binary (KiBps, MiBps, GiBps). + * + * The bit-vs-byte distinction rides on the case of 'b'/'B', so lookups must be + * case-sensitive. */ +const suffix_entry suffixes[] = { + /* time -> nanoseconds */ + {"ns", quantity::time, 1.0}, + {"us", quantity::time, 1.0e3}, + {"ms", quantity::time, 1.0e6}, + {"s", quantity::time, 1.0e9}, + + /* size -> bytes (binary IEC) */ + {"B", quantity::size, 1.0}, + {"KiB", quantity::size, KIB}, + {"MiB", quantity::size, MIB}, + {"GiB", quantity::size, GIB}, + /* size -> bytes (decimal SI) */ + {"KB", quantity::size, 1.0e3}, + {"MB", quantity::size, 1.0e6}, + {"GB", quantity::size, 1.0e9}, + + /* bandwidth -> bytes/second: bit rates (decimal, /8) */ + {"bps", quantity::bandwidth, 1.0 / 8.0}, + {"Kbps", quantity::bandwidth, 1.0e3 / 8.0}, + {"Mbps", quantity::bandwidth, 1.0e6 / 8.0}, + {"Gbps", quantity::bandwidth, 1.0e9 / 8.0}, + /* bandwidth -> bytes/second: byte rates (decimal) */ + {"Bps", quantity::bandwidth, 1.0}, + {"KBps", quantity::bandwidth, 1.0e3}, + {"MBps", quantity::bandwidth, 1.0e6}, + {"GBps", quantity::bandwidth, 1.0e9}, + /* bandwidth -> bytes/second: byte rates (binary IEC) */ + {"KiBps", quantity::bandwidth, KIB}, + {"MiBps", quantity::bandwidth, MIB}, + {"GiBps", quantity::bandwidth, GIB}, +}; + +} // namespace + +classified_value classify_value(std::string_view text) { + classified_value r; + + /* trim surrounding ASCII whitespace */ + size_t b = 0, e = text.size(); + while (b < e && std::isspace(static_cast(text[b]))) + ++b; + while (e > b && std::isspace(static_cast(text[e - 1]))) + --e; + std::string s(text.substr(b, e - b)); + if (s.empty()) { + r.form = value_form::plain; + return r; + } + + /* parse a leading base-10 number (strtod also accepts scientific notation, + * which we treat as a bare number since a model's reader accepts it too) */ + char* end = nullptr; + double num = std::strtod(s.c_str(), &end); + if (end == s.c_str()) { + /* no number at the front: a name, path, or enum -- not a quantity */ + r.form = value_form::plain; + return r; + } + r.number = num; + + std::string rest(end); /* everything after the number */ + if (rest.empty()) { + r.form = value_form::bare_number; + return r; + } + + for (const suffix_entry& se : suffixes) { + if (rest == se.suffix) { + r.form = value_form::with_unit; + r.kind = se.kind; + r.canonical = num * se.factor; + return r; + } + } + + /* a number followed by text that is not a recognized unit */ + r.form = value_form::unknown_suffix; + return r; +} + +std::string format_number(double v) { + char buf[64]; + + /* An integer-valued result is emitted without a decimal point. The bound + * keeps us within the range where a double represents every integer exactly, + * so "%.0f" is lossless; realistic converted values sit far below it. */ + if (std::isfinite(v) && std::fabs(v) < 1.0e15 && v == std::floor(v)) { + std::snprintf(buf, sizeof(buf), "%.0f", v); + return std::string(buf); + } + + /* Otherwise emit the shortest fixed-notation decimal that round-trips to the + * same double. The minimal precision that round-trips never leaves a trailing + * zero, so the result is already tidy. Fixed notation (never %g/%e) keeps an + * exponent out of the string a C reader will atof(). */ + for (int prec = 1; prec <= 17; ++prec) { + std::snprintf(buf, sizeof(buf), "%.*f", prec, v); + if (std::strtod(buf, nullptr) == v) + return std::string(buf); + } + + /* Fallback for an extreme magnitude no config conversion produces: the + * shortest round-tripping form, which may use an exponent. */ + std::snprintf(buf, sizeof(buf), "%.17g", v); + return std::string(buf); +} + +} // namespace config +} // namespace codes diff --git a/src/modelconfig/unit_convert.h b/src/modelconfig/unit_convert.h new file mode 100644 index 00000000..4ae29fd2 --- /dev/null +++ b/src/modelconfig/unit_convert.h @@ -0,0 +1,90 @@ +/* + * Copyright (C) 2013 University of Chicago. + * See COPYRIGHT notice in top-level directory. + * + */ + +#ifndef SRC_MODELCONFIG_UNIT_CONVERT_H +#define SRC_MODELCONFIG_UNIT_CONVERT_H + +/** + * @file unit_convert.h + * + * Pure parsing/conversion of unit-bearing configuration values, split out of the + * compiler core so it is unit-testable on its own. Like config_compiler it has + * **no** dependency on ROSS, MPI, or `abort`. + * + * A dimensioned config value may be written either as a bare number in the + * target model's internal unit or as a unit-bearing string (`"5ms"`, `"2KiB"`, + * `"100Gbps"`). This module classifies a raw value string against a fixed unit + * grammar and, for a recognized unit, normalizes it to a canonical base unit for + * its quantity: + * + * - time -> nanoseconds + * - size -> bytes + * - bandwidth -> bytes per second + * + * The compiler's per-model registry then divides that canonical value by the + * model's internal-unit scale to get the number the model actually reads. This + * module does not know model internals; it only parses and normalizes. + */ + +#include "config_compiler.h" /* config_error */ + +#include +#include + +namespace codes { +namespace config { + +/** The physical dimension of a configuration value. */ +enum class quantity { + time, ///< a duration; canonical base unit is the nanosecond + size, ///< a byte count; canonical base unit is the byte + bandwidth, ///< a data rate; canonical base unit is the byte per second +}; + +/** How a raw value string classifies against the unit grammar. */ +enum class value_form { + plain, ///< no leading number: a name, path, enum, or list value -- not a quantity + bare_number, ///< a number with no unit suffix + with_unit, ///< a number followed by a recognized unit suffix + unknown_suffix, ///< a number followed by unrecognized trailing text +}; + +/** + * The result of classify_value(). For @ref value_form::with_unit, @ref kind and + * @ref canonical are meaningful: @ref canonical is the value expressed in the + * canonical base unit for its @ref quantity (time -> ns, size -> bytes, + * bandwidth -> bytes/second). @ref number is the leading numeric part and is + * meaningful for every form except @ref value_form::plain. + */ +struct classified_value { + value_form form = value_form::plain; + quantity kind = quantity::time; ///< valid only when form == with_unit + double canonical = 0.0; ///< valid only when form == with_unit; base-unit value + double number = 0.0; ///< the leading numeric part (all forms but plain) +}; + +/** + * Classify a raw value string against the unit grammar. Pure and total: it never + * throws and never touches ROSS. Surrounding ASCII whitespace is ignored. The + * suffix match is exact and case-sensitive -- the bit/byte distinction is carried + * by case (`Gbps` gigabit/s vs `GBps` gigabyte/s), so it must be. See the .cxx + * for the accepted suffix table. + */ +classified_value classify_value(std::string_view text); + +/** + * Format a converted numeric value as a decimal string every model's C reader + * parses: an integer-valued result carries no decimal point (`"2048"`), and a + * fractional result is the shortest fixed-notation decimal that round-trips to + * the same double -- never scientific notation, so a model's atof/strtol reader + * sees no exponent surprise. + */ +std::string format_number(double v); + +} // namespace config +} // namespace codes + +#endif diff --git a/src/network-workloads/model-net-synthetic-dragonfly-all.c b/src/network-workloads/model-net-synthetic-dragonfly-all.c index 2d23aad6..fb9c8a4a 100644 --- a/src/network-workloads/model-net-synthetic-dragonfly-all.c +++ b/src/network-workloads/model-net-synthetic-dragonfly-all.c @@ -10,10 +10,9 @@ #include "codes/configuration.h" #include "codes/lp-type-lookup.h" #include "codes/congestion-controller-core.h" +#include "codes/codes-workload-config.h" #include -#define G_TW_END_OPT_OVERRIDE 1 - static int net_id = 0; static int traffic = 1; static double warm_up_time = 0.0; @@ -87,6 +86,17 @@ enum TRAFFIC { }; +/* friendly workload.traffic names -> this model's traffic enum, for the + * codes-workload-config helper. */ +static const struct codes_workload_traffic_name traffic_names[] = { + {"uniform", UNIFORM}, + {"rand_perm", RAND_PERM}, + {"nearest_group", NEAREST_GROUP}, + {"nearest_neighbor", NEAREST_NEIGHBOR}, + {"random_other_group", RANDOM_OTHER_GROUP}, + {NULL, 0}, +}; + struct svr_state { int msg_sent_count; /* requests sent */ int warm_msg_sent_count; @@ -653,6 +663,13 @@ int main(int argc, char** argv) { int num_nets; int* net_ids; + /* capture the option defaults before tw_init parses the command line, so the + * config-vs-CLI helper can tell whether the command line overrode each. */ + int traffic_default = traffic; + int num_msgs_default = num_msgs; + int payload_sz_default = PAYLOAD_SZ; + double arrival_time_default = arrival_time; + tw_opt_add(app_opt); tw_init(&argc, &argv); @@ -681,15 +698,27 @@ int main(int argc, char** argv) { codes_mapping_setup(); + /* apply synthetic workload params from a YAML workload:/jobs: config; the + * command line still wins over the config for each. Inert for a legacy + * .conf, which carries no WORKLOAD section. */ + codes_workload_config_apply_traffic(&traffic, traffic_default, traffic_names); + codes_workload_config_apply_int("num_messages", &num_msgs, num_msgs_default); + codes_workload_config_apply_int("payload_size", &PAYLOAD_SZ, payload_sz_default); + codes_workload_config_apply_double("arrival_time", &arrival_time, arrival_time_default); + codes_workload_config_check_unsupported_jobs("model-net-synthetic-dragonfly-all"); + net_ids = model_net_configure(&num_nets); //assert(num_nets==1); net_id = *net_ids; free(net_ids); - if (G_TW_END_OPT_OVERRIDE) { - /* 5 days of simulation time */ + /* Default run length (5 days), applied only when neither the config + * (PARAMS/end_time, honored by codes_mapping_setup) nor the command line + * (--end) set an end time. ROSS's compiled-in default is 100000.0, so an + * unchanged value means none was requested. This also lets --end take + * effect, which the previous unconditional override silently ignored. */ + if (g_tw_ts_end == 100000.0) g_tw_ts_end = s_to_ns(5 * 24 * 60 * 60); - } model_net_enable_sampling(sampling_interval, sampling_end_time); if (!(net_id == DRAGONFLY_DALLY || net_id == DRAGONFLY_PLUS || net_id == DRAGONFLY_CUSTOM || diff --git a/src/network-workloads/model-net-synthetic-fattree.c b/src/network-workloads/model-net-synthetic-fattree.c index e0753418..ea8f83c6 100644 --- a/src/network-workloads/model-net-synthetic-fattree.c +++ b/src/network-workloads/model-net-synthetic-fattree.c @@ -16,6 +16,7 @@ #include "codes/codes_mapping.h" #include "codes/configuration.h" #include "codes/lp-type-lookup.h" +#include "codes/codes-workload-config.h" #define PAYLOAD_SZ 512 @@ -75,6 +76,16 @@ enum TRAFFIC { 4 /* sends message to the next node (potentially connected to the same router) */ }; +/* friendly workload.traffic names -> this model's traffic enum, for the + * codes-workload-config helper. */ +static const struct codes_workload_traffic_name traffic_names[] = { + {"uniform", UNIFORM}, + {"bisection", BISECTION}, + {"nearest_group", NEAREST_GROUP}, + {"nearest_neighbor", NEAREST_NEIGHBOR}, + {NULL, 0}, +}; + struct svr_state { int msg_sent_count; /* requests sent */ int msg_recvd_count; /* requests recvd */ @@ -342,6 +353,11 @@ int main(int argc, char** argv) { lp_io_handle handle; + /* capture the option defaults before tw_init parses the command line, so the + * config-vs-CLI helper can tell whether the command line overrode each. */ + int traffic_default = traffic; + double arrival_time_default = arrival_time; + tw_opt_add(app_opt); tw_init(&argc, &argv); @@ -372,6 +388,12 @@ int main(int argc, char** argv) { codes_mapping_setup(); + /* apply synthetic workload params from a YAML workload:/jobs: config; the + * command line still wins over the config for each. Inert for a legacy + * .conf, which carries no WORKLOAD section. */ + codes_workload_config_apply_traffic(&traffic, traffic_default, traffic_names); + codes_workload_config_apply_double("arrival_time", &arrival_time, arrival_time_default); + codes_workload_config_check_unsupported_jobs("model-net-synthetic-fattree"); net_ids = model_net_configure(&num_nets); //assert(num_nets==1); diff --git a/src/network-workloads/model-net-synthetic-slimfly.c b/src/network-workloads/model-net-synthetic-slimfly.c index 3178cdbd..90a220b2 100644 --- a/src/network-workloads/model-net-synthetic-slimfly.c +++ b/src/network-workloads/model-net-synthetic-slimfly.c @@ -15,6 +15,7 @@ #include "codes/codes_mapping.h" #include "codes/configuration.h" #include "codes/lp-type-lookup.h" +#include "codes/codes-workload-config.h" #define LP_CONFIG_NM (model_net_lp_config_names[SLIMFLY]) @@ -94,6 +95,20 @@ enum TRAFFIC { BISECTION = 8 /* sends messages between paired nodes in opposing bisection partitions */ }; +/* friendly workload.traffic names -> this model's traffic enum, for the + * codes-workload-config helper. */ +static const struct codes_workload_traffic_name traffic_names[] = { + {"uniform", UNIFORM}, + {"worst_case", WORST_CASE}, + {"nearest_neighbor_1d", NEAREST_NEIGHBOR_1D}, + {"nearest_neighbor_2d", NEAREST_NEIGHBOR_2D}, + {"nearest_neighbor_3d", NEAREST_NEIGHBOR_3D}, + {"gather", GATHER}, + {"scatter", SCATTER}, + {"bisection", BISECTION}, + {NULL, 0}, +}; + struct svr_state { int msg_sent_count; /* requests sent */ int msg_recvd_count; /* requests recvd */ @@ -540,6 +555,13 @@ int main(int argc, char** argv) { int num_nets; int* net_ids; + /* capture the option defaults before tw_init parses the command line, so the + * config-vs-CLI helper can tell whether the command line overrode each. */ + int traffic_default = traffic; + int num_msgs_default = num_msgs; + int payload_size_default = payload_size; + double arrival_time_default = arrival_time; + tw_opt_add(app_opt); tw_init(&argc, &argv); #ifdef USE_RDAMARIS @@ -564,6 +586,17 @@ int main(int argc, char** argv) { svr_register_model_types(); codes_mapping_setup(); + + /* apply synthetic workload params from a YAML workload:/jobs: config; the + * command line still wins over the config for each. Inert for a legacy + * .conf, which carries no WORKLOAD section. Runs before the WORST_CASE + * setup below so a config-set traffic pattern is already in effect. */ + codes_workload_config_apply_traffic(&traffic, traffic_default, traffic_names); + codes_workload_config_apply_int("num_messages", &num_msgs, num_msgs_default); + codes_workload_config_apply_int("payload_size", &payload_size, payload_size_default); + codes_workload_config_apply_double("arrival_time", &arrival_time, arrival_time_default); + codes_workload_config_check_unsupported_jobs("model-net-synthetic-slimfly"); + net_ids = model_net_configure(&num_nets); // assert(num_nets==1); net_id = *net_ids; diff --git a/src/network-workloads/model-net-synthetic.c b/src/network-workloads/model-net-synthetic.c index 521bdaf4..7847f94a 100644 --- a/src/network-workloads/model-net-synthetic.c +++ b/src/network-workloads/model-net-synthetic.c @@ -16,6 +16,7 @@ #include "codes/configuration.h" #include "codes/lp-type-lookup.h" #include "codes/net/dragonfly.h" +#include "codes/codes-workload-config.h" #define PAYLOAD_SZ 2048 @@ -61,6 +62,15 @@ enum TRAFFIC { 3 /* sends message to the next node (potentially connected to the same router) */ }; +/* friendly workload.traffic names -> this model's traffic enum, for the + * codes-workload-config helper. */ +static const struct codes_workload_traffic_name traffic_names[] = { + {"uniform", UNIFORM}, + {"nearest_group", NEAREST_GROUP}, + {"nearest_neighbor", NEAREST_NEIGHBOR}, + {NULL, 0}, +}; + struct svr_state { int msg_sent_count; /* requests sent */ int msg_recvd_count; /* requests recvd */ @@ -316,6 +326,12 @@ int main(int argc, char** argv) { int num_nets; int* net_ids; + /* capture the option defaults before tw_init parses the command line, so the + * config-vs-CLI helper can tell whether the command line overrode each. */ + int traffic_default = traffic; + int num_msgs_default = num_msgs; + double arrival_time_default = arrival_time; + tw_opt_add(app_opt); tw_init(&argc, &argv); #ifdef USE_RDAMARIS @@ -342,13 +358,26 @@ int main(int argc, char** argv) { codes_mapping_setup(); + /* apply synthetic workload params from a YAML workload:/jobs: config; the + * command line still wins over the config for each. Inert for a legacy + * .conf, which carries no WORKLOAD section. */ + codes_workload_config_apply_traffic(&traffic, traffic_default, traffic_names); + codes_workload_config_apply_int("num_messages", &num_msgs, num_msgs_default); + codes_workload_config_apply_double("arrival_time", &arrival_time, arrival_time_default); + codes_workload_config_check_unsupported_jobs("model-net-synthetic"); + net_ids = model_net_configure(&num_nets); //assert(num_nets==1); net_id = *net_ids; free(net_ids); - /* 5 days of simulation time */ - g_tw_ts_end = s_to_ns(5 * 24 * 60 * 60); + /* Default run length (5 days), applied only when neither the config + * (PARAMS/end_time, honored by codes_mapping_setup) nor the command line + * (--end) set an end time. ROSS's compiled-in default is 100000.0, so an + * unchanged value means none was requested. This also lets --end take + * effect, which the previous unconditional assignment silently ignored. */ + if (g_tw_ts_end == 100000.0) + g_tw_ts_end = s_to_ns(5 * 24 * 60 * 60); model_net_enable_sampling(sampling_interval, sampling_end_time); if (net_id != DRAGONFLY) { diff --git a/src/util/codes-workload-config.c b/src/util/codes-workload-config.c new file mode 100644 index 00000000..f39ac792 --- /dev/null +++ b/src/util/codes-workload-config.c @@ -0,0 +1,73 @@ +/* + * Copyright (C) 2013 University of Chicago. + * See COPYRIGHT notice in top-level directory. + * + */ + +#include "codes/codes-workload-config.h" + +#include "codes/configuration.h" + +#include + +#include + +/* the section the YAML front-end emits synthetic workload params into. */ +#define CODES_WORKLOAD_SECTION "WORKLOAD" + +/* the section the compiler emits for an explicit multi-job placement; its + * num_jobs marker key is how a reader detects the multi-job form. */ +#define CODES_JOBS_SECTION "JOBS" + +int codes_workload_config_present(void) { + char buf[CONFIGURATION_MAX_NAME]; + return configuration_get_value(&config, CODES_WORKLOAD_SECTION, "type", NULL, buf, + sizeof(buf)) > 0; +} + +void codes_workload_config_check_unsupported_jobs(const char* model_name) { + int num_jobs; + if (configuration_get_value_int(&config, CODES_JOBS_SECTION, "num_jobs", NULL, &num_jobs) != 0) + return; /* no JOBS section: single-workload or legacy config */ + tw_error(TW_LOC, + "config error: this config places %d jobs (a multi-job placement), but %s runs a " + "single global synthetic workload and cannot execute per-job placement yet; use a " + "single workload: (or a one-job jobs: entry on all nodes), or model-net-mpi-replay " + "for multi-job runs", + num_jobs, model_name); +} + +void codes_workload_config_apply_int(const char* key, int* val, int cli_default) { + /* a value other than the registered default means the command line set it. */ + if (*val != cli_default) + return; + int v; + if (configuration_get_value_int(&config, CODES_WORKLOAD_SECTION, key, NULL, &v) == 0) + *val = v; +} + +void codes_workload_config_apply_double(const char* key, double* val, double cli_default) { + if (*val != cli_default) + return; + double v; + if (configuration_get_value_double(&config, CODES_WORKLOAD_SECTION, key, NULL, &v) == 0) + *val = v; +} + +void codes_workload_config_apply_traffic(int* val, int cli_default, + const struct codes_workload_traffic_name* names) { + if (*val != cli_default) + return; + char buf[CONFIGURATION_MAX_NAME]; + if (configuration_get_value(&config, CODES_WORKLOAD_SECTION, "traffic", NULL, buf, + sizeof(buf)) <= 0) + return; + for (const struct codes_workload_traffic_name* n = names; n->name != NULL; ++n) { + if (strcmp(n->name, buf) == 0) { + *val = n->value; + return; + } + } + tw_error(TW_LOC, "config error: workload traffic pattern \"%s\" is not supported by this model", + buf); +} diff --git a/src/util/codes_mapping.c b/src/util/codes_mapping.c index 3e4962c4..ad6e8a96 100644 --- a/src/util/codes_mapping.c +++ b/src/util/codes_mapping.c @@ -524,6 +524,25 @@ void codes_mapping_setup_with_seed_offset(int offset) { mem_factor = mem_factor_conf; g_tw_events_per_pe = mem_factor * codes_mapping_get_lps_for_pe(); + + // A config-provided simulation end time (PARAMS/end_time, in nanoseconds) + // sets g_tw_ts_end, but only when the command line did not. ROSS's --end + // option writes g_tw_ts_end directly and records no "was explicitly set" + // flag, so we compare against ROSS's compiled-in default (100000.0, see + // ross-global.c): an unchanged value means no --end was given and the config + // value takes over; any other value is treated as a command-line override and + // is left untouched. Passing --end=100000 explicitly is therefore + // indistinguishable from omitting it. A model that assigns g_tw_ts_end after + // this call (some synthetic mains hardcode their own) keeps its value; a model + // that never sets it honors the config. The key rides in PARAMS, so a legacy + // .conf can carry it too. + const double ross_default_ts_end = 100000.0; + double end_time_conf; + int end_rc = + configuration_get_value_double(&config, "PARAMS", "end_time", NULL, &end_time_conf); + if (end_rc == 0 && end_time_conf > 0.0 && g_tw_ts_end == ross_default_ts_end) + g_tw_ts_end = end_time_conf; + configuration_get_value_int(&config, "PARAMS", "message_size", NULL, &message_size); if (!message_size) { message_size = 256; diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index e7eb2b2d..ea82043c 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -16,6 +16,7 @@ option(CODES_ENABLE_ZMQML_HYBRID_TESTS add_executable(codes-config-compiler-test codes-config-compiler-test.cxx ${CMAKE_SOURCE_DIR}/src/modelconfig/config_compiler.cxx + ${CMAKE_SOURCE_DIR}/src/modelconfig/unit_convert.cxx ${CODES_RYML_IMPL_TU}) target_include_directories(codes-config-compiler-test SYSTEM PRIVATE ${CODES_RYML_INCLUDE_DIRS}) target_include_directories(codes-config-compiler-test PRIVATE ${CMAKE_SOURCE_DIR}/src/modelconfig) @@ -172,6 +173,29 @@ function(codes_add_run_test) endif() endfunction() +# codes_add_config_tree_test — assert a .conf and its YAML twin compile to the +# *same* config tree. +# +# Loads both files through config-tree-equivalence-test (the production loader + +# cf_equal) and passes iff every section, key and value matches. It runs no model, +# so it is far cheaper than the behavioral equivalence tests and checks every key +# rather than a marker/lp-io side effect -- catching defaulting/derivation drift a +# short run can mask. Use it *alongside* the lp-io / marker equivalence test for a +# twin, not instead of it: this proves the trees are identical, those prove the +# model behaves identically. +# +# codes_add_config_tree_test(NAME CONFIG_A CONFIG_B ) +# +# CONFIG_A/CONFIG_B are absolute paths (the runner executes in a temp dir). +function(codes_add_config_tree_test) + cmake_parse_arguments(T "" "NAME;CONFIG_A;CONFIG_B" "" ${ARGN}) + if(NOT T_NAME OR NOT T_CONFIG_A OR NOT T_CONFIG_B) + message(FATAL_ERROR "codes_add_config_tree_test: NAME, CONFIG_A and CONFIG_B are required") + endif() + codes_add_run_test(NAME ${T_NAME} BINARY config-tree-equivalence-test NP 1 + ARGS "${T_CONFIG_A}" "${T_CONFIG_B}") +endfunction() + # codes_add_lpio_equivalence_test — assert two configs produce equivalent results. # # Runs BINARY once per config (each with its own --lp-io-dir) and diffs the @@ -339,6 +363,13 @@ add_executable(codes-workload-test ) target_link_libraries(codes-workload-test PUBLIC codes) +# Config-tree equivalence harness: loads two config files through the production +# loader and asserts the two compiled trees are equal (cf_equal). Data-driven -- +# one binary taking the two config paths -- registered once per .conf/.yaml twin +# by codes_add_config_tree_test below. +add_executable(config-tree-equivalence-test config-tree-equivalence-test.c) +target_link_libraries(config-tree-equivalence-test PUBLIC codes) + # Tests are also not consistent with the files to compile, but # that's ok, there are more tests than binary files # Scripts still run as per-scenario shell files. Most single-run tests were @@ -647,3 +678,143 @@ codes_add_lpio_equivalence_test( ARGS --sync=1 --num_messages=1 CONFIG_A "${CMAKE_CURRENT_BINARY_DIR}/conf/dragonfly-custom/dfcustom-8group.conf" CONFIG_B "${CMAKE_CURRENT_BINARY_DIR}/conf/dragonfly-custom/dfcustom-8group.yaml") + +# Config-tree equivalence: for each .conf/.yaml twin above, assert the two files +# compile to an identical config tree (every section/key/value), independent of +# any model run. Registered for the same twin set as the behavioral equivalence +# tests. The surrogate twin is intentionally excluded: its config is assembled per +# run from a template by envsubst (SETUP), so there is no static file pair to diff. +codes_add_config_tree_test(NAME simplenet-config-tree + CONFIG_A "${_tconf}/modelnet-test-simplenet.conf" + CONFIG_B "${_tconf}/modelnet-test-simplenet.yaml") +codes_add_config_tree_test(NAME simplep2p-config-tree + CONFIG_A "${_tconf}/modelnet-test-simplep2p.conf" + CONFIG_B "${_tconf}/modelnet-test-simplep2p.yaml") +codes_add_config_tree_test(NAME loggp-config-tree + CONFIG_A "${_tconf}/modelnet-test-loggp.conf" + CONFIG_B "${_tconf}/modelnet-test-loggp.yaml") +codes_add_config_tree_test(NAME torus-config-tree + CONFIG_A "${_tconf}/modelnet-test-torus.conf" + CONFIG_B "${_tconf}/modelnet-test-torus.yaml") +codes_add_config_tree_test(NAME express-mesh-config-tree + CONFIG_A "${_tconf}/modelnet-test-em.conf" + CONFIG_B "${_tconf}/modelnet-test-em.yaml") +codes_add_config_tree_test(NAME slimfly-config-tree + CONFIG_A "${_tconf}/modelnet-test-slimfly.conf" + CONFIG_B "${_tconf}/modelnet-test-slimfly.yaml") +codes_add_config_tree_test(NAME dragonfly-config-tree + CONFIG_A "${_nwconf}/modelnet-synthetic-dragonfly.conf" + CONFIG_B "${_nwconf}/modelnet-synthetic-dragonfly.yaml") +codes_add_config_tree_test(NAME fattree-config-tree + CONFIG_A "${_nwconf}/modelnet-synthetic-fattree.conf" + CONFIG_B "${_nwconf}/modelnet-synthetic-fattree.yaml") +codes_add_config_tree_test(NAME lsm-config-tree + CONFIG_A "${_tconf}/lsm-test.conf" + CONFIG_B "${_tconf}/lsm-test.yaml") +# Same layout, lsm section pulled in with include: -- exercises the loader's +# collective include resolution, which must yield the identical tree. +codes_add_config_tree_test(NAME lsm-include-config-tree + CONFIG_A "${_tconf}/lsm-test.conf" + CONFIG_B "${_tconf}/lsm-test-include.yaml") +codes_add_config_tree_test(NAME resource-config-tree + CONFIG_A "${_tconf}/buffer_test.conf" + CONFIG_B "${_tconf}/buffer_test.yaml") +codes_add_config_tree_test(NAME ping-pong-config-tree + CONFIG_A "${CMAKE_BINARY_DIR}/doc/example/tutorial-ping-pong.conf" + CONFIG_B "${CMAKE_BINARY_DIR}/doc/example/tutorial-ping-pong.yaml") +# File-enumerated fabrics: the configure_file outputs (absolute connection-file +# paths) are compared -- the tree check never opens the connection files, so they +# need not exist for this test. +codes_add_config_tree_test(NAME dragonfly-dally-config-tree + CONFIG_A "${CMAKE_CURRENT_BINARY_DIR}/conf/dragonfly-dally/dfdally-72.conf" + CONFIG_B "${CMAKE_CURRENT_BINARY_DIR}/conf/dragonfly-dally/dfdally-72.yaml") +codes_add_config_tree_test(NAME dragonfly-plus-config-tree + CONFIG_A "${CMAKE_CURRENT_BINARY_DIR}/conf/dragonfly-plus/dfp-test.conf" + CONFIG_B "${CMAKE_CURRENT_BINARY_DIR}/conf/dragonfly-plus/dfp-test.yaml") +codes_add_config_tree_test(NAME dragonfly-custom-config-tree + CONFIG_A "${CMAKE_CURRENT_BINARY_DIR}/conf/dragonfly-custom/dfcustom-8group.conf" + CONFIG_B "${CMAKE_CURRENT_BINARY_DIR}/conf/dragonfly-custom/dfcustom-8group.yaml") + +# Negative twin: every codes_add_config_tree_test above pairs a config with the +# YAML meant to reproduce it, so they only ever prove cf_equal_report stays silent +# on a match. This one feeds it two configs that are *not* twins -- a fat-tree and +# a dragonfly fabric -- and asserts it accurately reports the divergence. The +# binary exits 1 (trees differ) and cf_equal_report prints a per-key report, so we +# gate on the report text with PASS_REGULAR_EXPRESSION (which passes on a match and +# ignores the non-zero exit) rather than the codes_add_config_tree_test helper, +# which expects a clean exit. "values differ" and "only in ... tree" are emitted +# only by cf_equal_report's divergence report, and any two distinct topologies +# produce at least one of each -- so a match proves it both detected the +# difference and wrote the report. +add_test(NAME config-tree-divergence + COMMAND ${MPIEXEC_EXECUTABLE} ${MPIEXEC_NUMPROC_FLAG} 1 ${MPIEXEC_PREFLAGS} + $ + "${_nwconf}/modelnet-synthetic-fattree.yaml" + "${_nwconf}/modelnet-synthetic-dragonfly.yaml" + ${MPIEXEC_POSTFLAGS}) +set_tests_properties(config-tree-divergence PROPERTIES + PASS_REGULAR_EXPRESSION "values differ|only in .* tree") + +# Resolved-config dump: loading a config with CODES_RESOLVED_CONFIG_DUMP set makes +# rank 0 print the fully-resolved config tree. Load a YAML config in load-only mode +# with the dump aimed at stdout ("-") and assert the dump carries a compiler- +# derived key (modelnet_order) -- i.e. the tree really is the resolved one, not the +# friendly YAML source. No model is run, so nothing is written to the build dir. +add_test(NAME resolved-config-dump + COMMAND ${MPIEXEC_EXECUTABLE} ${MPIEXEC_NUMPROC_FLAG} 1 ${MPIEXEC_PREFLAGS} + $ "${_tconf}/modelnet-test-simplenet.yaml" + ${MPIEXEC_POSTFLAGS}) +set_tests_properties(resolved-config-dump PROPERTIES + ENVIRONMENT "CODES_RESOLVED_CONFIG_DUMP=-" + PASS_REGULAR_EXPRESSION "modelnet_order") + +# Simulation end time from a YAML config: the top-level simulation: block sets an +# end_time that codes_mapping applies to ROSS's g_tw_ts_end at setup time. +# model-net-synthetic-fattree never assigns g_tw_ts_end itself, so the config +# value survives to the run and shows up in ROSS's "Simulation End Time" banner. +# The config sets 60000 ns; the marker asserts the run really ended there. +codes_add_run_test( + NAME modelnet-fattree-config-end-time + BINARY model-net-synthetic-fattree + ARGS --sync=1 -- "${_tconf}/sim-end-time-fattree.yaml" + MARKER "Simulation End Time.*60000.00") +# CLI precedence: the same config, but --end=40000 on the command line wins over +# the config's 60000 -- the banner must show the command-line value. +codes_add_run_test( + NAME modelnet-fattree-cli-end-time-overrides-config + BINARY model-net-synthetic-fattree + ARGS --sync=1 --end=40000 -- "${_tconf}/sim-end-time-fattree.yaml" + MARKER "Simulation End Time.*40000.00") + +# Synthetic workload params from a YAML workload: block. model-net-synthetic +# sends `num_messages` events per server and prints the count in each server's +# finalize line ("... sent_count N recvd_count ..."); the config sets 2, which is +# distinct from the model's built-in default of 20, so the marker proves the +# config value took effect. +codes_add_run_test( + NAME modelnet-synthetic-config-workload-params + BINARY model-net-synthetic + ARGS --sync=1 -- "${_tconf}/synthetic-workload-dragonfly.yaml" + MARKER "sent_count 2 recvd_count") +# CLI precedence: --num_messages=1 on the command line wins over the config's 2, +# so every server sends one message instead. +codes_add_run_test( + NAME modelnet-synthetic-cli-overrides-config-workload-params + BINARY model-net-synthetic + ARGS --sync=1 --num_messages=1 -- "${_tconf}/synthetic-workload-dragonfly.yaml" + MARKER "sent_count 1 recvd_count") + +# Multi-job guard: a jobs: block with more than one job compiles to a JOBS +# section, which the synthetic mains cannot execute (one global traffic pattern, +# no per-job destination dispatch). codes_workload_config_check_unsupported_jobs +# must abort the run with its diagnostic rather than silently running global +# traffic over the placement. Gate on the diagnostic text with +# PASS_REGULAR_EXPRESSION (a match passes and the abort's non-zero exit is +# ignored) instead of the clean-exit helpers; the guard fires before tw_run, so +# the aborted run leaves no output files behind. +add_test(NAME modelnet-synthetic-rejects-multi-job-config + COMMAND ${MPIEXEC_EXECUTABLE} ${MPIEXEC_NUMPROC_FLAG} 1 ${MPIEXEC_PREFLAGS} + $ --sync=1 + ${MPIEXEC_POSTFLAGS} -- "${_tconf}/multi-job-dragonfly.yaml") +set_tests_properties(modelnet-synthetic-rejects-multi-job-config PROPERTIES + PASS_REGULAR_EXPRESSION "places 2 jobs.*cannot execute per-job placement") diff --git a/tests/codes-config-compiler-test.cxx b/tests/codes-config-compiler-test.cxx index be2934b6..1dca6c00 100644 --- a/tests/codes-config-compiler-test.cxx +++ b/tests/codes-config-compiler-test.cxx @@ -6,15 +6,24 @@ // simulator behavior. This is the first real unit-test consumer of the vendored // GoogleTest framework; the model-net equivalence tests cover the emit path. #include "config_compiler.h" +#include "unit_convert.h" #include +#include +#include + +using codes::config::classified_value; +using codes::config::classify_value; using codes::config::compile; using codes::config::compiled_config; using codes::config::compiled_key; using codes::config::compiled_section; using codes::config::config_error; +using codes::config::format_number; using codes::config::parse_includes; +using codes::config::quantity; +using codes::config::value_form; namespace { @@ -742,6 +751,104 @@ schema_version: 1 EXPECT_EQ(find_key(*grp, "modelnet_torus_router"), nullptr); } +// --- compiler-derived PARAMS keys can't be shadowed by user params ---------- +// +// The compiler emits modelnet_order (derived from the model) as the first PARAMS +// key, then appends the user's fabric/component params. The config store returns +// the first match for a name, so a user param of the same name would land after +// the derived one and be silently ignored -- the front-end must reject that +// instead of silently dropping the user's value. + +TEST(ConfigCompiler, FlatRejectsUserModelnetOrderParam) { + // reachable via a flat component's pass-through params. + EXPECT_THROW(compile(R"( +schema_version: 1 +components: + cn: + model: nw-lp + network: simplenet + modelnet_order: bogus +topology: { format: flat, component: cn, nodes: 4 } +)"), + config_error); +} + +TEST(ConfigCompiler, ParametricRejectsUserModelnetOrderFabricKey) { + // reachable via a fabric's pass-through scalar keys. + EXPECT_THROW(compile(R"( +schema_version: 1 +components: + compute_host: { model: nw-lp } +topology: + format: parametric + fabric: + model: dragonfly + shape: + num_routers: 4 + modelnet_order: bogus + hosts: + component: compute_host +)"), + config_error); +} + +TEST(ConfigCompiler, ParametricRejectsUserModelnetOrderHostParam) { + // reachable via the host component's own params (appended to PARAMS). + EXPECT_THROW(compile(R"( +schema_version: 1 +components: + compute_host: + model: nw-lp + modelnet_order: bogus +topology: + format: parametric + fabric: + model: dragonfly + shape: + num_routers: 4 + hosts: + component: compute_host +)"), + config_error); +} + +TEST(ConfigCompiler, DerivedKeyGuardIsExactMatchNotPrefix) { + // a nearby, legitimate key that merely shares a prefix is accepted -- the + // guard is an exact-name match, not a prefix ban. + compiled_config c = compile(R"( +schema_version: 1 +components: + cn: + model: nw-lp + network: simplenet + modelnet_scheduler: fcfs +topology: { format: flat, component: cn, nodes: 4 } +)"); + const compiled_section* params = find_section(c, "PARAMS"); + ASSERT_NE(params, nullptr); + EXPECT_NE(find_key(*params, "modelnet_scheduler"), nullptr); +} + +TEST(ConfigCompiler, ExplicitGroupsAllowsUserModelnetOrder) { + // the explicit-groups form derives no PARAMS at all -- the user lays out + // everything, including modelnet_order -- so it is NOT guarded there. + compiled_config c = compile(R"( +schema_version: 1 +topology: + format: groups + params: + modelnet_order: [dragonfly, dragonfly_router] + groups: + G: { repetitions: 1, lps: { a: 1 } } +)"); + const compiled_section* params = find_section(c, "PARAMS"); + ASSERT_NE(params, nullptr); + const compiled_key* order = find_key(*params, "modelnet_order"); + ASSERT_NE(order, nullptr); + ASSERT_EQ(order->values.size(), 2u); + EXPECT_EQ(order->values.at(0), "dragonfly"); +} + // --- includes / multi-document merge ---------------------------------------- TEST(ConfigCompiler, IncludeMergesComponentsFromBaseDoc) { @@ -835,3 +942,880 @@ TEST(ConfigCompiler, ParseIncludesExtractsList) { EXPECT_EQ(many.at(0), "a.yaml"); EXPECT_EQ(many.at(1), "b.yaml"); } + +// ============================================================================ +// Unit-bearing values +// ============================================================================ +// +// A dimensioned config value may be a bare number in the model's internal unit +// or a unit-bearing string that is converted to that unit. The parsing/ +// conversion core (unit_convert) is tested directly first, then end-to-end +// through compile(). + +namespace { + +constexpr double KIB = 1024.0; +constexpr double MIB = 1024.0 * 1024.0; +constexpr double GIB = 1024.0 * 1024.0 * 1024.0; + +// Fetch the first value of a PARAMS key from a compiled config (empty if absent). +std::string params_value(const compiled_config& c, const char* key) { + const compiled_section* p = find_section(c, "PARAMS"); + if (!p) + return {}; + const compiled_key* k = find_key(*p, key); + return (k && !k->values.empty()) ? k->values.at(0) : std::string(); +} + +// A parametric dragonfly (num_routers: 4) carrying one extra fabric-level scalar +// key -- the pass-through path most model knobs take into PARAMS. +std::string dragonfly_with(const std::string& key, const std::string& value) { + return std::string(R"( +schema_version: 1 +components: + compute_host: { model: nw-lp } +topology: + format: parametric + fabric: + model: dragonfly + shape: + num_routers: 4 + )") + key + ": " + value + R"( + hosts: + component: compute_host +)"; +} + +// A single-key parametric fabric of the named model, for the flat-bandwidth +// models (torus/fattree) whose link_bandwidth unit differs. +std::string torus_link_bw(const std::string& value) { + return std::string(R"( +schema_version: 1 +components: + compute_host: { model: nw-lp } +topology: + format: parametric + fabric: + model: torus + shape: + n_dims: 1 + dim_length: "8" + link_bandwidth: )") + value + R"( + hosts: + component: compute_host +)"; +} + +std::string fattree_link_bw(const std::string& value) { + return std::string(R"( +schema_version: 1 +components: + compute_host: { model: nw-lp } +topology: + format: parametric + fabric: + model: fattree + shape: + num_levels: 3 + switch_count: 32 + switch_radix: 8 + link_bandwidth: )") + value + R"( + hosts: + component: compute_host +)"; +} + +// A flat simplenet component carrying one extra scalar key. +std::string simplenet_with(const std::string& key, const std::string& value) { + return std::string(R"( +schema_version: 1 +components: + cn: + model: nw-lp + network: simplenet + )") + key + ": " + value + R"( +topology: + format: flat + component: cn + nodes: 4 +)"; +} + +} // namespace + +// --- classify_value: bare / plain / units -------------------------------- + +TEST(UnitConvert, ClassifiesBareNumberAndPlainString) { + EXPECT_EQ(classify_value("512").form, value_form::bare_number); + EXPECT_EQ(classify_value("5.25").form, value_form::bare_number); + EXPECT_EQ(classify_value("1.5e6").form, value_form::bare_number); // scientific = bare + EXPECT_EQ(classify_value("-3").form, value_form::bare_number); + + EXPECT_EQ(classify_value("adaptive").form, value_form::plain); + EXPECT_EQ(classify_value("some/path.conf").form, value_form::plain); + EXPECT_EQ(classify_value("").form, value_form::plain); + EXPECT_EQ(classify_value("4,2,2").form, value_form::unknown_suffix); // number + junk +} + +TEST(UnitConvert, ParsesTimeSuffixesToNanoseconds) { + classified_value ns = classify_value("40ns"); + EXPECT_EQ(ns.form, value_form::with_unit); + EXPECT_EQ(ns.kind, quantity::time); + EXPECT_DOUBLE_EQ(ns.canonical, 40.0); + EXPECT_DOUBLE_EQ(classify_value("10us").canonical, 10000.0); + EXPECT_DOUBLE_EQ(classify_value("1.5ms").canonical, 1.5e6); + EXPECT_DOUBLE_EQ(classify_value("2s").canonical, 2.0e9); +} + +TEST(UnitConvert, ParsesSizeSuffixesToBytes) { + classified_value b = classify_value("1500B"); + EXPECT_EQ(b.kind, quantity::size); + EXPECT_DOUBLE_EQ(b.canonical, 1500.0); + EXPECT_DOUBLE_EQ(classify_value("2KiB").canonical, 2048.0); + EXPECT_DOUBLE_EQ(classify_value("4MiB").canonical, 4.0 * MIB); + EXPECT_DOUBLE_EQ(classify_value("1GiB").canonical, GIB); + // decimal SI forms are accepted and are 1000-based. + EXPECT_DOUBLE_EQ(classify_value("2KB").canonical, 2000.0); + EXPECT_DOUBLE_EQ(classify_value("1MB").canonical, 1.0e6); +} + +TEST(UnitConvert, ParsesBandwidthBitAndByteRates) { + // bit rates are decimal, /8 to bytes/second. + classified_value g = classify_value("100Gbps"); + EXPECT_EQ(g.kind, quantity::bandwidth); + EXPECT_DOUBLE_EQ(g.canonical, 100.0e9 / 8.0); + EXPECT_DOUBLE_EQ(classify_value("8bps").canonical, 1.0); + // byte rates: decimal and binary forms. + EXPECT_DOUBLE_EQ(classify_value("2.5GBps").canonical, 2.5e9); + EXPECT_DOUBLE_EQ(classify_value("1GiBps").canonical, GIB); + EXPECT_DOUBLE_EQ(classify_value("1KiBps").canonical, KIB); +} + +TEST(UnitConvert, SuffixMatchIsCaseSensitiveBitVsByte) { + // lowercase 'b' = bits, uppercase 'B' = bytes: the two differ by a factor 8. + EXPECT_DOUBLE_EQ(classify_value("1Gbps").canonical, 1.0e9 / 8.0); + EXPECT_DOUBLE_EQ(classify_value("1GBps").canonical, 1.0e9); +} + +TEST(UnitConvert, RejectsAndPassesTrailingText) { + EXPECT_EQ(classify_value("5xz").form, value_form::unknown_suffix); + EXPECT_EQ(classify_value("512qux").form, value_form::unknown_suffix); + // surrounding whitespace is ignored. + EXPECT_EQ(classify_value(" 5ms ").form, value_form::with_unit); + EXPECT_DOUBLE_EQ(classify_value(" 5ms ").canonical, 5.0e6); +} + +// --- format_number: integers vs fractions, round-trip -------------------- + +TEST(UnitConvert, FormatsIntegerValuesWithoutDecimalPoint) { + EXPECT_EQ(format_number(2048.0), "2048"); + EXPECT_EQ(format_number(1500.0), "1500"); + EXPECT_EQ(format_number(0.0), "0"); + EXPECT_EQ(format_number(1.0e9), "1000000000"); +} + +TEST(UnitConvert, FormatsFractionsAsPlainDecimalThatRoundTrips) { + EXPECT_EQ(format_number(1.5), "1.5"); + EXPECT_EQ(format_number(2.5), "2.5"); + // a converted bandwidth: 2.5 GB/s in GiB/s -- no exponent, round-trips. + double v = 2.5e9 / GIB; + std::string s = format_number(v); + EXPECT_EQ(s.find('e'), std::string::npos); + EXPECT_EQ(s.find('E'), std::string::npos); + EXPECT_DOUBLE_EQ(std::strtod(s.c_str(), nullptr), v); +} + +// --- end-to-end: conversion through compile() ---------------------------- + +TEST(ConfigCompilerUnits, ConvertsSizeSuffixToBytes) { + EXPECT_EQ(params_value(compile(dragonfly_with("packet_size", "2KiB")), "packet_size"), "2048"); + EXPECT_EQ(params_value(compile(dragonfly_with("message_size", "1500B")), "message_size"), + "1500"); + EXPECT_EQ(params_value(compile(dragonfly_with("chunk_size", "4MiB")), "chunk_size"), + format_number(4.0 * MIB)); +} + +TEST(ConfigCompilerUnits, ConvertsTimeSuffixToNanoseconds) { + EXPECT_EQ(params_value(compile(dragonfly_with("router_delay", "1.5us")), "router_delay"), + "1500"); + EXPECT_EQ(params_value(compile(simplenet_with("net_startup_ns", "5ms")), "net_startup_ns"), + "5000000"); +} + +TEST(ConfigCompilerUnits, ConvertsBandwidthToGiBPerSecond) { + // cn_bandwidth is read in GiB/s; "2.5GBps" is 2.5 GB/s (decimal) = 2.5e9 B/s. + std::string got = + params_value(compile(dragonfly_with("cn_bandwidth", "2.5GBps")), "cn_bandwidth"); + EXPECT_EQ(got, format_number(2.5e9 / GIB)); +} + +TEST(ConfigCompilerUnits, BandwidthConversionIsPerModel) { + // The same "8Gbps" (1e9 B/s) resolves differently: torus reads GiB/s, fattree + // reads bytes/ns (= GB/s decimal). fattree lands on a clean 1. + std::string torus = params_value(compile(torus_link_bw("8Gbps")), "link_bandwidth"); + std::string fattree = params_value(compile(fattree_link_bw("8Gbps")), "link_bandwidth"); + EXPECT_EQ(fattree, "1"); + EXPECT_EQ(torus, format_number(1.0e9 / GIB)); + EXPECT_NE(torus, fattree); +} + +TEST(ConfigCompilerUnits, ConvertsSimplenetBandwidthToMebibytesPerSecond) { + // net_bw_mbps is read in MiB/s despite the name; 1 GiB/s = 1024 MiB/s. + EXPECT_EQ(params_value(compile(simplenet_with("net_bw_mbps", "1GiBps")), "net_bw_mbps"), + "1024"); +} + +TEST(ConfigCompilerUnits, ConvertsMicrosecondCountingWindow) { + // dragonfly QoS counting_* windows are read in microseconds, not ns. + EXPECT_EQ(params_value(compile(R"( +schema_version: 1 +components: + compute_host: { model: nw-lp } +topology: + format: parametric + fabric: + model: dragonfly-dally + shape: + num_routers: 4 + num_groups: 9 + num_cns_per_router: 2 + num_global_channels: 2 + counting_start: 5ms + hosts: + component: compute_host +)"), + "counting_start"), + "5000"); // 5 ms = 5000 us +} + +TEST(ConfigCompilerUnits, ConvertsLinkClassBandwidthBlock) { + // the links: sugar reaches the same conversion (local_bandwidth here). + std::string got = params_value(compile(R"( +schema_version: 1 +components: + compute_host: { model: nw-lp } +topology: + format: parametric + fabric: + model: dragonfly + shape: + num_routers: 4 + links: + local: { bandwidth: 1GiBps } + hosts: + component: compute_host +)"), + "local_bandwidth"); + EXPECT_EQ(got, "1"); // 1 GiB/s in GiB/s +} + +// --- bare numbers keep their exact spelling (identity) -------------------- + +TEST(ConfigCompilerUnits, BareNumbersPassThroughVerbatim) { + // A bare number already means the model's internal unit and is emitted + // byte-for-byte -- this is what keeps every existing .conf twin equivalent. + EXPECT_EQ(params_value(compile(dragonfly_with("packet_size", "512")), "packet_size"), "512"); + EXPECT_EQ(params_value(compile(dragonfly_with("cn_bandwidth", "5.25")), "cn_bandwidth"), + "5.25"); + EXPECT_EQ(params_value(compile(simplenet_with("net_bw_mbps", "20000")), "net_bw_mbps"), + "20000"); + // a non-unit string on an unclassified knob passes through untouched. + const compiled_config c = compile(torus_link_bw("2.0")); + EXPECT_EQ(params_value(c, "dim_length"), "8"); // number+junk on unclassified: verbatim +} + +// --- negative suite ------------------------------------------------------- + +TEST(ConfigCompilerUnits, RejectsUnknownUnitSuffixOnDimensionedParam) { + EXPECT_THROW(compile(dragonfly_with("packet_size", "512qux")), config_error); + EXPECT_THROW(compile(dragonfly_with("chunk_size", "32xyz")), config_error); +} + +TEST(ConfigCompilerUnits, RejectsWrongQuantityUnit) { + // a time unit on a size parameter is a mistake, not a silent misread. + EXPECT_THROW(compile(dragonfly_with("packet_size", "5ms")), config_error); + // a size unit on a bandwidth parameter, likewise. + EXPECT_THROW(compile(dragonfly_with("cn_bandwidth", "5MiB")), config_error); +} + +TEST(ConfigCompilerUnits, RejectsNegativeDimensionedValue) { + EXPECT_THROW(compile(dragonfly_with("cn_bandwidth", "-1Gbps")), config_error); + EXPECT_THROW(compile(dragonfly_with("packet_size", "-5B")), config_error); +} + +TEST(ConfigCompilerUnits, RejectsUnitSuffixOnUnclassifiableParam) { + // num_vcs is a plain count the model atof()s; a unit there would be silently + // truncated to its bare number, so it is rejected up front. + EXPECT_THROW(compile(dragonfly_with("num_vcs", "100Gbps")), config_error); + EXPECT_THROW(compile(dragonfly_with("num_vcs", "4KiB")), config_error); +} + +// --------------------------------------------------------------------------- +// simulation: block -- run-level settings (end_time, pe_mem_factor) resolved to +// the PARAMS keys codes_mapping reads. +// --------------------------------------------------------------------------- + +namespace { +// kFlatBase plus a trailing top-level simulation: block (body already indented). +std::string flat_with_simulation(const std::string& body) { + return std::string(kFlatBase) + "simulation:\n" + body; +} +} // namespace + +TEST(ConfigCompilerSimulation, EmitsEndTimeAndMemFactorToParams) { + // both settings land in PARAMS -- the exact section/keys codes_mapping reads. + compiled_config c = compile(flat_with_simulation(" end_time: 5000\n pe_mem_factor: 512\n")); + EXPECT_EQ(params_value(c, "end_time"), "5000"); + EXPECT_EQ(params_value(c, "pe_mem_factor"), "512"); +} + +TEST(ConfigCompilerSimulation, EndTimeAcceptsTimeUnits) { + // a bare number is nanoseconds; a unit-bearing value converts to ns. + EXPECT_EQ(params_value(compile(flat_with_simulation(" end_time: 100us\n")), "end_time"), + "100000"); + EXPECT_EQ(params_value(compile(flat_with_simulation(" end_time: 2ms\n")), "end_time"), + "2000000"); + EXPECT_EQ(params_value(compile(flat_with_simulation(" end_time: 250\n")), "end_time"), "250"); +} + +TEST(ConfigCompilerSimulation, RejectsUnknownKeyInBlock) { + EXPECT_THROW(compile(flat_with_simulation(" bogus: 1\n")), config_error); +} + +TEST(ConfigCompilerSimulation, RejectsNonPositiveMemFactor) { + EXPECT_THROW(compile(flat_with_simulation(" pe_mem_factor: 0\n")), config_error); + EXPECT_THROW(compile(flat_with_simulation(" pe_mem_factor: -4\n")), config_error); +} + +TEST(ConfigCompilerSimulation, RejectsNonIntegerMemFactor) { + EXPECT_THROW(compile(flat_with_simulation(" pe_mem_factor: 3.5\n")), config_error); + EXPECT_THROW(compile(flat_with_simulation(" pe_mem_factor: abc\n")), config_error); +} + +TEST(ConfigCompilerSimulation, RejectsMalformedEndTime) { + // not a number, not positive, negative, and an unrecognized suffix. + EXPECT_THROW(compile(flat_with_simulation(" end_time: abc\n")), config_error); + EXPECT_THROW(compile(flat_with_simulation(" end_time: 0\n")), config_error); + EXPECT_THROW(compile(flat_with_simulation(" end_time: -5us\n")), config_error); + EXPECT_THROW(compile(flat_with_simulation(" end_time: 100xyz\n")), config_error); +} + +TEST(ConfigCompilerSimulation, RejectsWrongQuantityUnitOnEndTime) { + // a size or bandwidth unit on a time value is a mistake, not a silent misread. + EXPECT_THROW(compile(flat_with_simulation(" end_time: 2KiB\n")), config_error); + EXPECT_THROW(compile(flat_with_simulation(" end_time: 5Gbps\n")), config_error); +} + +TEST(ConfigCompilerSimulation, ConflictsWithPassthroughParam) { + // A key set both as a fabric pass-through and in simulation: would be silently + // dropped by the config store's first-match lookup, so it is rejected. + EXPECT_THROW( + compile(dragonfly_with("pe_mem_factor", "256") + "simulation:\n pe_mem_factor: 512\n"), + config_error); + EXPECT_THROW(compile(dragonfly_with("end_time", "5000") + "simulation:\n end_time: 9000\n"), + config_error); +} + +TEST(ConfigCompilerSimulation, PassthroughParamAloneStillWorks) { + // Without a simulation: block, pe_mem_factor on the fabric passes through to + // PARAMS exactly as before -- no regression for existing configs. + compiled_config c = compile(dragonfly_with("pe_mem_factor", "256")); + EXPECT_EQ(params_value(c, "pe_mem_factor"), "256"); +} + +TEST(ConfigCompilerSimulation, WorksWithParametricTopology) { + compiled_config c = + compile(std::string(kParametricDragonfly) + "simulation:\n end_time: 50us\n"); + EXPECT_EQ(params_value(c, "end_time"), "50000"); +} + +TEST(ConfigCompilerSimulation, WorksWithExplicitGroups) { + // the explicit-groups form builds PARAMS itself; the simulation keys still + // land in it. + compiled_config c = compile(std::string(kGroupsBase) + "simulation:\n pe_mem_factor: 64\n"); + EXPECT_EQ(params_value(c, "pe_mem_factor"), "64"); +} + +TEST(ConfigCompilerSimulation, ConflictsWithExplicitGroupsParam) { + // a pe_mem_factor written directly in the explicit-groups params: collides + // with simulation.pe_mem_factor exactly like the pass-through fabric case. + const char* yaml = R"( +schema_version: 1 +topology: + format: groups + params: + pe_mem_factor: 256 + groups: + GRP: + repetitions: 1 + lps: + a: 1 +simulation: + pe_mem_factor: 512 +)"; + EXPECT_THROW(compile(yaml), config_error); +} + +// --- workload: shortcut and jobs: block ----------------------------------- +// +// A node's workload (what it runs) is configured separately from the node model +// (what it is): an inline `workload:` on the topology's compute component is the +// single-workload shortcut, and a top-level `jobs:` block is the explicit +// multi-job form. Both lower to a WORKLOAD (or JOBS) section a synthetic main +// reads. Only synthetic is wired; other types are recognized but rejected. + +namespace { + +// Fetch the first value of a WORKLOAD-section key (empty if absent). +std::string workload_value(const compiled_config& c, const char* key) { + const compiled_section* w = find_section(c, "WORKLOAD"); + if (!w) + return {}; + const compiled_key* k = find_key(*w, key); + return (k && !k->values.empty()) ? k->values.at(0) : std::string(); +} + +// Deep structural equality of two compiled configs (section/key/value/subsection, +// order included) -- used to prove two spellings compile to the identical tree. +bool key_eq(const compiled_key& a, const compiled_key& b) { + return a.name == b.name && a.values == b.values; +} +bool section_eq(const compiled_section& a, const compiled_section& b) { + if (a.name != b.name || a.keys.size() != b.keys.size() || + a.subsections.size() != b.subsections.size()) + return false; + for (size_t i = 0; i < a.keys.size(); ++i) + if (!key_eq(a.keys[i], b.keys[i])) + return false; + for (size_t i = 0; i < a.subsections.size(); ++i) + if (!section_eq(a.subsections[i], b.subsections[i])) + return false; + return true; +} +bool config_eq(const compiled_config& a, const compiled_config& b) { + if (a.sections.size() != b.sections.size()) + return false; + for (size_t i = 0; i < a.sections.size(); ++i) + if (!section_eq(a.sections[i], b.sections[i])) + return false; + return true; +} + +// A flat 4-node config whose compute component carries an inline workload: block. +// `body` supplies the workload's keys, each indented six spaces. +std::string flat_with_workload(const std::string& body) { + return std::string(R"( +schema_version: 1 +components: + cn: + model: nw-lp + network: simplenet + message_size: 464 + workload: +)") + body + + R"( +topology: + format: flat + component: cn + nodes: 4 +)"; +} + +// A flat 4-node config with a top-level jobs: block. `body` supplies the job +// list entries (each `- id:` indented two spaces). +std::string flat_with_jobs(const std::string& body) { + return std::string(kFlatBase) + "jobs:\n" + body; +} + +} // namespace + +TEST(ConfigCompilerWorkload, InlineEmitsResolvedWorkloadSection) { + // payload_size resolves to bytes, arrival_time to nanoseconds; traffic stays + // a verbatim pattern name and num_messages a positive integer. + compiled_config c = compile(flat_with_workload(" type: synthetic\n" + " traffic: uniform\n" + " num_messages: 30\n" + " payload_size: \"2KiB\"\n" + " arrival_time: \"1us\"\n")); + EXPECT_EQ(workload_value(c, "type"), "synthetic"); + EXPECT_EQ(workload_value(c, "traffic"), "uniform"); + EXPECT_EQ(workload_value(c, "num_messages"), "30"); + EXPECT_EQ(workload_value(c, "payload_size"), "2048"); + EXPECT_EQ(workload_value(c, "arrival_time"), "1000"); +} + +TEST(ConfigCompilerWorkload, NoWorkloadEmitsNoSection) { + // A config without workload:/jobs: emits no WORKLOAD section, so a legacy + // config's tree is unchanged. + compiled_config c = compile(kFlatBase); + EXPECT_EQ(find_section(c, "WORKLOAD"), nullptr); + EXPECT_EQ(find_section(c, "JOBS"), nullptr); +} + +TEST(ConfigCompilerWorkload, RejectsUnknownKey) { + EXPECT_THROW(compile(flat_with_workload(" type: synthetic\n bogus: 1\n")), + config_error); +} + +TEST(ConfigCompilerWorkload, RejectsMissingType) { + EXPECT_THROW(compile(flat_with_workload(" traffic: uniform\n")), config_error); +} + +TEST(ConfigCompilerWorkload, RejectsNonSyntheticType) { + // dumpi etc. are recognized as a workload shape but not yet configurable here. + EXPECT_THROW(compile(flat_with_workload(" type: dumpi\n trace: app.dumpi\n")), + config_error); +} + +TEST(ConfigCompilerWorkload, RejectsWrongUnitOnArrivalTime) { + EXPECT_THROW(compile(flat_with_workload(" type: synthetic\n" + " arrival_time: \"2KiB\"\n")), + config_error); +} + +TEST(ConfigCompilerWorkload, RejectsWrongUnitOnPayloadSize) { + EXPECT_THROW(compile(flat_with_workload(" type: synthetic\n" + " payload_size: \"1us\"\n")), + config_error); +} + +TEST(ConfigCompilerWorkload, RejectsNonPositiveMessageCount) { + EXPECT_THROW(compile(flat_with_workload(" type: synthetic\n num_messages: 0\n")), + config_error); + EXPECT_THROW(compile(flat_with_workload(" type: synthetic\n num_messages: -5\n")), + config_error); +} + +TEST(ConfigCompilerWorkload, RejectsNonIntegerMessageCount) { + EXPECT_THROW(compile(flat_with_workload(" type: synthetic\n num_messages: 3.5\n")), + config_error); +} + +TEST(ConfigCompilerWorkload, RejectsWorkloadOnNonComputeComponent) { + // a workload: on a component the topology does not run is meaningless; reject + // it rather than silently ignore it. + const char* yaml = R"( +schema_version: 1 +components: + cn: + model: nw-lp + network: simplenet + other: + model: nw-lp + workload: + type: synthetic +topology: + format: flat + component: cn + nodes: 4 +)"; + EXPECT_THROW(compile(yaml), config_error); +} + +TEST(ConfigCompilerWorkload, RejectsWorkloadWithExplicitGroups) { + const char* yaml = R"( +schema_version: 1 +components: + cn: + model: nw-lp + workload: + type: synthetic +topology: + format: groups + groups: + GRP: + repetitions: 1 + lps: + a: 1 +)"; + EXPECT_THROW(compile(yaml), config_error); +} + +TEST(ConfigCompilerWorkload, RejectsInlineWorkloadAndJobsTogether) { + // exactly one form is allowed. + const char* yaml = R"( +schema_version: 1 +components: + cn: + model: nw-lp + network: simplenet + workload: + type: synthetic +topology: + format: flat + component: cn + nodes: 4 +jobs: + - id: j + workload: + type: synthetic + ranks: 4 + placement: + policy: contiguous +)"; + EXPECT_THROW(compile(yaml), config_error); +} + +TEST(ConfigCompilerJobs, SingleAllNodesJobDesugarsToInlineWorkload) { + // A single synthetic job placed contiguously on every node must compile to + // the exact same tree as the inline workload: shortcut. + const char* inln = R"( +schema_version: 1 +components: + cn: + model: nw-lp + network: simplenet + message_size: 464 + workload: + type: synthetic + traffic: uniform + num_messages: 30 +topology: + format: flat + component: cn + nodes: 4 +)"; + const char* jobs = R"( +schema_version: 1 +components: + cn: + model: nw-lp + network: simplenet + message_size: 464 +topology: + format: flat + component: cn + nodes: 4 +jobs: + - id: everything + workload: + type: synthetic + traffic: uniform + num_messages: 30 + ranks: 4 + placement: + policy: contiguous +)"; + compiled_config a = compile(inln); + compiled_config b = compile(jobs); + EXPECT_TRUE(config_eq(a, b)); + // and it really is the desugared shortcut: a WORKLOAD section, no JOBS. + EXPECT_NE(find_section(b, "WORKLOAD"), nullptr); + EXPECT_EQ(find_section(b, "JOBS"), nullptr); +} + +TEST(ConfigCompilerJobs, MultiJobEmitsJobsSectionWithNodeAllocation) { + // two explicitly-placed jobs on a 4-node topology -> a JOBS section, one + // subsection per job carrying its workload, ranks, and resolved node list. + compiled_config c = compile(flat_with_jobs(R"( - id: front + workload: + type: synthetic + traffic: uniform + ranks: 2 + placement: + nodes: [0, 1] + - id: back + workload: + type: synthetic + traffic: nearest_neighbor + ranks: 2 + placement: + nodes: [2, 3] +)")); + EXPECT_EQ(find_section(c, "WORKLOAD"), nullptr); + const compiled_section* jobs = find_section(c, "JOBS"); + ASSERT_NE(jobs, nullptr); + EXPECT_EQ(find_key(*jobs, "num_jobs")->values.at(0), "2"); + const compiled_section* front = find_subsection(*jobs, "front"); + const compiled_section* back = find_subsection(*jobs, "back"); + ASSERT_NE(front, nullptr); + ASSERT_NE(back, nullptr); + EXPECT_EQ(find_key(*front, "traffic")->values.at(0), "uniform"); + EXPECT_EQ(find_key(*front, "ranks")->values.at(0), "2"); + const compiled_key* fnodes = find_key(*front, "nodes"); + ASSERT_NE(fnodes, nullptr); + EXPECT_EQ(fnodes->values, (std::vector{"0", "1"})); + const compiled_key* bnodes = find_key(*back, "nodes"); + ASSERT_NE(bnodes, nullptr); + EXPECT_EQ(bnodes->values, (std::vector{"2", "3"})); +} + +TEST(ConfigCompilerJobs, ContiguousPolicyPacksInDeclaredOrder) { + // two contiguous jobs pack from node 0 in order: [0,1] then [2,3]. + compiled_config c = compile(flat_with_jobs(R"( - id: a + workload: { type: synthetic } + ranks: 2 + placement: { policy: contiguous } + - id: b + workload: { type: synthetic } + ranks: 2 + placement: { policy: contiguous } +)")); + const compiled_section* jobs = find_section(c, "JOBS"); + ASSERT_NE(jobs, nullptr); + EXPECT_EQ(find_key(*find_subsection(*jobs, "a"), "nodes")->values, + (std::vector{"0", "1"})); + EXPECT_EQ(find_key(*find_subsection(*jobs, "b"), "nodes")->values, + (std::vector{"2", "3"})); +} + +TEST(ConfigCompilerJobs, RejectsDuplicateId) { + EXPECT_THROW(compile(flat_with_jobs(R"( - id: dup + workload: { type: synthetic } + ranks: 2 + placement: { policy: contiguous } + - id: dup + workload: { type: synthetic } + ranks: 2 + placement: { policy: contiguous } +)")), + config_error); +} + +TEST(ConfigCompilerJobs, RejectsEmptyId) { + EXPECT_THROW(compile(flat_with_jobs(R"( - id: "" + workload: { type: synthetic } + ranks: 4 + placement: { policy: contiguous } +)")), + config_error); +} + +TEST(ConfigCompilerJobs, RejectsMissingWorkload) { + EXPECT_THROW(compile(flat_with_jobs(R"( - id: j + ranks: 4 + placement: { policy: contiguous } +)")), + config_error); +} + +TEST(ConfigCompilerJobs, RejectsNonPositiveRanks) { + EXPECT_THROW(compile(flat_with_jobs(R"( - id: j + workload: { type: synthetic } + ranks: 0 + placement: { policy: contiguous } +)")), + config_error); +} + +TEST(ConfigCompilerJobs, RejectsNonIntegerRanks) { + EXPECT_THROW(compile(flat_with_jobs(R"( - id: j + workload: { type: synthetic } + ranks: two + placement: { policy: contiguous } +)")), + config_error); +} + +TEST(ConfigCompilerJobs, RejectsMissingPlacement) { + EXPECT_THROW(compile(flat_with_jobs(R"( - id: j + workload: { type: synthetic } + ranks: 4 +)")), + config_error); +} + +TEST(ConfigCompilerJobs, RejectsPlacementWithBothForms) { + EXPECT_THROW(compile(flat_with_jobs(R"( - id: j + workload: { type: synthetic } + ranks: 4 + placement: + policy: contiguous + nodes: [0, 1, 2, 3] +)")), + config_error); +} + +TEST(ConfigCompilerJobs, RejectsPlacementWithNeitherForm) { + EXPECT_THROW(compile(flat_with_jobs(R"( - id: j + workload: { type: synthetic } + ranks: 4 + placement: {} +)")), + config_error); +} + +TEST(ConfigCompilerJobs, RejectsUnknownPlacementPolicy) { + EXPECT_THROW(compile(flat_with_jobs(R"( - id: j + workload: { type: synthetic } + ranks: 4 + placement: { policy: scattered } +)")), + config_error); +} + +TEST(ConfigCompilerJobs, RejectsNodeOutOfRange) { + // node 9 does not exist in a 4-node topology. + EXPECT_THROW(compile(flat_with_jobs(R"( - id: j + workload: { type: synthetic } + ranks: 1 + placement: { nodes: [9] } +)")), + config_error); +} + +TEST(ConfigCompilerJobs, RejectsDoubleBookedNode) { + EXPECT_THROW(compile(flat_with_jobs(R"( - id: a + workload: { type: synthetic } + ranks: 2 + placement: { nodes: [0, 1] } + - id: b + workload: { type: synthetic } + ranks: 2 + placement: { nodes: [1, 2] } +)")), + config_error); +} + +TEST(ConfigCompilerJobs, RejectsRanksNodeListMismatch) { + EXPECT_THROW(compile(flat_with_jobs(R"( - id: j + workload: { type: synthetic } + ranks: 3 + placement: { nodes: [0, 1] } +)")), + config_error); +} + +TEST(ConfigCompilerJobs, RejectsTotalRanksExceedingSlots) { + // two contiguous jobs of 3 ranks each need 6 slots; the topology has 4. + EXPECT_THROW(compile(flat_with_jobs(R"( - id: a + workload: { type: synthetic } + ranks: 3 + placement: { policy: contiguous } + - id: b + workload: { type: synthetic } + ranks: 3 + placement: { policy: contiguous } +)")), + config_error); +} + +TEST(ConfigCompilerJobs, RejectsQosKey) { + // qos is recognized but not yet wired. + EXPECT_THROW(compile(flat_with_jobs(R"( - id: j + workload: { type: synthetic } + ranks: 4 + placement: { policy: contiguous } + qos: 1 +)")), + config_error); +} + +TEST(ConfigCompilerJobs, RejectsUnknownJobKey) { + EXPECT_THROW(compile(flat_with_jobs(R"( - id: j + workload: { type: synthetic } + ranks: 4 + placement: { policy: contiguous } + bogus: 1 +)")), + config_error); +} + +TEST(ConfigCompilerJobs, RejectsNonSyntheticJobWorkload) { + EXPECT_THROW(compile(flat_with_jobs(R"( - id: j + workload: { type: dumpi, trace: app.dumpi } + ranks: 4 + placement: { policy: contiguous } +)")), + config_error); +} diff --git a/tests/conf/modelnet-test-simplenet.yaml b/tests/conf/modelnet-test-simplenet.yaml index c7c5109b..405a129e 100644 --- a/tests/conf/modelnet-test-simplenet.yaml +++ b/tests/conf/modelnet-test-simplenet.yaml @@ -5,16 +5,22 @@ schema_version: 1 # workload plus its NIC model) and a node count -- simplenet is a uniform # all-to-all model, so there is no matrix file and no explicit node graph, and # the net params sit directly on the component. +# +# The dimensioned params carry explicit units (see doc/dev/yaml-config.md): they +# convert to the exact same internal values as the bare numbers in the .conf twin +# -- sizes to bytes, the startup latency to ns, and net_bw_mbps to MiB/s (its +# internal unit, which the "mbps" name does not match). Bandwidth especially +# reads better tagged, since the internal unit differs across models. components: compute_node: model: nw-lp network: simplenet - packet_size: 512 - message_size: 464 + packet_size: 512B + message_size: 464B modelnet_scheduler: fcfs - net_startup_ns: 1.5 - net_bw_mbps: 20000 + net_startup_ns: 1.5ns + net_bw_mbps: 20000MiBps topology: format: flat diff --git a/tests/conf/multi-job-dragonfly.yaml b/tests/conf/multi-job-dragonfly.yaml new file mode 100644 index 00000000..74668a15 --- /dev/null +++ b/tests/conf/multi-job-dragonfly.yaml @@ -0,0 +1,45 @@ +schema_version: 1 + +# Drives the multi-job guard. Two jobs compile to a JOBS section (num_jobs: 2 +# plus a per-job subsection with the resolved allocation), which no synthetic +# main can execute: they generate one global traffic pattern with no per-job +# destination dispatch. A synthetic main loading this config must abort with +# codes_workload_config_check_unsupported_jobs()'s diagnostic instead of +# silently running global traffic over the placement. +# +# Topology matches synthetic-workload-dragonfly.yaml (num_routers: 4 -> 72 +# compute nodes); the two contiguous jobs book 40 + 32 = 72 slots. + +components: + compute_host: + model: nw-lp + +topology: + format: parametric + fabric: + model: dragonfly + shape: + num_routers: 4 + links: + local: { bandwidth: 5.25, vc_size: 4096 } + global: { bandwidth: 4.7, vc_size: 8192 } + cn: { bandwidth: 5.25, vc_size: 4096 } + routing: + algorithm: adaptive + packet_size: 512 + chunk_size: 32 + num_vcs: 1 + modelnet_scheduler: fcfs + message_size: 512 + hosts: + component: compute_host + +jobs: + - id: production + workload: { type: synthetic, traffic: uniform, num_messages: 2 } + ranks: 40 + placement: { policy: contiguous } + - id: background + workload: { type: synthetic, traffic: uniform, arrival_time: "500ns" } + ranks: 32 + placement: { policy: contiguous } diff --git a/tests/conf/sim-end-time-fattree.yaml b/tests/conf/sim-end-time-fattree.yaml new file mode 100644 index 00000000..edb8d820 --- /dev/null +++ b/tests/conf/sim-end-time-fattree.yaml @@ -0,0 +1,40 @@ +schema_version: 1 + +# Drives the simulation end-time seam: the top-level simulation: block sets a +# run end time (in nanoseconds) that codes_mapping applies to ROSS's g_tw_ts_end +# when the command line did not pass --end. The fat-tree fabric below is the same +# one the model-net-synthetic-fattree equivalence tests use; only the end time is +# distinctive so a run's "Simulation End Time" banner is checkable. + +components: + compute_host: + model: nw-lp + +topology: + format: parametric + fabric: + model: fattree + shape: + num_levels: 3 + switch_count: 32 + switch_radix: 8 + routing: + algorithm: adaptive + ft_type: 0 + packet_size: 512 + message_size: 512 + chunk_size: 512 + modelnet_scheduler: fcfs + router_delay: 90 + terminal_radix: 1 + soft_delay: 1000 + vc_size: 65536 + cn_vc_size: 65536 + link_bandwidth: 12.5 + cn_bandwidth: 12.5 + rail_routing: adaptive + hosts: + component: compute_host + +simulation: + end_time: "60us" # 60000 ns; applied unless the CLI passes --end diff --git a/tests/conf/synthetic-workload-dragonfly.yaml b/tests/conf/synthetic-workload-dragonfly.yaml new file mode 100644 index 00000000..e1c4f04f --- /dev/null +++ b/tests/conf/synthetic-workload-dragonfly.yaml @@ -0,0 +1,40 @@ +schema_version: 1 + +# Drives the synthetic workload: seam. The compute component carries an inline +# workload: block, which the compiler lowers to a WORKLOAD section; a synthetic +# main applies each param unless the command line overrode it. num_messages: 2 +# is distinctive (the model's built-in default is 20), so a server's finalize +# line ("... sent_count 2 recvd_count ...") shows the config value took effect, +# and passing --num_messages=1 shows the command line winning over the config. +# +# A small regular dragonfly (num_routers: 4 -> 72 compute nodes) keeps the run +# cheap. The finite num_messages drains the event queue, so no end time is set. + +components: + compute_host: + model: nw-lp + workload: + type: synthetic + traffic: uniform + num_messages: 2 + arrival_time: "500ns" + +topology: + format: parametric + fabric: + model: dragonfly + shape: + num_routers: 4 + links: + local: { bandwidth: 5.25, vc_size: 4096 } + global: { bandwidth: 4.7, vc_size: 8192 } + cn: { bandwidth: 5.25, vc_size: 4096 } + routing: + algorithm: adaptive + packet_size: 512 + chunk_size: 32 + num_vcs: 1 + modelnet_scheduler: fcfs + message_size: 512 + hosts: + component: compute_host diff --git a/tests/config-tree-equivalence-test.c b/tests/config-tree-equivalence-test.c new file mode 100644 index 00000000..9584e6a4 --- /dev/null +++ b/tests/config-tree-equivalence-test.c @@ -0,0 +1,92 @@ +/* + * Copyright (C) 2013 University of Chicago. + * See COPYRIGHT notice in top-level directory. + * + */ + +/* + * config-tree-equivalence-test [] + * + * With two configs: loads both through the production loader (configuration_load, + * so a .conf goes through the legacy text parser and a .yaml/.json through the + * YAML front-end, including collective include resolution) and asserts the two + * resulting config trees are equal with cf_equal. + * + * This is a cheap, per-key check: it never runs a model, so unlike the lp-io / + * marker equivalence tests it compares *every* section, key and value between a + * .conf and its YAML twin -- catching a defaulted or drifted key that a short + * behavioral run can mask -- at a fraction of the cost. configuration_load uses + * MPI (collective reads), so the binary initializes MPI and runs as a singleton; + * the comparison itself is identical on every rank. + * + * With one config: just loads it (a config linter / no-run load). Combined with + * CODES_RESOLVED_CONFIG_DUMP this prints the resolved config tree without running + * a model, which is how the resolved-config-dump test exercises that knob. + * + * Exit status: 0 on success (one config loaded, or two equal trees), 1 if two + * trees differ (cf_equal_report has printed which section/key diverged), 2 on a + * usage or load error. + */ + +#include +#include + +#include +#include + +int main(int argc, char** argv) { + MPI_Init(&argc, &argv); + + int rank = 0; + MPI_Comm_rank(MPI_COMM_WORLD, &rank); + + if (argc != 2 && argc != 3) { + if (rank == 0) + fprintf(stderr, "usage: %s []\n", argv[0]); + MPI_Finalize(); + return 2; + } + + ConfigHandle ha = NULL; + ConfigHandle hb = NULL; + int rc = 0; + + if (configuration_load(argv[1], MPI_COMM_WORLD, &ha) != 0 || ha == NULL) { + fprintf(stderr, "config-tree-equivalence: could not load \"%s\"\n", argv[1]); + rc = 2; + goto done; + } + + /* One config: load-only (a linter; with CODES_RESOLVED_CONFIG_DUMP set, the + * loader dumps the resolved tree). */ + if (argc == 2) { + if (rank == 0) + printf("config loaded: %s\n", argv[1]); + rc = 0; + goto done; + } + + if (configuration_load(argv[2], MPI_COMM_WORLD, &hb) != 0 || hb == NULL) { + fprintf(stderr, "config-tree-equivalence: could not load \"%s\"\n", argv[2]); + rc = 2; + goto done; + } + + if (cf_equal_report(ha, hb, stderr)) { + if (rank == 0) + printf("config trees identical:\n A = %s\n B = %s\n", argv[1], argv[2]); + rc = 0; + } else { + fprintf(stderr, "config-tree-equivalence: trees differ:\n A = %s\n B = %s\n", argv[1], + argv[2]); + rc = 1; + } + +done: + if (ha) + cf_free(ha); + if (hb) + cf_free(hb); + MPI_Finalize(); + return rc; +}