From d8c56d53e80abcb5cbd49afc4622eaa302450bb5 Mon Sep 17 00:00:00 2001 From: lorenzg Date: Fri, 14 Aug 2026 11:04:35 +0200 Subject: [PATCH 1/6] add check_multi --- docs/checks/commands/check_multi.md | 74 +++++++ packaging/snclient.ini | 8 + pkg/snclient/check_multi.go | 301 ++++++++++++++++++++++++++++ pkg/snclient/check_multi_test.go | 220 ++++++++++++++++++++ pkg/snclient/checkdata.go | 78 +++++-- pkg/snclient/config.go | 36 +++- 6 files changed, 693 insertions(+), 24 deletions(-) create mode 100644 docs/checks/commands/check_multi.md create mode 100644 pkg/snclient/check_multi.go create mode 100644 pkg/snclient/check_multi_test.go diff --git a/docs/checks/commands/check_multi.md b/docs/checks/commands/check_multi.md new file mode 100644 index 00000000..68dda21b --- /dev/null +++ b/docs/checks/commands/check_multi.md @@ -0,0 +1,74 @@ +--- +title: multi +--- + +## check_multi + +Runs multiple checks and aggregates their status, output and performance data. + +In order to use this plugin, you need to enable 'CheckMulti' in the '[/modules]' section of the snclient.ini. + +- [Examples](#examples) +- [Argument Defaults](#argument-defaults) +- [Attributes](#attributes) + +## Implementation + +| Windows | Linux | FreeBSD | MacOSX | +|:------------------:|:------------------:|:------------------:|:------------------:| +| :white_check_mark: | :white_check_mark: | :white_check_mark: | :white_check_mark: | + +## Examples + +### Inline Checks + + check_multi "check=check_process process=123" "check=check_process process=345" "warn=none" "crit=ok_count ne 2" + OK - 2 plugins checked, 2 ok + +### Config Section Checks + +Define checks in `snclient.ini` under `[/settings/check/multi/]`: + + [/settings/check/multi/mycheck] + check_process process=123 + check_process process=345 + +Run the configured multi check: + + check_multi "config=mycheck" "warn=none" "crit=ok_count ne 2" + OK - 2 plugins checked, 2 ok + +### External Script Config + + [/settings/check/multi/custom] + /opt/script/test.sh -H 123 + /opt/script/test2.sh -W 123 + +Run the configured multi check: + + check_multi "config=custom" "warn=problem_count gt 0" + +## Check Specific Arguments + +| Argument | Default | Description | +| --- | --- | --- | +| check | | Inline check command to execute (can be specified multiple times) | +| config | | Config section name under `/settings/check/multi/` to execute | + +## Attributes + +| Filter / Threshold | Default | Description | +| --- | --- | --- | +| warn | `warning_count > 0` | Warning threshold | +| crit | `critical_count > 0 \|\| unknown_count > 0` | Critical threshold | +| count | | Total number of checks executed | +| ok_count | | Number of checks in OK state | +| warning_count | | Number of checks in WARNING state | +| critical_count | | Number of checks in CRITICAL state | +| unknown_count | | Number of checks in UNKNOWN state | +| problem_count | | Number of checks in non-OK state | +| name | | Name/tag of the check | +| command | | Command executed | +| state | | Exit code of the check (0=OK, 1=WARNING, 2=CRITICAL, 3=UNKNOWN) | +| status | | Status text of the check (OK, WARNING, CRITICAL, UNKNOWN) | +| output | | Output of the check | diff --git a/packaging/snclient.ini b/packaging/snclient.ini index 51432c9b..3d8e20da 100644 --- a/packaging/snclient.ini +++ b/packaging/snclient.ini @@ -70,6 +70,9 @@ CheckWMI = disabled ; CheckLogFile - Controls whether check_logfile is allowed or not. CheckLogFile = disabled +; CheckMulti - Controls whether check_multi is allowed or not. +CheckMulti = enabled + [/settings/default] ; allowed hosts - Comma separated list of ips/networks/hostname allowed to connect. @@ -357,6 +360,11 @@ allowed pattern += /var/log/snclient/snclient.log max lines per file limit = 1000000 +[/settings/check/multi] +; max checks - Maximum number of checks check_multi can execute. +max checks = 20 + + ; External script settings - General settings for the external scripts module (CheckExternalScripts). [/settings/external scripts] diff --git a/pkg/snclient/check_multi.go b/pkg/snclient/check_multi.go new file mode 100644 index 00000000..cee23603 --- /dev/null +++ b/pkg/snclient/check_multi.go @@ -0,0 +1,301 @@ +package snclient + +import ( + "context" + "fmt" + "strings" + + "github.com/consol-monitoring/snclient/pkg/utils" +) + +func init() { + AvailableChecks["check_multi"] = CheckEntry{"check_multi", NewCheckMulti} +} + +type CheckMulti struct { + checks []string + config string +} + +func NewCheckMulti() CheckHandler { + return &CheckMulti{ + checks: make([]string, 0), + } +} + +func (l *CheckMulti) Build() *CheckData { + return &CheckData{ + name: "check_multi", + description: "Runs multiple checks and aggregates their status, output and performance data.", + implemented: ALL, + disableFilter: true, + result: &CheckResult{ + State: CheckExitOK, + }, + args: map[string]CheckArgument{ + "check": {value: &l.checks, description: "Inline check command to execute (can be specified multiple times)"}, + "config": {value: &l.config, description: "Config section name under /settings/check/multi/ to execute"}, + }, + conditionAlias: map[string]map[string]string{ + "warning_count": {"warn_count": "warning_count"}, + "critical_count": {"crit_count": "critical_count"}, + }, + attributes: []CheckAttribute{ + {name: "count", description: "Total number of checks executed", unit: UNone}, + {name: "ok_count", description: "Number of checks in OK state", unit: UNone}, + {name: "warning_count", description: "Number of checks in WARNING state", unit: UNone}, + {name: "critical_count", description: "Number of checks in CRITICAL state", unit: UNone}, + {name: "unknown_count", description: "Number of checks in UNKNOWN state", unit: UNone}, + {name: "problem_count", description: "Number of checks in non-OK state", unit: UNone}, + {name: "name", description: "Name/tag of the check", unit: UNone}, + {name: "command", description: "Command executed", unit: UNone}, + {name: "state", description: "Exit code of the check (0=OK, 1=WARNING, 2=CRITICAL, 3=UNKNOWN)", unit: UNone}, + {name: "status", description: "Status text of the check (OK, WARNING, CRITICAL, UNKNOWN)", unit: UNone}, + {name: "output", description: "Output of the check", unit: UNone}, + }, + defaultWarning: "warning_count > 0", + defaultCritical: "critical_count > 0 || unknown_count > 0", + okSyntax: "%(status) - %(count) plugins checked, %(ok_count) ok", + topSyntax: "%(status) - %(count) plugins checked: %(ok_count) ok, %(warning_count) warning, %(critical_count) critical, %(unknown_count) unknown%(problem_list)", + detailSyntax: "[%(status)] %(name): %(output)", + emptySyntax: "%(status) - no checks executed", + emptyState: CheckExitUnknown, + exampleDefault: ` + check_multi "check=check_process process=123" "check=check_process process=345" "warn=none" "crit=ok_count ne 2" + OK - 2 plugins checked, 2 ok + `, + } +} + +type multiChildCheck struct { + tag string + cmdStr string + isInline bool +} + +func (l *CheckMulti) Check(ctx context.Context, snc *Agent, check *CheckData, _ []Argument) (*CheckResult, error) { + enabled, _, _ := snc.config.Section("/modules").GetBool("CheckMulti") + if !enabled { + return &CheckResult{ + State: CheckExitUnknown, + Output: "module CheckMulti is not enabled in /modules section", + }, nil + } + + maxChecks, ok, err := snc.config.Section("/settings/check/multi").GetInt("max checks") + if err != nil || !ok || maxChecks <= 0 { + maxChecks = 20 + } + + childChecks, res := l.buildChildChecks(snc) + if res != nil { + return res, nil + } + + if len(childChecks) == 0 { + return &CheckResult{ + State: CheckExitUnknown, + Output: "no checks or config specified", + }, nil + } + + if int64(len(childChecks)) > maxChecks { + return &CheckResult{ + State: CheckExitUnknown, + Output: fmt.Sprintf("number of checks (%d) exceeds max checks limit (%d)", len(childChecks), maxChecks), + }, nil + } + + return l.executeChildChecks(ctx, snc, check, childChecks) +} + +// buildChildChecks assembles the list of child checks from config section and inline args. +func (l *CheckMulti) buildChildChecks(snc *Agent) ([]multiChildCheck, *CheckResult) { + childChecks := []multiChildCheck{} + + if l.config != "" { + configChecks, res := l.buildConfigChecks(snc) + if res != nil { + return nil, res + } + childChecks = append(childChecks, configChecks...) + } + + for _, inlineCmd := range l.checks { + inlineCmd = strings.TrimSpace(inlineCmd) + if inlineCmd == "" { + continue + } + childChecks = append(childChecks, multiChildCheck{ + tag: "", + cmdStr: inlineCmd, + isInline: true, + }) + } + + return childChecks, nil +} + +// buildConfigChecks loads checks from the named config section. +func (l *CheckMulti) buildConfigChecks(snc *Agent) ([]multiChildCheck, *CheckResult) { + secName := "/settings/check/multi/" + l.config + sec := snc.config.Section(secName) + + if len(sec.keys) == 0 { + return nil, &CheckResult{ + State: CheckExitUnknown, + Output: fmt.Sprintf("no checks defined in config section %s", secName), + } + } + + childChecks := make([]multiChildCheck, 0, len(sec.keys)) + + for _, key := range sec.keys { + rawCmd, tag := l.resolveConfigEntry(snc, key, sec.data[key]) + childChecks = append(childChecks, multiChildCheck{ + tag: tag, + cmdStr: rawCmd, + isInline: false, + }) + } + + return childChecks, nil +} + +// resolveConfigEntry determines the raw command and tag for a single config section entry. +func (l *CheckMulti) resolveConfigEntry(snc *Agent, key, val string) (rawCmd, tag string) { + rawCmd = key + tag = "" + + if val == "" { + return rawCmd, tag + } + + if _, isKnown := snc.getCheck(key, false); isKnown { + return key + " " + val, key + } + + return val, key +} + +// executeChildChecks runs all child checks and aggregates results. +func (l *CheckMulti) executeChildChecks(ctx context.Context, snc *Agent, check *CheckData, childChecks []multiChildCheck) (*CheckResult, error) { + var count, okCount, warnCount, critCount, unknownCount int64 + + detailsList := make([]string, 0, len(childChecks)) + allMetrics := make([]*CheckMetric, 0) + + for idx, chk := range childChecks { + res, fatal := l.runChildCheck(ctx, snc, check, chk) + if fatal { + return res, nil + } + + count++ + switch res.State { + case CheckExitOK: + okCount++ + case CheckExitWarning: + warnCount++ + case CheckExitCritical: + critCount++ + default: + unknownCount++ + } + + tokens := utils.Tokenize(chk.cmdStr) + cmdName := chk.cmdStr + if len(tokens) > 0 { + cmdName = tokens[0] + } + + tag := chk.tag + if tag == "" { + tag = cmdName + } + + firstLine := strings.TrimSpace(strings.Split(res.Output, "\n")[0]) + detailsList = append(detailsList, fmt.Sprintf("[% 2d] %s %s", idx+1, tag, res.Output)) + + entry := map[string]string{ + "idx": fmt.Sprintf("%d", idx+1), + "name": tag, + "command": chk.cmdStr, + "state": fmt.Sprintf("%d", res.State), + "status": res.StateString(), + "output": firstLine, + "_state": fmt.Sprintf("%d", res.State), + "_count": "1", + } + check.listData = append(check.listData, entry) + + for _, m := range res.Metrics { + metricCopy := *m + metricCopy.Name = fmt.Sprintf("%s::%s", tag, m.Name) + allMetrics = append(allMetrics, &metricCopy) + } + } + + problemCount := warnCount + critCount + unknownCount + check.details = map[string]string{ + "count": fmt.Sprintf("%d", count), + "ok_count": fmt.Sprintf("%d", okCount), + "warning_count": fmt.Sprintf("%d", warnCount), + "warn_count": fmt.Sprintf("%d", warnCount), + "critical_count": fmt.Sprintf("%d", critCount), + "crit_count": fmt.Sprintf("%d", critCount), + "unknown_count": fmt.Sprintf("%d", unknownCount), + "problem_count": fmt.Sprintf("%d", problemCount), + } + + check.result.Metrics = allMetrics + check.result.Details = strings.Join(detailsList, "\n") + + return check.Finalize() +} + +// runChildCheck executes a single child check and returns its result. +// The second return value is true when the error is fatal and the caller should stop processing. +func (l *CheckMulti) runChildCheck(ctx context.Context, snc *Agent, check *CheckData, chk multiChildCheck) (*CheckResult, bool) { + tokens := utils.Tokenize(chk.cmdStr) + tokens, err := utils.TrimQuotesList(tokens) + + if err != nil || len(tokens) == 0 { + return &CheckResult{ + State: CheckExitUnknown, + Output: fmt.Sprintf("failed to parse check command: %s", chk.cmdStr), + }, true + } + + cmdName := tokens[0] + cmdArgs := tokens[1:] + + _, isKnown := snc.getCheck(cmdName, false) + + if chk.isInline && !isKnown { + return &CheckResult{ + State: CheckExitUnknown, + Output: fmt.Sprintf("unknown check command: %s (inline checks only support existing check commands)", cmdName), + }, true + } + + if isKnown { + return snc.RunCheckWithContext(ctx, cmdName, cmdArgs, 0, nil, false), false + } + + stdout, stderr, exitCode, _ := snc.runExternalCheckString(ctx, chk.cmdStr, int64(check.timeout)) + out := stdout + if stderr != "" { + if out != "" { + out += "\n" + } + out += "[" + stderr + "]" + } + res := &CheckResult{ + State: exitCode, + Output: out, + } + res.ParsePerformanceDataFromOutput() + + return res, false +} diff --git a/pkg/snclient/check_multi_test.go b/pkg/snclient/check_multi_test.go new file mode 100644 index 00000000..885bfb81 --- /dev/null +++ b/pkg/snclient/check_multi_test.go @@ -0,0 +1,220 @@ +package snclient + +import ( + "fmt" + "os" + "path/filepath" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestCheckMultiInline(t *testing.T) { + config := ` +[/modules] +CheckMulti = enabled +` + snc := StartTestAgent(t, config) + defer StopTestAgent(t, snc) + + // 1. Basic inline checks - all OK + res := snc.RunCheck("check_multi", []string{ + "check=check_dummy 0 'dummy ok 1'", + "check=check_dummy 0 'dummy ok 2'", + }) + assert.Equalf(t, CheckExitOK, res.State, "state OK") + assert.Contains(t, res.Output, "2 plugins checked, 2 ok") + assert.Contains(t, res.Details, "dummy ok 1") + assert.Contains(t, res.Details, "dummy ok 2") + + // 2. Inline checks with warning and critical (default thresholds) + res = snc.RunCheck("check_multi", []string{ + "check=check_dummy 0 'dummy ok'", + "check=check_dummy 1 'dummy warn'", + }) + assert.Equalf(t, CheckExitWarning, res.State, "state WARNING") + assert.Contains(t, res.Output, "2 plugins checked: 1 ok, 1 warning, 0 critical, 0 unknown") + + res = snc.RunCheck("check_multi", []string{ + "check=check_dummy 0 'dummy ok'", + "check=check_dummy 2 'dummy crit'", + }) + assert.Equalf(t, CheckExitCritical, res.State, "state CRITICAL") + assert.Contains(t, res.Output, "2 plugins checked: 1 ok, 0 warning, 1 critical, 0 unknown") + + // 3. Custom conditions: warn=none crit=ok_count ne 2 + res = snc.RunCheck("check_multi", []string{ + "check=check_dummy 0 'dummy 1'", + "check=check_dummy 0 'dummy 2'", + "warn=none", + "crit=ok_count ne 2", + }) + assert.Equalf(t, CheckExitOK, res.State, "state OK when ok_count == 2") + + res = snc.RunCheck("check_multi", []string{ + "check=check_dummy 0 'dummy 1'", + "check=check_dummy 1 'dummy 2'", + "warn=none", + "crit=ok_count ne 2", + }) + assert.Equalf(t, CheckExitCritical, res.State, "state CRITICAL when ok_count != 2") + + // 4. Custom conditions: warn=problem_count gt 0 + res = snc.RunCheck("check_multi", []string{ + "check=check_dummy 0 'dummy 1'", + "check=check_dummy 1 'dummy 2'", + "warn=problem_count gt 0", + "crit=none", + }) + assert.Equalf(t, CheckExitWarning, res.State, "state WARNING when problem_count > 0") + + // 5. Unknown/inline checks restriction (cannot run arbitrary external commands inline) + res = snc.RunCheck("check_multi", []string{ + "check=/bin/nonexistent_or_external_script -H 123", + }) + assert.Equalf(t, CheckExitUnknown, res.State, "state UNKNOWN for unregistered inline command") + assert.Contains(t, res.Output, "unknown check command") + + // 6. Inline check with check_process or check_cpu + res = snc.RunCheck("check_multi", []string{ + "check=check_cpu warn=load=101 crit=load=102", + "warn=none", + "crit=ok_count ne 1", + }) + assert.Equalf(t, CheckExitOK, res.State, "state OK for check_cpu inline") + assert.Contains(t, res.Details, "check_cpu") + + // 7. Filter argument is disabled/rejected + res = snc.RunCheck("check_multi", []string{ + "check=check_dummy 0 'ok'", + "filter=state=1", + }) + assert.Equalf(t, CheckExitUnknown, res.State, "state UNKNOWN when filter argument is used") + assert.Contains(t, res.Output, "filter is disabled for this check") +} + +func TestCheckMultiLimits(t *testing.T) { + config := ` +[/modules] +CheckMulti = enabled + +[/settings/check/multi] +max checks = 2 +` + snc := StartTestAgent(t, config) + defer StopTestAgent(t, snc) + + // Under limit: 2 checks + res := snc.RunCheck("check_multi", []string{ + "check=check_dummy 0 'ok 1'", + "check=check_dummy 0 'ok 2'", + }) + assert.Equalf(t, CheckExitOK, res.State, "state OK for 2 checks") + + // Exceeds limit: 3 checks + res = snc.RunCheck("check_multi", []string{ + "check=check_dummy 0 'ok 1'", + "check=check_dummy 0 'ok 2'", + "check=check_dummy 0 'ok 3'", + }) + assert.Equalf(t, CheckExitUnknown, res.State, "state UNKNOWN when exceeding max checks") + assert.Contains(t, res.Output, "exceeds max checks limit") +} + +func TestCheckMultiDisabled(t *testing.T) { + config := ` +[/modules] +CheckMulti = disabled +` + snc := StartTestAgent(t, config) + defer StopTestAgent(t, snc) + + res := snc.RunCheck("check_multi", []string{ + "check=check_dummy 0 'ok 1'", + }) + assert.Equalf(t, CheckExitUnknown, res.State, "state UNKNOWN when module is disabled") + assert.Contains(t, res.Output, "module CheckMulti is not enabled") +} + +func TestCheckMultiConfigSection(t *testing.T) { + // Create a temporary shell script to test external scripts in config + tmpDir := t.TempDir() + script1 := filepath.Join(tmpDir, "test1.sh") + script2 := filepath.Join(tmpDir, "test2.sh") + + err := os.WriteFile(script1, []byte("#!/bin/sh\necho \"SCRIPT 1 OK | perf1=10;20;30\"\nexit 0\n"), 0o600) + require.NoError(t, err) + require.NoError(t, os.Chmod(script1, 0o700)) + err = os.WriteFile(script2, []byte("#!/bin/sh\necho \"SCRIPT 2 WARNING | perf2=50;40;60\"\nexit 1\n"), 0o600) + require.NoError(t, err) + require.NoError(t, os.Chmod(script2, 0o700)) + + config := fmt.Sprintf(` +[/modules] +CheckMulti = enabled + +[/settings/check/multi/mycheck] +check_dummy 0 ok1 +check_dummy 0 ok2 + +[/settings/check/multi/custom] +%s -H 123 +%s -W 123 + +[/settings/check/multi/named] +first = check_dummy 0 ok_first +second = %s -H 456 +`, script1, script2, script1) + + snc := StartTestAgent(t, config) + defer StopTestAgent(t, snc) + + // Test config=mycheck (builtin checks in config) + res := snc.RunCheck("check_multi", []string{ + "config=mycheck", + "warn=none", + "crit=ok_count ne 2", + }) + assert.Equalf(t, CheckExitOK, res.State, "state OK for mycheck config") + assert.Contains(t, res.Output, "2 plugins checked, 2 ok") + + // Test config=custom (external scripts in config) + res = snc.RunCheck("check_multi", []string{ + "config=custom", + "warn=problem_count gt 0", + "crit=none", + }) + assert.Equalf(t, CheckExitWarning, res.State, "state WARNING for custom config") + assert.Contains(t, res.Output, "2 plugins checked: 1 ok, 1 warning, 0 critical, 0 unknown") + assert.Contains(t, res.Details, "SCRIPT 1 OK") + assert.Contains(t, res.Details, "SCRIPT 2 WARNING") + + // Test config=named (named check tags in config) + res = snc.RunCheck("check_multi", []string{ + "config=named", + }) + assert.Equalf(t, CheckExitOK, res.State, "state OK for named config") + assert.Contains(t, res.Details, "first") + assert.Contains(t, res.Details, "second") + + // Test non-existing config + res = snc.RunCheck("check_multi", []string{ + "config=doesnotexist", + }) + assert.Equalf(t, CheckExitUnknown, res.State, "state UNKNOWN for missing config section") + assert.Contains(t, res.Output, "no checks defined in config section") +} + +func TestCheckMultiIndex(t *testing.T) { + config := ` +[/modules] +CheckMulti = enabled +` + snc := StartTestAgent(t, config) + defer StopTestAgent(t, snc) + + res := snc.RunCheck("check_index", []string{"filter=name = 'check_multi'"}) + assert.Equalf(t, CheckExitOK, res.State, "state OK for check_index") + assert.Contains(t, res.Output, "check_multi") +} diff --git a/pkg/snclient/checkdata.go b/pkg/snclient/checkdata.go index 9729acba..aff2ed01 100644 --- a/pkg/snclient/checkdata.go +++ b/pkg/snclient/checkdata.go @@ -102,6 +102,7 @@ type CheckData struct { defaultFilter string conditionAlias map[string]map[string]string // replacement map of equivalent condition values conditionColAlias map[string][]string // if there are filter for given column, apply to alias columns too + disableFilter bool // disable filter argument for checks where filtering listData makes no sense args map[string]CheckArgument extraArgs map[string]CheckArgument // internal, map of expanded args argsPassthrough bool // allow arbitrary arguments without complaining about unknown argument @@ -280,10 +281,12 @@ func (cd *CheckData) buildListMacros() map[string]string { okList := make([]string, 0) warnList := make([]string, 0) critList := make([]string, 0) + unknownList := make([]string, 0) count := int64(0) okCount := int64(0) warnCount := int64(0) critCount := int64(0) + unknownCount := int64(0) for _, entry := range cd.listData { weight := int64(1) if w, ok := entry["_count"]; ok { @@ -308,6 +311,9 @@ func (cd *CheckData) buildListMacros() map[string]string { case "2": critList = append(critList, expanded) critCount += weight + case "3": + unknownList = append(unknownList, expanded) + unknownCount += weight } } @@ -315,17 +321,21 @@ func (cd *CheckData) buildListMacros() map[string]string { cd.listCombine = ", " } result := map[string]string{ - "count": fmt.Sprintf("%d", count), - "list": strings.Join(list, cd.listCombine), - "ok_count": fmt.Sprintf("%d", okCount), - "ok_list": "", - "warn_count": fmt.Sprintf("%d", warnCount), - "warn_list": "", - "crit_count": fmt.Sprintf("%d", critCount), - "crit_list": "", - "problem_count": fmt.Sprintf("%d", warnCount+critCount), - "problem_list": "", - "detail_list": "", + "count": fmt.Sprintf("%d", count), + "list": strings.Join(list, cd.listCombine), + "ok_count": fmt.Sprintf("%d", okCount), + "ok_list": "", + "warn_count": fmt.Sprintf("%d", warnCount), + "warning_count": fmt.Sprintf("%d", warnCount), + "warn_list": "", + "crit_count": fmt.Sprintf("%d", critCount), + "critical_count": fmt.Sprintf("%d", critCount), + "crit_list": "", + "unknown_count": fmt.Sprintf("%d", unknownCount), + "unknown_list": "", + "problem_count": fmt.Sprintf("%d", warnCount+critCount+unknownCount), + "problem_list": "", + "detail_list": "", } problemList := []string{} @@ -342,6 +352,11 @@ func (cd *CheckData) buildListMacros() map[string]string { problemList = append(problemList, result["warn_list"]) detailList = append(detailList, result["warn_list"]) } + if len(unknownList) > 0 { + result["unknown_list"] = "unknown(" + strings.Join(unknownList, cd.listCombine) + ")" + problemList = append(problemList, result["unknown_list"]) + detailList = append(detailList, result["unknown_list"]) + } if len(okList) > 0 { result["ok_list"] = strings.Join(okList, cd.listCombine) detailList = append(detailList, result["ok_list"]) @@ -363,17 +378,21 @@ func (cd *CheckData) buildListMacrosFromSingleEntry() map[string]string { } result := map[string]string{ - "count": "1", - "list": expanded, - "ok_count": "0", - "ok_list": "", - "warn_count": "0", - "warn_list": "", - "crit_count": "0", - "crit_list": "", - "problem_count": "0", - "problem_list": "", - "detail_list": expanded, + "count": "1", + "list": expanded, + "ok_count": "0", + "ok_list": "", + "warn_count": "0", + "warning_count": "0", + "warn_list": "", + "crit_count": "0", + "critical_count": "0", + "crit_list": "", + "unknown_count": "0", + "unknown_list": "", + "problem_count": "0", + "problem_list": "", + "detail_list": expanded, } numWarn := 0 @@ -386,12 +405,21 @@ func (cd *CheckData) buildListMacrosFromSingleEntry() map[string]string { result["problem_list"] = expanded result["warn_list"] = expanded result["warn_count"] = "1" + result["warning_count"] = "1" + result["problem_count"] = "1" numWarn = 1 case "2": result["problem_list"] = expanded result["crit_list"] = expanded result["crit_count"] = "1" + result["critical_count"] = "1" + result["problem_count"] = "1" numCrit = 1 + case "3": + result["problem_list"] = expanded + result["unknown_list"] = expanded + result["unknown_count"] = "1" + result["problem_count"] = "1" } cd.buildCountMetrics(1, numCrit, numWarn) @@ -731,6 +759,9 @@ func (cd *CheckData) processArgs(sanitized []Argument, defaultWarning, defaultCr } cd.critThreshold = append(cd.critThreshold, cond) case "filter+": + if cd.disableFilter { + return nil, false, fmt.Errorf("%s is disabled for this check", keyword) + } applyDefaultFilter = false filter, err2 := cd.appendDefaultThreshold(keyword, argValue, cd.defaultFilter, cd.filter) if err2 != nil { @@ -738,6 +769,9 @@ func (cd *CheckData) processArgs(sanitized []Argument, defaultWarning, defaultCr } cd.filter = filter case "filter": + if cd.disableFilter { + return nil, false, fmt.Errorf("%s is disabled for this check", keyword) + } applyDefaultFilter = false cond, err2 := NewCondition(argValue, &cd.attributes) if err2 != nil { diff --git a/pkg/snclient/config.go b/pkg/snclient/config.go index ea0975e7..b2afb2a7 100644 --- a/pkg/snclient/config.go +++ b/pkg/snclient/config.go @@ -323,16 +323,44 @@ func (config *Config) ParseINI(configData, iniPath string, snc *Agent) error { continue } + isMultiSection := strings.HasPrefix(currentSection.name, "/settings/check/multi/") + // parse key and value val := strings.SplitN(line, "=", 2) + + // bare line (no '='): only allowed in check/multi sections, treated as raw command if len(val) < 2 { - parseErrors = append(parseErrors, fmt.Errorf("parse error in %s:%d: found key without '='", iniPath, lineNr)) + if !isMultiSection { + parseErrors = append(parseErrors, fmt.Errorf("parse error in %s:%d: found key without '='", iniPath, lineNr)) + + continue + } + if err := currentSection.SetRaw(line, ""); err != nil { + parseErrors = append(parseErrors, fmt.Errorf("config error in %s:%d: %s", iniPath, lineNr, err.Error())) + } + if len(currentComments) > 0 { + currentSection.comments[line] = currentComments + currentComments = make([]string, 0) + } continue } val[0] = strings.TrimSpace(val[0]) val[1] = strings.TrimSpace(val[1]) + // key contains space (e.g. 'check_process process=123'): also a raw command line in check/multi sections + if isMultiSection && strings.Contains(val[0], " ") { + if err := currentSection.SetRaw(line, ""); err != nil { + parseErrors = append(parseErrors, fmt.Errorf("config error in %s:%d: %s", iniPath, lineNr, err.Error())) + } + if len(currentComments) > 0 { + currentSection.comments[line] = currentComments + currentComments = make([]string, 0) + } + + continue + } + // silently skip UNKNOWN values which were placeholder in nsclient if val[1] == "UNKNOWN" { continue @@ -794,7 +822,11 @@ func (cs *ConfigSection) String() string { // none-multiline entries case 0, 1: if val == "" { - data = append(data, fmt.Sprintf("%s =", key)) + if strings.HasPrefix(cs.name, "/settings/check/multi/") { + data = append(data, key) + } else { + data = append(data, fmt.Sprintf("%s =", key)) + } } else { data = append(data, fmt.Sprintf("%s = %s", key, strings.Join(raw, ""))) } From 15e36f1591f559d8bc77f8bae510c918544409a6 Mon Sep 17 00:00:00 2001 From: lorenzg Date: Fri, 14 Aug 2026 12:02:49 +0200 Subject: [PATCH 2/6] fix test, problem_count should be 1 here --- pkg/snclient/check_files_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/snclient/check_files_test.go b/pkg/snclient/check_files_test.go index fa9ffe7e..243e76dd 100644 --- a/pkg/snclient/check_files_test.go +++ b/pkg/snclient/check_files_test.go @@ -76,7 +76,7 @@ func TestCheckFiles(t *testing.T) { res = snc.RunCheck("check_files", []string{"path=./t/checksum.txt", "crit=md5_checksum != 3687C5D7106484CD61CDE867A2A999FA"}) assert.Equalf(t, CheckExitCritical, res.State, "CRITICAL") - assert.Contains(t, string(res.BuildPluginOutput()), "0/1 files") + assert.Contains(t, string(res.BuildPluginOutput()), "1/1 files") res = snc.RunCheck("check_files", []string{"path=./t/checksum.txt", "crit=sha1_checksum == 4EE4BFE9AA51E56A7BD5CCF4785C35A27EE022F8"}) assert.Equalf(t, CheckExitOK, res.State, "state OK") From c8b007d6f4191da7c9e4f540448c82622a740833 Mon Sep 17 00:00:00 2001 From: lorenzg Date: Fri, 14 Aug 2026 13:25:04 +0200 Subject: [PATCH 3/6] fix windows tests --- README.md | 1 + pkg/snclient/check_multi_test.go | 42 ++++++++++++++++++++++++++------ 2 files changed, 36 insertions(+), 7 deletions(-) diff --git a/README.md b/README.md index 9b7586dc..f1df62ec 100644 --- a/README.md +++ b/README.md @@ -76,6 +76,7 @@ Further details are covered in the [documentation](https://omd.consol.de/docs/sn | **check_mailq** | | X | X | X | | **check_memory** | X | X | X | X | | **check_mount** | X | X | X | X | +| **check_multi** | X | X | X | X | | **check_network** | X | X | X | X | | **check_nsc_web** | X | X | X | X | | **check_ntp_offset** | X | X | X | X | diff --git a/pkg/snclient/check_multi_test.go b/pkg/snclient/check_multi_test.go index 885bfb81..ca7537b0 100644 --- a/pkg/snclient/check_multi_test.go +++ b/pkg/snclient/check_multi_test.go @@ -4,6 +4,7 @@ import ( "fmt" "os" "path/filepath" + "runtime" "testing" "github.com/stretchr/testify/assert" @@ -138,17 +139,44 @@ CheckMulti = disabled } func TestCheckMultiConfigSection(t *testing.T) { - // Create a temporary shell script to test external scripts in config + // Create temporary scripts to test external scripts in config tmpDir := t.TempDir() - script1 := filepath.Join(tmpDir, "test1.sh") - script2 := filepath.Join(tmpDir, "test2.sh") + var scriptExt string + var script1Content, script2Content string - err := os.WriteFile(script1, []byte("#!/bin/sh\necho \"SCRIPT 1 OK | perf1=10;20;30\"\nexit 0\n"), 0o600) + if runtime.GOOS == "windows" { + scriptExt = ".ps1" + script1Content = `Write-Output "SCRIPT 1 OK | perf1=10;20;30" +exit 0 +` + script2Content = `Write-Output "SCRIPT 2 WARNING | perf2=50;40;60" +exit 1 +` + } else { + scriptExt = ".sh" + script1Content = `#!/bin/sh +echo "SCRIPT 1 OK | perf1=10;20;30" +exit 0 +` + script2Content = `#!/bin/sh +echo "SCRIPT 2 WARNING | perf2=50;40;60" +exit 1 +` + } + + script1 := filepath.Join(tmpDir, "test1"+scriptExt) + script2 := filepath.Join(tmpDir, "test2"+scriptExt) + + err := os.WriteFile(script1, []byte(script1Content), 0o600) require.NoError(t, err) - require.NoError(t, os.Chmod(script1, 0o700)) - err = os.WriteFile(script2, []byte("#!/bin/sh\necho \"SCRIPT 2 WARNING | perf2=50;40;60\"\nexit 1\n"), 0o600) + + err = os.WriteFile(script2, []byte(script2Content), 0o600) require.NoError(t, err) - require.NoError(t, os.Chmod(script2, 0o700)) + + if runtime.GOOS != "windows" { + require.NoError(t, os.Chmod(script1, 0o700)) + require.NoError(t, os.Chmod(script2, 0o700)) + } config := fmt.Sprintf(` [/modules] From 4fc5bc2a8819b04134f029ec68e6f81f4c749d05 Mon Sep 17 00:00:00 2001 From: lorenzg Date: Fri, 14 Aug 2026 13:44:53 +0200 Subject: [PATCH 4/6] fix DOC_COMMANDS --- Makefile | 1 + 1 file changed, 1 insertion(+) diff --git a/Makefile b/Makefile index db681a21..d616c7fe 100644 --- a/Makefile +++ b/Makefile @@ -737,6 +737,7 @@ DOC_COMMANDS=\ check_mailq \ check_memory \ check_mount \ + check_multi \ check_network \ check_ntp_offset \ check_omd \ From 461bc15628a28360a3f0321c1ec5244af5ceb613 Mon Sep 17 00:00:00 2001 From: lorenzg Date: Fri, 14 Aug 2026 14:55:30 +0200 Subject: [PATCH 5/6] update docs --- docs/checks/commands/check_multi.md | 112 ++++++++++++++++++---------- pkg/snclient/check_multi.go | 43 +++++++++-- 2 files changed, 111 insertions(+), 44 deletions(-) diff --git a/docs/checks/commands/check_multi.md b/docs/checks/commands/check_multi.md index 68dda21b..53984adb 100644 --- a/docs/checks/commands/check_multi.md +++ b/docs/checks/commands/check_multi.md @@ -6,7 +6,27 @@ title: multi Runs multiple checks and aggregates their status, output and performance data. -In order to use this plugin, you need to enable 'CheckMulti' in the '[/modules]' section of the snclient.ini. + By default 'CheckMulti' is enabled, but you can disable it in the '[/modules]' section of the snclient_local.ini. + You can also set a limit for the number of checks that can be executed in the '[/settings/check/multi]' section + of the snclient_local.ini. + + When using the inline mode, you can only use available commands (run 'check_index' to get a full list). + + You can also define custom check sections in the config file, for example: + [/settings/check/multi/mycheck] + check_process process=123 + check_process process=345 + + This can be executed with 'check_multi "config=mycheck"'. + + It's also possible to use custom scripts in the config section, for example: + [/settings/check/multi/myscript] + /path/to/plugin1 + /path/to/plugin2 + /path/to/plugin3 + + This can be executed with 'check_multi "config=myscript"'. + - [Examples](#examples) - [Argument Defaults](#argument-defaults) @@ -20,55 +40,71 @@ In order to use this plugin, you need to enable 'CheckMulti' in the '[/modules]' ## Examples -### Inline Checks - - check_multi "check=check_process process=123" "check=check_process process=345" "warn=none" "crit=ok_count ne 2" - OK - 2 plugins checked, 2 ok +### Default Check -### Config Section Checks + check_multi "check=check_process 'process=firefox'" "check=check_memory 'crit=used_pct gt 80%'" + OK - 2 plugins checked, 2 ok | 'check_process::count'=1;;;0 'check_process::rss'=258686976B;;;0 ... + [ 1] check_process OK - all 1 processes are ok. + [ 2] check_memory OK - physical = 12.22 GiB/16.00 GiB (76.4%), swap = 1.95 GiB/3.00 GiB (65.0%) -Define checks in `snclient.ini` under `[/settings/check/multi/]`: + You can define warning/critical conditions based on the number of checks in a certain state (see attributes below): - [/settings/check/multi/mycheck] - check_process process=123 - check_process process=345 + check_multi "check=check_dummy 0 'OK'" "check=check_dummy 1 'WARNING'" "critical=problem_count gt 0" + CRITICAL - 2 plugins checked: 1 ok, 1 warning, 0 critical, 0 unknown + [ 1] check_dummy OK + [ 2] check_dummy WARNING -Run the configured multi check: +### Example using NRPE and Naemon - check_multi "config=mycheck" "warn=none" "crit=ok_count ne 2" - OK - 2 plugins checked, 2 ok +Naemon Config -### External Script Config + define command{ + command_name check_nrpe + command_line $USER1$/check_nrpe -H $HOSTADDRESS$ -n -c $ARG1$ -a $ARG2$ + } - [/settings/check/multi/custom] - /opt/script/test.sh -H 123 - /opt/script/test2.sh -W 123 + define service { + host_name testhost + service_description check_multi + use generic-service + check_command check_nrpe!check_multi! + } -Run the configured multi check: +## Argument Defaults - check_multi "config=custom" "warn=problem_count gt 0" +| Argument | Default Value | +| ------------- | ----------------------------------------------------------------------------------------------------- | +| warning | warning_count > 0 | +| critical | critical_count > 0 \|\| unknown_count > 0 | +| empty-state | 3 (UNKNOWN) | +| empty-syntax | %(status) - no checks executed | +| top-syntax | %(status) - %(count) plugins checked: %(ok_count) ok, %(warning_count) warning, %(critical_count) critical, %(unknown_count) unknown%(problem_list) | +| ok-syntax | %(status) - %(count) plugins checked, %(ok_count) ok | +| detail-syntax | [%(status)] %(name): %(output) | ## Check Specific Arguments -| Argument | Default | Description | -| --- | --- | --- | -| check | | Inline check command to execute (can be specified multiple times) | -| config | | Config section name under `/settings/check/multi/` to execute | +| Argument | Description | +| -------- | ------------------------------------------------------------------------ | +| check | Check command to execute (can be specified multiple times) | +| config | Config section name under [/settings/check/multi/< section >] to execute | ## Attributes -| Filter / Threshold | Default | Description | -| --- | --- | --- | -| warn | `warning_count > 0` | Warning threshold | -| crit | `critical_count > 0 \|\| unknown_count > 0` | Critical threshold | -| count | | Total number of checks executed | -| ok_count | | Number of checks in OK state | -| warning_count | | Number of checks in WARNING state | -| critical_count | | Number of checks in CRITICAL state | -| unknown_count | | Number of checks in UNKNOWN state | -| problem_count | | Number of checks in non-OK state | -| name | | Name/tag of the check | -| command | | Command executed | -| state | | Exit code of the check (0=OK, 1=WARNING, 2=CRITICAL, 3=UNKNOWN) | -| status | | Status text of the check (OK, WARNING, CRITICAL, UNKNOWN) | -| output | | Output of the check | +### Filter Keywords + +these can be used in filters and thresholds (along with the default attributes): + +| Attribute | Description | +| -------------- | --------------------------------------------------------------- | +| count | Total number of checks executed | +| ok_count | Number of checks in OK state | +| warning_count | Number of checks in WARNING state | +| critical_count | Number of checks in CRITICAL state | +| unknown_count | Number of checks in UNKNOWN state | +| problem_count | Number of checks in non-OK state | +| name | Name/tag of the check | +| command | Command executed | +| state | Exit code of the check (0=OK, 1=WARNING, 2=CRITICAL, 3=UNKNOWN) | +| status | Status text of the check (OK, WARNING, CRITICAL, UNKNOWN) | +| output | Output of the check | diff --git a/pkg/snclient/check_multi.go b/pkg/snclient/check_multi.go index cee23603..b75c276f 100644 --- a/pkg/snclient/check_multi.go +++ b/pkg/snclient/check_multi.go @@ -25,16 +25,38 @@ func NewCheckMulti() CheckHandler { func (l *CheckMulti) Build() *CheckData { return &CheckData{ - name: "check_multi", - description: "Runs multiple checks and aggregates their status, output and performance data.", + name: "check_multi", + description: `Runs multiple checks and aggregates their status, output and performance data. + + By default 'CheckMulti' is enabled, but you can disable it in the '[/modules]' section of the snclient_local.ini. + You can also set a limit for the number of checks that can be executed in the '[/settings/check/multi]' section + of the snclient_local.ini. + + When using the inline mode, you can only use available commands (run 'check_index' to get a full list). + + You can also define custom check sections in the config file, for example: + [/settings/check/multi/mycheck] + check_process process=123 + check_process process=345 + + This can be executed with 'check_multi "config=mycheck"'. + + It's also possible to use custom scripts in the config section, for example: + [/settings/check/multi/myscript] + /path/to/plugin1 + /path/to/plugin2 + /path/to/plugin3 + + This can be executed with 'check_multi "config=myscript"'. +`, implemented: ALL, disableFilter: true, result: &CheckResult{ State: CheckExitOK, }, args: map[string]CheckArgument{ - "check": {value: &l.checks, description: "Inline check command to execute (can be specified multiple times)"}, - "config": {value: &l.config, description: "Config section name under /settings/check/multi/ to execute"}, + "check": {value: &l.checks, description: "Check command to execute (can be specified multiple times)"}, + "config": {value: &l.config, description: "Config section name under [/settings/check/multi/< section >] to execute"}, }, conditionAlias: map[string]map[string]string{ "warning_count": {"warn_count": "warning_count"}, @@ -61,8 +83,17 @@ func (l *CheckMulti) Build() *CheckData { emptySyntax: "%(status) - no checks executed", emptyState: CheckExitUnknown, exampleDefault: ` - check_multi "check=check_process process=123" "check=check_process process=345" "warn=none" "crit=ok_count ne 2" - OK - 2 plugins checked, 2 ok + check_multi "check=check_process 'process=firefox'" "check=check_memory 'crit=used_pct gt 80%'" + OK - 2 plugins checked, 2 ok | 'check_process::count'=1;;;0 'check_process::rss'=258686976B;;;0 ... + [ 1] check_process OK - all 1 processes are ok. + [ 2] check_memory OK - physical = 12.22 GiB/16.00 GiB (76.4%), swap = 1.95 GiB/3.00 GiB (65.0%) + + You can define warning/critical conditions based on the number of checks in a certain state (see attributes below): + + check_multi "check=check_dummy 0 'OK'" "check=check_dummy 1 'WARNING'" "critical=problem_count gt 0" + CRITICAL - 2 plugins checked: 1 ok, 1 warning, 0 critical, 0 unknown + [ 1] check_dummy OK + [ 2] check_dummy WARNING `, } } From 6542be3e8703e8d28115aa38d1a06e95528527a5 Mon Sep 17 00:00:00 2001 From: lorenzg Date: Fri, 14 Aug 2026 15:41:09 +0200 Subject: [PATCH 6/6] improve docs --- docs/checks/commands/check_multi.md | 4 ++-- pkg/snclient/check_multi.go | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/docs/checks/commands/check_multi.md b/docs/checks/commands/check_multi.md index 53984adb..9e987874 100644 --- a/docs/checks/commands/check_multi.md +++ b/docs/checks/commands/check_multi.md @@ -7,8 +7,8 @@ title: multi Runs multiple checks and aggregates their status, output and performance data. By default 'CheckMulti' is enabled, but you can disable it in the '[/modules]' section of the snclient_local.ini. - You can also set a limit for the number of checks that can be executed in the '[/settings/check/multi]' section - of the snclient_local.ini. + You can also set 'max checks' in the '[/settings/check/multi]' section of the snclient_local.ini, which limits + the number of checks that can be configured. When using the inline mode, you can only use available commands (run 'check_index' to get a full list). diff --git a/pkg/snclient/check_multi.go b/pkg/snclient/check_multi.go index b75c276f..2da7ec4e 100644 --- a/pkg/snclient/check_multi.go +++ b/pkg/snclient/check_multi.go @@ -29,8 +29,8 @@ func (l *CheckMulti) Build() *CheckData { description: `Runs multiple checks and aggregates their status, output and performance data. By default 'CheckMulti' is enabled, but you can disable it in the '[/modules]' section of the snclient_local.ini. - You can also set a limit for the number of checks that can be executed in the '[/settings/check/multi]' section - of the snclient_local.ini. + You can also set 'max checks' in the '[/settings/check/multi]' section of the snclient_local.ini, which limits + the number of checks that can be configured. When using the inline mode, you can only use available commands (run 'check_index' to get a full list).