diff --git a/Makefile b/Makefile index db681a21..fa60b351 100644 --- a/Makefile +++ b/Makefile @@ -278,7 +278,7 @@ citest: tools vendor # $(MAKE) benchmark # - # Race rondition tests + # Race condition tests # $(MAKE) racetest # @@ -737,6 +737,7 @@ DOC_COMMANDS=\ check_mailq \ check_memory \ check_mount \ + check_multi \ check_network \ check_ntp_offset \ check_omd \ diff --git a/README.md b/README.md index 9b7586dc..f1df62ec 100644 --- a/README.md +++ b/README.md @@ -76,6 +76,7 @@ Further details are covered in the [documentation](https://omd.consol.de/docs/sn | **check_mailq** | | X | X | X | | **check_memory** | X | X | X | X | | **check_mount** | X | X | X | X | +| **check_multi** | X | X | X | X | | **check_network** | X | X | X | X | | **check_nsc_web** | X | X | X | X | | **check_ntp_offset** | X | X | X | X | diff --git a/docs/checks/commands/check_multi.md b/docs/checks/commands/check_multi.md new file mode 100644 index 00000000..aed5a914 --- /dev/null +++ b/docs/checks/commands/check_multi.md @@ -0,0 +1,121 @@ +--- +title: multi +--- + +## check_multi + +Runs multiple checks and aggregates their status, output and performance data. + + By default 'CheckMulti' is enabled, but you can disable it in the '[/modules]' section of the snclient_local.ini. + You can also set 'max checks' in the '[/settings/check/multi]' section of the snclient_local.ini, which limits + the number of checks that can be configured. + + When using the inline mode, you can only use available commands (run 'check_index' to get a full list). + + You can also define custom check sections in the config file, for example: + [/settings/check/multi/mycheck] + command[alias1] = check_process process=123 + command[alias2] = check_process process=345 + + This can be executed with 'check_multi "config=mycheck"'. + + It's also possible to use custom scripts in the config section, for example: + [/settings/check/multi/myscript] + command[alias1] = /path/to/plugin1 + command[alias2] = /path/to/plugin2 + command[alias3] = /path/to/plugin3 + + This can be executed with 'check_multi "config=myscript"'. + + +- [Examples](#examples) +- [Argument Defaults](#argument-defaults) +- [Attributes](#attributes) + +## Implementation + +| Windows | Linux | FreeBSD | MacOSX | +|:------------------:|:------------------:|:------------------:|:------------------:| +| :white_check_mark: | :white_check_mark: | :white_check_mark: | :white_check_mark: | + +## Examples + +### Default Check + + check_multi "command[check_process]=check_process 'process=firefox'" "command[check_memory]=check_memory 'type=physical' 'crit=used_pct gt 80%'" + OK - 2 plugins checked, 2 ok |'check_process::count'=1;;;0 ... 'check_memory::physical %'=78.7%;;;0;100 + [check_process] OK - all 1 processes are ok. + [check_memory] OK - physical = 12.59 GiB/16.00 GiB (78.7%) + + You can define 'warning' and 'critical' conditions based on the number of checks in a certain state (see attributes below): + + check_multi "command[check_dummy1]=check_dummy 0 'OK - check works'" "command[check_dummy2]=check_dummy 1 'WARNING - problem found'" "critical=problem_count gt 0" + CRITICAL - 2 plugins checked: 1 ok, 1 warning, 0 critical, 0 unknown - warning(check_dummy2: WARNING - problem found) + [check_dummy1] OK - check works + [check_dummy2] WARNING - problem found + + You can also override the 'top-syntax' and use IF ELSE statements to get a certain output based on the results: + + check_multi "command[check_dummy1]=check_dummy 0 'OK'" "command[check_dummy2]=check_dummy 2 'CRITICAL'" \ + "top-syntax={{ if ok_count gt 0 }}OK - %(ok_count)/%(count) checks are OK {{ ELSE }}CRITICAL - all checks failed{{ END }}" + OK - 1/2 checks are OK + [check_dummy1] OK + [check_dummy2] CRITICAL + +### Example using NRPE and Naemon + +Naemon Config + + define command{ + command_name check_nrpe + command_line $USER1$/check_nrpe -H $HOSTADDRESS$ -n -c $ARG1$ -a $ARG2$ + } + + define service { + host_name testhost + service_description check_multi + use generic-service + check_command check_nrpe!check_multi! + } + +## Argument Defaults + +| Argument | Default Value | +| ------------- | ----------------------------------------------------------------------------------------------------- | +| warning | warning_count > 0 | +| critical | critical_count > 0 | +| unknown | unknown_count > 0 | +| empty-state | 3 (UNKNOWN) | +| empty-syntax | %(status) - no checks executed | +| top-syntax | %(status) - %(count) plugins checked: %(ok_count) ok, %(warning_count) warning, %(critical_count) critical, %(unknown_count) unknown - %(problem_list) | +| ok-syntax | {{ if problem_count gt 0 }}%(status) - %(count) plugins checked: %(ok_count) ok, %(warning_count) warning, %(critical_count) critical, %(unknown_count) unknown - %(problem_list){{ ELSE }}%(status) - %(count) plugins checked, %(ok_count) ok{{ END }} | +| detail-syntax | %(name): %(output) | + +## Check Specific Arguments + +| Argument | Description | +| -------- | ------------------------------------------------------------------------- | +| command | Check command to execute with mandatory unique tag, e.g. command[tag]=... | +| config | Config section name under [/settings/check/multi/< section >] to execute | + +## Attributes + +### Filter Keywords + +these can be used in filters and thresholds (along with the default attributes): + +| Attribute | Description | +| -------------- | --------------------------------------------------------------- | +| count | Total number of checks executed | +| ok_count | Number of checks in OK state | +| warning_count | Number of checks in WARNING state | +| critical_count | Number of checks in CRITICAL state | +| unknown_count | Number of checks in UNKNOWN state | +| problem_count | Number of checks in non-OK state | +| name | Name/Tag of the check | +| tag | Alias for name | +| command | Command executed | +| state | Exit code of the check (0=OK, 1=WARNING, 2=CRITICAL, 3=UNKNOWN) | +| status | Status text of the check (OK, WARNING, CRITICAL, UNKNOWN) | +| output | Check output | +| shortoutput | First line of the check output | diff --git a/packaging/snclient.ini b/packaging/snclient.ini index 51432c9b..d4e82ce3 100644 --- a/packaging/snclient.ini +++ b/packaging/snclient.ini @@ -70,6 +70,9 @@ CheckWMI = disabled ; CheckLogFile - Controls whether check_logfile is allowed or not. CheckLogFile = disabled +; CheckMulti - Controls whether check_multi is allowed or not. +CheckMulti = enabled + [/settings/default] ; allowed hosts - Comma separated list of ips/networks/hostname allowed to connect. @@ -357,6 +360,11 @@ allowed pattern += /var/log/snclient/snclient.log max lines per file limit = 1000000 +[/settings/check/multi] +; max checks - Maximum number of checks that can be configured within check_multi (applies to config and inline). +max checks = 20 + + ; External script settings - General settings for the external scripts module (CheckExternalScripts). [/settings/external scripts] diff --git a/pkg/snclient/check_files_test.go b/pkg/snclient/check_files_test.go index fa9ffe7e..243e76dd 100644 --- a/pkg/snclient/check_files_test.go +++ b/pkg/snclient/check_files_test.go @@ -76,7 +76,7 @@ func TestCheckFiles(t *testing.T) { res = snc.RunCheck("check_files", []string{"path=./t/checksum.txt", "crit=md5_checksum != 3687C5D7106484CD61CDE867A2A999FA"}) assert.Equalf(t, CheckExitCritical, res.State, "CRITICAL") - assert.Contains(t, string(res.BuildPluginOutput()), "0/1 files") + assert.Contains(t, string(res.BuildPluginOutput()), "1/1 files") res = snc.RunCheck("check_files", []string{"path=./t/checksum.txt", "crit=sha1_checksum == 4EE4BFE9AA51E56A7BD5CCF4785C35A27EE022F8"}) assert.Equalf(t, CheckExitOK, res.State, "state OK") diff --git a/pkg/snclient/check_multi.go b/pkg/snclient/check_multi.go new file mode 100644 index 00000000..ca0b5128 --- /dev/null +++ b/pkg/snclient/check_multi.go @@ -0,0 +1,438 @@ +package snclient + +import ( + "context" + "fmt" + "maps" + "strings" + + "github.com/consol-monitoring/snclient/pkg/convert" + "github.com/consol-monitoring/snclient/pkg/utils" +) + +func init() { + AvailableChecks["check_multi"] = CheckEntry{"check_multi", NewCheckMulti} +} + +type ( + checkMultiConfigKey struct{} + checkMultiDepthKey struct{} + checkMultiCounterKey struct{} +) + +type checkMultiCounter struct { + count int64 + maxChecks int64 +} + +type CheckMulti struct { + commands TaggedCommandList + config string +} + +var checkMultiAttributes = []CheckAttribute{ + {name: "count", description: "Total number of checks executed"}, + {name: "ok_count", description: "Number of checks in OK state"}, + {name: "warning_count", description: "Number of checks in WARNING state"}, + {name: "critical_count", description: "Number of checks in CRITICAL state"}, + {name: "unknown_count", description: "Number of checks in UNKNOWN state"}, + {name: "problem_count", description: "Number of checks in non-OK state"}, + {name: "name", description: "Name/Tag of the check"}, + {name: "tag", description: "Alias for name"}, + {name: "command", description: "Command executed"}, + {name: "state", description: "Exit code of the check (0=OK, 1=WARNING, 2=CRITICAL, 3=UNKNOWN)"}, + {name: "status", description: "Status text of the check (OK, WARNING, CRITICAL, UNKNOWN)"}, + {name: "output", description: "Check output"}, + {name: "shortoutput", description: "First line of the check output"}, +} + +func NewCheckMulti() CheckHandler { + return &CheckMulti{ + commands: make(TaggedCommandList, 0), + } +} + +func (l *CheckMulti) Build() *CheckData { + return &CheckData{ + name: "check_multi", + description: `Runs multiple checks and aggregates their status, output and performance data. + + By default 'CheckMulti' is enabled, but you can disable it in the '[/modules]' section of the snclient_local.ini. + You can also set 'max checks' in the '[/settings/check/multi]' section of the snclient_local.ini, which limits + the number of checks that can be configured. + + When using the inline mode, you can only use available commands (run 'check_index' to get a full list). + + You can also define custom check sections in the config file, for example: + [/settings/check/multi/mycheck] + command[alias1] = check_process process=123 + command[alias2] = check_process process=345 + + This can be executed with 'check_multi "config=mycheck"'. + + It's also possible to use custom scripts in the config section, for example: + [/settings/check/multi/myscript] + command[alias1] = /path/to/plugin1 + command[alias2] = /path/to/plugin2 + command[alias3] = /path/to/plugin3 + + This can be executed with 'check_multi "config=myscript"'. +`, + implemented: ALL, + disableFilter: true, + result: &CheckResult{ + State: CheckExitOK, + }, + args: map[string]CheckArgument{ + "command": {value: &l.commands, description: "Check command to execute with mandatory unique tag, e.g. command[tag]=..."}, + "config": {value: &l.config, description: "Config section name under [/settings/check/multi/< section >] to execute"}, + }, + conditionAlias: map[string]map[string]string{ + "warning_count": {"warn_count": "warning_count"}, + "critical_count": {"crit_count": "critical_count"}, + }, + attributes: checkMultiAttributes, + defaultWarning: "warning_count > 0", + defaultCritical: "critical_count > 0", + defaultUnknown: "unknown_count > 0", + okSyntax: "{{ if problem_count gt 0 }}%(status) - %(count) plugins checked: " + + "%(ok_count) ok, %(warning_count) warning, %(critical_count) critical, " + + "%(unknown_count) unknown - %(problem_list){{ ELSE }}%(status) - " + + "%(count) plugins checked, %(ok_count) ok{{ END }}", + topSyntax: "%(status) - %(count) plugins checked: %(ok_count) ok, %(warning_count) warning, %(critical_count) critical, %(unknown_count) unknown - %(problem_list)", + detailSyntax: "%(name): %(output)", + emptySyntax: "%(status) - no checks executed", + emptyState: CheckExitUnknown, + exampleDefault: ` + check_multi "command[check_process]=check_process 'process=firefox'" "command[check_memory]=check_memory 'type=physical' 'crit=used_pct gt 80%'" + OK - 2 plugins checked, 2 ok |'check_process::count'=1;;;0 ... 'check_memory::physical %'=78.7%;;;0;100 + [check_process] OK - all 1 processes are ok. + [check_memory] OK - physical = 12.59 GiB/16.00 GiB (78.7%) + + You can define 'warning' and 'critical' conditions based on the number of checks in a certain state (see attributes below): + + check_multi "command[check_dummy1]=check_dummy 0 'OK - check works'" "command[check_dummy2]=check_dummy 1 'WARNING - problem found'" "critical=problem_count gt 0" + CRITICAL - 2 plugins checked: 1 ok, 1 warning, 0 critical, 0 unknown - warning(check_dummy2: WARNING - problem found) + [check_dummy1] OK - check works + [check_dummy2] WARNING - problem found + + You can also override the 'top-syntax' and use IF ELSE statements to get a certain output based on the results: + + check_multi "command[check_dummy1]=check_dummy 0 'OK'" "command[check_dummy2]=check_dummy 2 'CRITICAL'" \ + "top-syntax={{ if ok_count gt 0 }}OK - %(ok_count)/%(count) checks are OK {{ ELSE }}CRITICAL - all checks failed{{ END }}" + OK - 1/2 checks are OK + [check_dummy1] OK + [check_dummy2] CRITICAL + `, + } +} + +type multiChildCheck struct { + tag string + cmdStr string + isInline bool +} + +func (l *CheckMulti) Check(ctx context.Context, snc *Agent, check *CheckData, _ []Argument) (*CheckResult, error) { + enabled, _, _ := snc.config.Section("/modules").GetBool("CheckMulti") + if !enabled { + return &CheckResult{ + State: CheckExitUnknown, + Output: "module CheckMulti is not enabled in /modules section", + }, nil + } + + depth, _ := ctx.Value(checkMultiDepthKey{}).(int) + if depth > 5 { + return &CheckResult{ + State: CheckExitUnknown, + Output: "recursion limit exceeded for check_multi", + }, nil + } + ctx = context.WithValue(ctx, checkMultiDepthKey{}, depth+1) + + maxChecks, ok, err := snc.config.Section("/settings/check/multi").GetInt("max checks") + if err != nil || !ok || maxChecks <= 0 { + maxChecks = 20 + } + + if _, ok := ctx.Value(checkMultiCounterKey{}).(*checkMultiCounter); !ok { + counter := &checkMultiCounter{maxChecks: maxChecks} + ctx = context.WithValue(ctx, checkMultiCounterKey{}, counter) + } + + activeConfigs, _ := ctx.Value(checkMultiConfigKey{}).(map[string]bool) + if activeConfigs == nil { + activeConfigs = make(map[string]bool) + } + + if l.config != "" { + if activeConfigs[l.config] { + return &CheckResult{ + State: CheckExitUnknown, + Output: fmt.Sprintf("loop detected: check_multi config %s is already running in the call chain", l.config), + }, nil + } + newActive := make(map[string]bool, len(activeConfigs)+1) + maps.Copy(newActive, activeConfigs) + newActive[l.config] = true + ctx = context.WithValue(ctx, checkMultiConfigKey{}, newActive) + } + + childChecks, res := l.buildChildChecks(snc) + if res != nil { + return res, nil + } + + if len(childChecks) == 0 { + return &CheckResult{ + State: CheckExitUnknown, + Output: "no checks or config specified", + }, nil + } + + return l.executeChildChecks(ctx, snc, check, childChecks) +} + +// buildChildChecks assembles the list of child checks from config section and inline args. +func (l *CheckMulti) buildChildChecks(snc *Agent) ([]multiChildCheck, *CheckResult) { + childChecks := []multiChildCheck{} + seenTags := make(map[string]bool) + + if l.config != "" { + configChecks, res := l.buildConfigChecks(snc) + if res != nil { + return nil, res + } + for _, chk := range configChecks { + if seenTags[chk.tag] { + return nil, &CheckResult{ + State: CheckExitUnknown, + Output: fmt.Sprintf("duplicate command tag: %s", chk.tag), + } + } + seenTags[chk.tag] = true + childChecks = append(childChecks, chk) + } + } + + for _, cmd := range l.commands { + if seenTags[cmd.Tag] { + return nil, &CheckResult{ + State: CheckExitUnknown, + Output: fmt.Sprintf("duplicate command tag: %s", cmd.Tag), + } + } + seenTags[cmd.Tag] = true + childChecks = append(childChecks, multiChildCheck{ + tag: cmd.Tag, + cmdStr: cmd.Command, + isInline: true, + }) + } + + return childChecks, nil +} + +// buildConfigChecks loads checks from the named config section. +func (l *CheckMulti) buildConfigChecks(snc *Agent) ([]multiChildCheck, *CheckResult) { + secName := "/settings/check/multi/" + l.config + sec, ok := snc.config.sections[secName] + + if !ok || len(sec.keys) == 0 { + return nil, &CheckResult{ + State: CheckExitUnknown, + Output: fmt.Sprintf("no checks defined in config section %s", secName), + } + } + + childChecks := make([]multiChildCheck, 0, len(sec.keys)) + + for _, key := range sec.keys { + rawVal := sec.data[key] + if !strings.HasPrefix(key, "command[") || !strings.HasSuffix(key, "]") { + return nil, &CheckResult{ + State: CheckExitUnknown, + Output: fmt.Sprintf("invalid check_multi config entry: %s (must be in format command[tag]=)", key), + } + } + tag := strings.TrimSpace(strings.TrimSuffix(strings.TrimPrefix(key, "command["), "]")) + if strings.ContainsAny(tag, DefaultNastyCharacters+"=") { + return nil, &CheckResult{ + State: CheckExitUnknown, + Output: fmt.Sprintf("command tag contains invalid characters: %s", tag), + } + } + if strings.TrimSpace(tag) == "" { + return nil, &CheckResult{ + State: CheckExitUnknown, + Output: "empty command tag in config section", + } + } + if strings.TrimSpace(rawVal) == "" { + return nil, &CheckResult{ + State: CheckExitUnknown, + Output: fmt.Sprintf("empty command for tag %s in config section", tag), + } + } + childChecks = append(childChecks, multiChildCheck{ + tag: tag, + cmdStr: rawVal, + isInline: false, + }) + } + + return childChecks, nil +} + +// executeChildChecks runs all child checks and aggregates results. +func (l *CheckMulti) executeChildChecks(ctx context.Context, snc *Agent, check *CheckData, childChecks []multiChildCheck) (*CheckResult, error) { + var count, okCount, warnCount, critCount, unknownCount int64 + + detailsList := make([]string, 0, len(childChecks)) + allMetrics := make([]*CheckMetric, 0) + + hasEntryThresholds := check.HasThreshold("name") || check.HasThreshold("tag") || check.HasThreshold("command") || + check.HasThreshold("output") || check.HasThreshold("shortoutput") || check.HasThreshold("status") || check.HasThreshold("state") + + for _, chk := range childChecks { + if res := l.incrementCheckMultiCounter(ctx); res != nil { + return res, nil + } + + res, fatal := l.runChildCheck(ctx, snc, check, chk) + if fatal { + return res, nil + } + + tag := chk.tag + childOutput := res.BuildOutputString() + + firstLine := strings.TrimSpace(strings.Split(childOutput, "\n")[0]) + literalOutput := check.result.LiteralizeDetails(fmt.Sprintf("[%s] %s", tag, childOutput)) + detailsList = append(detailsList, literalOutput) + + entryState := fmt.Sprintf("%d", res.State) + entry := map[string]string{ + "name": tag, + "tag": tag, + "command": chk.cmdStr, + "state": entryState, + "status": res.StateString(), + "shortoutput": firstLine, + "output": childOutput, + "_state": entryState, + "_skip": "1", + "_count": "1", + } + + if hasEntryThresholds { + thresholdEntry := maps.Clone(entry) + check.Check(thresholdEntry, check.warnThreshold, check.critThreshold, check.unknownThreshold, check.okThreshold) + check.result.EscalateStatus(convert.Int64(thresholdEntry["_state"])) + } + + count++ + switch entry["_state"] { + case "0": + okCount++ + case "1": + warnCount++ + case "2": + critCount++ + default: + unknownCount++ + } + + check.listData = append(check.listData, entry) + + allMetrics = appendChildMetrics(allMetrics, res, tag) + } + + problemCount := warnCount + critCount + unknownCount + check.details = map[string]string{ + "count": fmt.Sprintf("%d", count), + "ok_count": fmt.Sprintf("%d", okCount), + "warning_count": fmt.Sprintf("%d", warnCount), + "warn_count": fmt.Sprintf("%d", warnCount), + "critical_count": fmt.Sprintf("%d", critCount), + "crit_count": fmt.Sprintf("%d", critCount), + "unknown_count": fmt.Sprintf("%d", unknownCount), + "problem_count": fmt.Sprintf("%d", problemCount), + } + + check.result.Metrics = allMetrics + check.result.Details = strings.Join(detailsList, "\n") + + return check.Finalize() +} + +func appendChildMetrics(allMetrics []*CheckMetric, res *CheckResult, tag string) []*CheckMetric { + for _, m := range res.Metrics { + metricCopy := *m + metricCopy.Name = fmt.Sprintf("%s::%s", tag, m.Name) + metricCopy.SkipStateCheck = true + allMetrics = append(allMetrics, &metricCopy) + } + + return allMetrics +} + +func (l *CheckMulti) incrementCheckMultiCounter(ctx context.Context) *CheckResult { + counter, _ := ctx.Value(checkMultiCounterKey{}).(*checkMultiCounter) + counter.count++ + if counter.count > counter.maxChecks { + return &CheckResult{ + State: CheckExitUnknown, + Output: fmt.Sprintf("number of checks (%d) exceeds max checks limit (%d)", counter.count, counter.maxChecks), + } + } + + return nil +} + +// runChildCheck executes a single child check and returns its result. +// The second return value is true when the error is fatal and the caller should stop processing. +func (l *CheckMulti) runChildCheck(ctx context.Context, snc *Agent, check *CheckData, chk multiChildCheck) (*CheckResult, bool) { + tokens := utils.Tokenize(chk.cmdStr) + tokens, err := utils.TrimQuotesList(tokens) + + if err != nil || len(tokens) == 0 { + return &CheckResult{ + State: CheckExitUnknown, + Output: fmt.Sprintf("failed to parse check command: %s", chk.cmdStr), + }, true + } + + cmdName := tokens[0] + cmdArgs := tokens[1:] + + _, isKnown := snc.getCheck(cmdName, false) + + if chk.isInline && !isKnown { + return &CheckResult{ + State: CheckExitUnknown, + Output: fmt.Sprintf("unknown check command: %s (inline checks only support existing check commands)", cmdName), + }, true + } + + if isKnown { + return snc.RunCheckWithContext(ctx, cmdName, cmdArgs, 0, nil, false), false + } + + stdout, stderr, exitCode, _ := snc.runExternalCheckString(ctx, chk.cmdStr, int64(check.timeout)) + out := stdout + if stderr != "" && !strings.Contains(out, stderr) { + if out != "" { + out += "\n" + } + out += "[" + stderr + "]" + } + res := &CheckResult{ + State: exitCode, + Output: out, + } + res.ParsePerformanceDataFromOutput() + + return res, false +} diff --git a/pkg/snclient/check_multi_test.go b/pkg/snclient/check_multi_test.go new file mode 100644 index 00000000..4f0e733b --- /dev/null +++ b/pkg/snclient/check_multi_test.go @@ -0,0 +1,397 @@ +package snclient + +import ( + "fmt" + "os" + "path/filepath" + "runtime" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestCheckMultiInline(t *testing.T) { + config := ` +[/modules] +CheckMulti = enabled +` + snc := StartTestAgent(t, config) + defer StopTestAgent(t, snc) + + // 1. Basic inline checks with mandatory tags - all OK + res := snc.RunCheck("check_multi", []string{ + "command[d1]=check_dummy 0 'dummy ok 1'", + "command[d2]=check_dummy 0 'dummy ok 2'", + }) + assert.Equalf(t, CheckExitOK, res.State, "state OK") + assert.Contains(t, res.Output, "2 plugins checked, 2 ok") + assert.Contains(t, res.Details, "[d1] dummy ok 1") + assert.Contains(t, res.Details, "[d2] dummy ok 2") + + // 2. Inline checks with warning and critical (default thresholds) + res = snc.RunCheck("check_multi", []string{ + "command[d1]=check_dummy 0 'dummy ok'", + "command[d2]=check_dummy 1 'dummy warn'", + }) + assert.Equalf(t, CheckExitWarning, res.State, "state WARNING") + assert.Contains(t, res.Output, "2 plugins checked: 1 ok, 1 warning, 0 critical, 0 unknown") + + res = snc.RunCheck("check_multi", []string{ + "command[test1]=check_dummy 1 WARN", + "warn=none", + }) + assert.Equalf(t, CheckExitOK, res.State, "state OK when warning threshold is none") + assert.Contains(t, res.Output, "OK - 1 plugins checked: 0 ok, 1 warning, 0 critical, 0 unknown - test1: WARN") + + res = snc.RunCheck("check_multi", []string{ + "command[d1]=check_dummy 0 'dummy ok'", + "command[d2]=check_dummy 2 'dummy crit'", + }) + assert.Equalf(t, CheckExitCritical, res.State, "state CRITICAL") + assert.Contains(t, res.Output, "2 plugins checked: 1 ok, 0 warning, 1 critical, 0 unknown") + + // 3. Custom conditions: warn=none crit=ok_count ne 2 + res = snc.RunCheck("check_multi", []string{ + "command[d1]=check_dummy 0 'dummy 1'", + "command[d2]=check_dummy 0 'dummy 2'", + "warn=none", + "crit=ok_count ne 2", + }) + assert.Equalf(t, CheckExitOK, res.State, "state OK when ok_count == 2") + + res = snc.RunCheck("check_multi", []string{ + "command[d1]=check_dummy 0 'dummy 1'", + "command[d2]=check_dummy 1 'dummy 2'", + "warn=none", + "crit=ok_count ne 2", + }) + assert.Equalf(t, CheckExitCritical, res.State, "state CRITICAL when ok_count != 2") + + // 4. Custom condition on entry attribute: critical=name eq 'alias2' and state=2 + res = snc.RunCheck("check_multi", []string{ + "command[alias1]=check_dummy 2 'crit 1'", + "command[alias2]=check_dummy 0 'ok 2'", + "warn=none", + "crit=name eq 'alias2' and state=2", + }) + assert.Equalf(t, CheckExitOK, res.State, "state OK when alias2 is not in state 2") + + res = snc.RunCheck("check_multi", []string{ + "command[alias1]=check_dummy 0 'ok 1'", + "command[alias2]=check_dummy 2 'crit 2'", + "warn=none", + "crit=name eq 'alias2' and state=2", + }) + assert.Equalf(t, CheckExitCritical, res.State, "state CRITICAL when alias2 is in state 2") + + // 5. Mandatory tag validation: missing tag & duplicate tag + res = snc.RunCheck("check_multi", []string{ + "command=check_dummy 0 'ok'", + }) + assert.Equalf(t, CheckExitUnknown, res.State, "state UNKNOWN when command has no tag") + assert.Contains(t, res.Output, "command argument requires a unique tag") + + res = snc.RunCheck("check_multi", []string{ + "command[dup]=check_dummy 0 'ok 1'", + "command[dup]=check_dummy 0 'ok 2'", + }) + assert.Equalf(t, CheckExitUnknown, res.State, "state UNKNOWN when command tag is duplicated") + assert.Contains(t, res.Output, "duplicate command tag: dup") + + // 6. Unknown/inline checks restriction (cannot run arbitrary external commands inline) + res = snc.RunCheck("check_multi", []string{ + "command[ext]=/bin/nonexistent_or_external_script -H 123", + }) + assert.Equalf(t, CheckExitUnknown, res.State, "state UNKNOWN for unregistered inline command") + assert.Contains(t, res.Output, "unknown check command") + + // 7. Filter argument is disabled/rejected + res = snc.RunCheck("check_multi", []string{ + "command[d1]=check_dummy 0 'ok'", + "filter=state=1", + }) + assert.Equalf(t, CheckExitUnknown, res.State, "state UNKNOWN when filter argument is used") + assert.Contains(t, res.Output, "filter is disabled for this check") + + // 8. Severity hierarchy: UNKNOWN > CRITICAL > WARNING > OK + res = snc.RunCheck("check_multi", []string{ + "command[d1]=check_dummy 0 'ok'", + "command[d2]=check_dummy 3 'unknown check'", + }) + assert.Equalf(t, CheckExitUnknown, res.State, "state UNKNOWN when child check is unknown by default") + + res = snc.RunCheck("check_multi", []string{ + "command[d1]=check_dummy 1 'warn check'", + "command[d2]=check_dummy 3 'unknown check'", + }) + assert.Equalf(t, CheckExitUnknown, res.State, "state UNKNOWN takes precedence over WARNING") + + res = snc.RunCheck("check_multi", []string{ + "command[d1]=check_dummy 2 'crit check'", + "command[d2]=check_dummy 3 'unknown check'", + }) + assert.Equalf(t, CheckExitUnknown, res.State, "state UNKNOWN takes precedence over CRITICAL") +} + +func TestCheckMultiDefaultEnabled(t *testing.T) { + snc := StartTestAgent(t, "") + defer StopTestAgent(t, snc) + + res := snc.RunCheck("check_multi", []string{ + "command[d1]=check_dummy 0 'default enabled'", + }) + assert.Equalf(t, CheckExitOK, res.State, "state OK when CheckMulti is enabled by default") +} + +func TestCheckMultiPreservesLiteralChildOutput(t *testing.T) { + snc := StartTestAgent(t, "") + defer StopTestAgent(t, snc) + + res := snc.RunCheck("check_multi", []string{ + "command[child]=check_dummy 0 '%(count) {{ IF condition }}literal{{ END }}'", + }) + assert.Equalf(t, CheckExitOK, res.State, "state OK for literal child output") + assert.Contains(t, res.Details, "%(count) {{ IF condition }}literal{{ END }}") +} + +func TestTaggedNonCommandArgumentRejected(t *testing.T) { + snc := StartTestAgent(t, "") + defer StopTestAgent(t, snc) + + res := snc.RunCheck("check_files", []string{"path[x]=/tmp"}) + assert.Equalf(t, CheckExitUnknown, res.State, "state UNKNOWN for tagged non-command argument") + assert.Contains(t, res.Output, "does not support tags") +} + +func TestCheckMultiPriorityThreshold(t *testing.T) { + config := ` +[/modules] +CheckMulti = enabled +` + snc := StartTestAgent(t, config) + defer StopTestAgent(t, snc) + + args := []string{ + "command[prio]=check_dummy 2 'priority critical'", + "command[dummy2]=check_dummy 0 'dummy 2'", + "command[dummy3]=check_dummy 0 'dummy 3'", + "warn=none", + "unknown=none", + "crit=name eq 'prio' and state ne '0'", + } + res := snc.RunCheck("check_multi", args) + assert.Equalf(t, CheckExitCritical, res.State, "state CRITICAL when prio is not OK") + assert.Contains(t, res.Details, "[prio] priority critical") + + res = snc.RunCheck("check_multi", []string{ + "command[prio]=check_dummy 0 'priority ok'", + "command[dummy2]=check_dummy 3 'dummy 2 unknown'", + "command[dummy3]=check_dummy 2 'dummy 3 critical'", + "warn=none", + "unknown=none", + "crit=name eq 'prio' and state ne '0'", + }) + assert.Equalf(t, CheckExitOK, res.State, "state OK when only non-prio checks are problems") + assert.Contains(t, res.Details, "[dummy2] dummy 2 unknown") + assert.Contains(t, res.Details, "[dummy3] dummy 3 critical") +} + +func TestCheckMultiLimits(t *testing.T) { + config := ` +[/modules] +CheckMulti = enabled + +[/settings/check/multi] +max checks = 4 + +[/settings/check/multi/nested] +command[d1] = check_dummy 0 'nested 1' +command[d2] = check_dummy 0 'nested 2' +` + snc := StartTestAgent(t, config) + defer StopTestAgent(t, snc) + + // Under limit: 2 checks + res := snc.RunCheck("check_multi", []string{ + "command[d1]=check_dummy 0 'ok 1'", + "command[d2]=check_dummy 0 'ok 2'", + }) + assert.Equalf(t, CheckExitOK, res.State, "state OK for 2 checks") + + // Exceeds limit: 5 checks + res = snc.RunCheck("check_multi", []string{ + "command[d1]=check_dummy 0 'ok 1'", + "command[d2]=check_dummy 0 'ok 2'", + "command[d3]=check_dummy 0 'ok 3'", + "command[d4]=check_dummy 0 'ok 4'", + "command[d5]=check_dummy 0 'ok 5'", + }) + assert.Equalf(t, CheckExitUnknown, res.State, "state UNKNOWN when exceeding max checks") + assert.Contains(t, res.Output, "exceeds max checks limit") + + // Nested checks share the same cumulative execution count. + res = snc.RunCheck("check_multi", []string{ + "command[d1]=check_dummy 0 'outer 1'", + "command[d2]=check_dummy 0 'outer 2'", + "command[nested]=check_multi config=nested", + }) + assert.Equalf(t, CheckExitUnknown, res.State, "state UNKNOWN when nested checks exceed max checks") + assert.Contains(t, res.Details, "number of checks (5) exceeds max checks limit (4)") +} + +func TestCheckMultiDisabled(t *testing.T) { + config := ` +[/modules] +CheckMulti = disabled +` + snc := StartTestAgent(t, config) + defer StopTestAgent(t, snc) + + res := snc.RunCheck("check_multi", []string{ + "command[d1]=check_dummy 0 'ok 1'", + }) + assert.Equalf(t, CheckExitUnknown, res.State, "state UNKNOWN when module is disabled") + assert.Contains(t, res.Output, "module CheckMulti is not enabled") +} + +func TestCheckMultiConfigSection(t *testing.T) { + // Create temporary scripts to test external scripts in config + tmpDir := t.TempDir() + var scriptExt string + var script1Content, script2Content string + + if runtime.GOOS == "windows" { + scriptExt = ".ps1" + script1Content = `Write-Output "SCRIPT 1 OK | perf1=10;20;30" +exit 0 +` + script2Content = `Write-Output "SCRIPT 2 WARNING | perf2=50;40;60" +exit 1 +` + } else { + scriptExt = ".sh" + script1Content = `#!/bin/sh +echo "SCRIPT 1 OK | perf1=10;20;30" +exit 0 +` + script2Content = `#!/bin/sh +echo "SCRIPT 2 WARNING | perf2=50;40;60" +exit 1 +` + } + + script1 := filepath.Join(tmpDir, "test1"+scriptExt) + script2 := filepath.Join(tmpDir, "test2"+scriptExt) + + err := os.WriteFile(script1, []byte(script1Content), 0o600) + require.NoError(t, err) + + err = os.WriteFile(script2, []byte(script2Content), 0o600) + require.NoError(t, err) + + if runtime.GOOS != "windows" { + require.NoError(t, os.Chmod(script1, 0o700)) + require.NoError(t, os.Chmod(script2, 0o700)) + } + + config := fmt.Sprintf(` +[/modules] +CheckMulti = enabled + +[/settings/check/multi/mycheck] +command[c1] = check_dummy 0 ok1 +command[c2] = check_dummy 0 ok2 + +[/settings/check/multi/custom] +command[s1] = %s -H 123 +command[s2] = %s -W 123 + +[/settings/check/multi/loop] +command[sub] = check_multi config=loop + +[/settings/check/multi/loopA] +command[b] = check_multi config=loopB + +[/settings/check/multi/loopB] +command[a] = check_multi config=loopA + +[/settings/check/multi/inner] +command[leaf] = check_dummy 0 'nested detail' + +[/settings/check/multi/duplicate] +command[foo] = check_dummy 0 first +command[ foo ] = check_dummy 0 second +`, script1, script2) + + snc := StartTestAgent(t, config) + defer StopTestAgent(t, snc) + + // Test config=mycheck (builtin checks in config) + res := snc.RunCheck("check_multi", []string{ + "config=mycheck", + "warn=none", + "crit=ok_count ne 2", + }) + assert.Equalf(t, CheckExitOK, res.State, "state OK for mycheck config") + assert.Contains(t, res.Output, "2 plugins checked, 2 ok") + + // Test config=custom (external scripts in config) + res = snc.RunCheck("check_multi", []string{ + "config=custom", + "warn=problem_count gt 0", + "crit=none", + }) + assert.Equalf(t, CheckExitWarning, res.State, "state WARNING for custom config") + assert.Contains(t, res.Output, "2 plugins checked: 1 ok, 1 warning, 0 critical, 0 unknown") + assert.Contains(t, res.Details, "SCRIPT 1 OK") + assert.Contains(t, res.Details, "SCRIPT 2 WARNING") + + // Nested detail output is included in the parent output attribute. + res = snc.RunCheck("check_multi", []string{ + "command[nested]=check_multi config=inner", + }) + assert.Equalf(t, CheckExitOK, res.State, "state OK for nested detail output") + assert.Contains(t, res.BuildOutputString(), "nested detail") + assert.Contains(t, res.Details, "nested detail") + + res = snc.RunCheck("check_multi", []string{ + "config=duplicate", + }) + assert.Equalf(t, CheckExitUnknown, res.State, "state UNKNOWN for whitespace-duplicate config tags") + assert.Contains(t, res.Output, "duplicate command tag: foo") + + // Test direct loop detection: check_multi config=loop + res = snc.RunCheck("check_multi", []string{ + "config=loop", + }) + assert.Equalf(t, CheckExitUnknown, res.State, "state UNKNOWN for loop config") + assert.Contains(t, res.Output, "loop detected") + + // Test indirect loop detection: loopA -> loopB -> loopA + res = snc.RunCheck("check_multi", []string{ + "config=loopA", + }) + assert.Equalf(t, CheckExitUnknown, res.State, "state UNKNOWN for indirect loop") + assert.Contains(t, res.Output, "loop detected") + + // Test non-existing config + res = snc.RunCheck("check_multi", []string{ + "config=doesnotexist", + }) + assert.Equalf(t, CheckExitUnknown, res.State, "state UNKNOWN for missing config section") + assert.Contains(t, res.Output, "no checks defined in config section") +} + +func TestCheckMultiIndex(t *testing.T) { + config := ` +[/modules] +CheckMulti = enabled +` + snc := StartTestAgent(t, config) + defer StopTestAgent(t, snc) + + res := snc.RunCheck("check_index", []string{"filter=name = 'check_multi'"}) + assert.Equalf(t, CheckExitOK, res.State, "state OK for check_index") + assert.Contains(t, res.Output, "check_multi") +} diff --git a/pkg/snclient/checkdata.go b/pkg/snclient/checkdata.go index 5b080cc2..e996a979 100644 --- a/pkg/snclient/checkdata.go +++ b/pkg/snclient/checkdata.go @@ -42,12 +42,20 @@ const ( type CommaStringList []string +type TaggedCommand struct { + Tag string + Command string +} + +type TaggedCommandList []TaggedCommand + type CheckArgument struct { value any // reference to storage pointer description string // used in help isFilter bool // if true, default filter is not used when this argument is set defaultCritical string // overrides default filter if argument is used - defaultWarning string // same for critical condition + defaultWarning string // same for warning condition + defaultUnknown string // same for unknown condition } // Implemented defines the available supported operating systems @@ -103,6 +111,7 @@ type CheckData struct { defaultFilter string conditionAlias map[string]map[string]string // replacement map of equivalent condition values conditionColAlias map[string][]string // if there are filter for given column, apply to alias columns too + disableFilter bool // disable filter argument for checks where filtering listData makes no sense args map[string]CheckArgument extraArgs map[string]CheckArgument // internal, map of expanded args argsPassthrough bool // allow arbitrary arguments without complaining about unknown argument @@ -113,6 +122,8 @@ type CheckData struct { defaultWarning string critThreshold ConditionList defaultCritical string + unknownThreshold ConditionList + defaultUnknown string okThreshold ConditionList detailSyntax string topSyntax string @@ -123,7 +134,7 @@ type CheckData struct { emptyStateSet bool details map[string]string listData []map[string]string - listCombine string // join string for detail list + listCombine string // join string for detail output listCombineSet bool // has the listCombine been set by user showAll bool // flag if check called with show-all addCountMetrics bool @@ -159,13 +170,14 @@ func (cd *CheckData) Finalize() (*CheckResult, error) { log.Debugf("filter: %s", cd.filter.String()) log.Debugf("condition warning: %s", cd.warnThreshold.String()) log.Debugf("condition critical: %s", cd.critThreshold.String()) + log.Debugf("condition unknown: %s", cd.unknownThreshold.String()) log.Debugf("condition ok: %s", cd.okThreshold.String()) // Run thresholds once on cd.details. This is done separately than metrics or entries // cd.details are of type map[string]string, // same as elements of the slice cd.listData, but there is only one per check // This can possibly set a value to cd.details[_state] , influencing check state - log.Tracef("checking warning, critical, and ok thresholds on check details") - cd.Check(cd.details, cd.warnThreshold, cd.critThreshold, cd.okThreshold) + log.Tracef("checking warning, critical, unknown, and ok thresholds on check details") + cd.Check(cd.details, cd.warnThreshold, cd.critThreshold, cd.unknownThreshold, cd.okThreshold) log.Tracef("details:") logTraceASCIIMap(cd.details) @@ -216,7 +228,7 @@ func (cd *CheckData) finalizeOutput() (*CheckResult, error) { // each entry in the list data is individually checked // This can possibly set "_state" of each entry, influencing the final state - cd.Check(entry, cd.warnThreshold, cd.critThreshold, cd.okThreshold) + cd.Check(entry, cd.warnThreshold, cd.critThreshold, cd.unknownThreshold, cd.okThreshold) } } @@ -242,8 +254,8 @@ func (cd *CheckData) finalizeOutput() (*CheckResult, error) { cd.result.ApplyPerfSyntax(cd.perfSyntax, cd.timezone) // Run a separate check on the macros - log.Tracef("checking warning, critical, and ok thresholds on check macros") - cd.Check(finalMacros, cd.warnThreshold, cd.critThreshold, cd.okThreshold) + log.Tracef("checking warning, critical, unknown, and ok thresholds on check macros") + cd.Check(finalMacros, cd.warnThreshold, cd.critThreshold, cd.unknownThreshold, cd.okThreshold) log.Tracef("checking warning, critical, and ok thresholds on check metrics") cd.setStateFromMaps(finalMacros) @@ -281,10 +293,12 @@ func (cd *CheckData) buildListMacros() map[string]string { okList := make([]string, 0) warnList := make([]string, 0) critList := make([]string, 0) + unknownList := make([]string, 0) count := int64(0) okCount := int64(0) warnCount := int64(0) critCount := int64(0) + unknownCount := int64(0) for _, entry := range cd.listData { weight := int64(1) if w, ok := entry["_count"]; ok { @@ -309,6 +323,9 @@ func (cd *CheckData) buildListMacros() map[string]string { case "2": critList = append(critList, expanded) critCount += weight + case "3": + unknownList = append(unknownList, expanded) + unknownCount += weight } } @@ -316,17 +333,21 @@ func (cd *CheckData) buildListMacros() map[string]string { cd.listCombine = ", " } result := map[string]string{ - "count": fmt.Sprintf("%d", count), - "list": strings.Join(list, cd.listCombine), - "ok_count": fmt.Sprintf("%d", okCount), - "ok_list": "", - "warn_count": fmt.Sprintf("%d", warnCount), - "warn_list": "", - "crit_count": fmt.Sprintf("%d", critCount), - "crit_list": "", - "problem_count": fmt.Sprintf("%d", warnCount+critCount), - "problem_list": "", - "detail_list": "", + "count": fmt.Sprintf("%d", count), + "list": strings.Join(list, cd.listCombine), + "ok_count": fmt.Sprintf("%d", okCount), + "ok_list": "", + "warn_count": fmt.Sprintf("%d", warnCount), + "warning_count": fmt.Sprintf("%d", warnCount), + "warn_list": "", + "crit_count": fmt.Sprintf("%d", critCount), + "critical_count": fmt.Sprintf("%d", critCount), + "crit_list": "", + "unknown_count": fmt.Sprintf("%d", unknownCount), + "unknown_list": "", + "problem_count": fmt.Sprintf("%d", warnCount+critCount+unknownCount), + "problem_list": "", + "detail_list": "", } problemList := []string{} @@ -343,6 +364,11 @@ func (cd *CheckData) buildListMacros() map[string]string { problemList = append(problemList, result["warn_list"]) detailList = append(detailList, result["warn_list"]) } + if len(unknownList) > 0 { + result["unknown_list"] = "unknown(" + strings.Join(unknownList, cd.listCombine) + ")" + problemList = append(problemList, result["unknown_list"]) + detailList = append(detailList, result["unknown_list"]) + } if len(okList) > 0 { result["ok_list"] = strings.Join(okList, cd.listCombine) detailList = append(detailList, result["ok_list"]) @@ -364,17 +390,21 @@ func (cd *CheckData) buildListMacrosFromSingleEntry() map[string]string { } result := map[string]string{ - "count": "1", - "list": expanded, - "ok_count": "0", - "ok_list": "", - "warn_count": "0", - "warn_list": "", - "crit_count": "0", - "crit_list": "", - "problem_count": "0", - "problem_list": "", - "detail_list": expanded, + "count": "1", + "list": expanded, + "ok_count": "0", + "ok_list": "", + "warn_count": "0", + "warning_count": "0", + "warn_list": "", + "crit_count": "0", + "critical_count": "0", + "crit_list": "", + "unknown_count": "0", + "unknown_list": "", + "problem_count": "0", + "problem_list": "", + "detail_list": expanded, } numWarn := 0 @@ -387,12 +417,21 @@ func (cd *CheckData) buildListMacrosFromSingleEntry() map[string]string { result["problem_list"] = expanded result["warn_list"] = expanded result["warn_count"] = "1" + result["warning_count"] = "1" + result["problem_count"] = "1" numWarn = 1 case "2": result["problem_list"] = expanded result["crit_list"] = expanded result["crit_count"] = "1" + result["critical_count"] = "1" + result["problem_count"] = "1" numCrit = 1 + case "3": + result["problem_list"] = expanded + result["unknown_list"] = expanded + result["unknown_count"] = "1" + result["problem_count"] = "1" } cd.buildCountMetrics(1, numCrit, numWarn) @@ -445,13 +484,27 @@ func (cd *CheckData) setStateFromMaps(macros map[string]string) { cd.result.EscalateStatus(3) } - switch { - case macros["crit_count"] != "0": - cd.result.EscalateStatus(2) - macros["_state"] = "2" - case macros["warn_count"] != "0": - cd.result.EscalateStatus(1) - macros["_state"] = "1" + // Only escalate based on counts if the user hasn't explicitly set the threshold. + // This respects explicit thresholds like "crit=none" which disable escalation. + if !cd.hasArgsSupplied["unknown"] && !cd.hasArgsSupplied["unknown+"] { + if macros["unknown_count"] != "0" && macros["unknown_count"] != "" { + cd.result.EscalateStatus(3) + macros["_state"] = "3" + } + } + + if !cd.hasArgsSupplied["crit"] && !cd.hasArgsSupplied["critical"] && !cd.hasArgsSupplied["crit+"] && !cd.hasArgsSupplied["critical+"] { + if macros["crit_count"] != "0" && macros["crit_count"] != "" { + cd.result.EscalateStatus(2) + macros["_state"] = "2" + } + } + + if !cd.hasArgsSupplied["warn"] && !cd.hasArgsSupplied["warning"] && !cd.hasArgsSupplied["warn+"] && !cd.hasArgsSupplied["warning+"] { + if macros["warn_count"] != "0" && macros["warn_count"] != "" { + cd.result.EscalateStatus(1) + macros["_state"] = "1" + } } if state, ok := cd.details["_state"]; ok { @@ -461,9 +514,15 @@ func (cd *CheckData) setStateFromMaps(macros map[string]string) { cd.details["_state"] = fmt.Sprintf("%d", cd.result.State) } -// Check tries warn/crit/ok conditions against given data and sets result state. +func (cd *CheckData) markCheckMultiThresholdSupplied(keyword string) { + if cd.name == "check_multi" { + cd.hasArgsSupplied[keyword] = true + } +} + +// Check tries warn/crit/unknown/ok conditions against given data and sets result state. // The data argument can be anything that has the correct keys that conditions use -func (cd *CheckData) Check(data map[string]string, warnCond, critCond, okCond ConditionList) { +func (cd *CheckData) Check(data map[string]string, warnCond, critCond, unknownCond, okCond ConditionList) { data["_state"] = fmt.Sprintf("%d", CheckExitOK) for i := range warnCond { @@ -480,6 +539,13 @@ func (cd *CheckData) Check(data map[string]string, warnCond, critCond, okCond Co } } + for i := range unknownCond { + if res, ok := unknownCond[i].Match(data); res && ok { + log.Debugf("This given data matched the UNKNOWN condition: '%s' ", unknownCond[i].DetailedString()) + data["_state"] = fmt.Sprintf("%d", CheckExitUnknown) + } + } + for i := range okCond { if res, ok := okCond[i].Match(data); res && ok { log.Debugf("This given data matched the OK condition: '%s' ", okCond[i].DetailedString()) @@ -492,6 +558,10 @@ func (cd *CheckData) Check(data map[string]string, warnCond, critCond, okCond Co func (cd *CheckData) CheckMetrics(okCond ConditionList) { // each metric is ran through conditions individually for _, metric := range cd.result.Metrics { + if metric.SkipStateCheck { + continue + } + state := CheckExitOK if metric.CheckForThresholds(&metric.Warning) { @@ -648,18 +718,20 @@ func (cd *CheckData) parseArgs(args []string) (argList []Argument, err error) { argList = make([]Argument, 0, len(args)) cd.expandArgDefinitions() - sanitized, defaultWarning, defaultCritical, applyDefaultFilter, err := cd.preParseArgs(args) + pre, err := cd.preParseArgs(args) if err != nil { return nil, err } + applyDefaultFilter := pre.applyDefaultFilter + // skip argument parsing for external scripts if _, ok := AvailableChecks[cd.name]; !ok && cd.argsPassthrough { - for _, arg := range sanitized { + for _, arg := range pre.sanitized { argList = append(argList, Argument{key: arg.key, value: arg.value}) } } else { - argList, applyDefaultFilter, err = cd.processArgs(sanitized, defaultWarning, defaultCritical, applyDefaultFilter) + argList, applyDefaultFilter, err = cd.processArgs(pre) if err != nil { return nil, err } @@ -670,7 +742,7 @@ func (cd *CheckData) parseArgs(args []string) (argList []Argument, err error) { cd.timezone = timeZone } - err = cd.setFallbacks(applyDefaultFilter, defaultWarning, defaultCritical) + err = cd.setFallbacks(applyDefaultFilter, pre.defaultWarning, pre.defaultCritical, pre.defaultUnknown) if err != nil { return nil, err } @@ -681,13 +753,13 @@ func (cd *CheckData) parseArgs(args []string) (argList []Argument, err error) { return argList, nil } -//nolint:funlen,gocyclo // it is not complex, it is just a long list of options -func (cd *CheckData) processArgs(sanitized []Argument, defaultWarning, defaultCritical string, initialApplyDefaultFilter bool) (argList []Argument, applyDefaultFilter bool, err error) { +//nolint:funlen,gocyclo,maintidx // it is not complex, it is just a long list of options +func (cd *CheckData) processArgs(pre *preParsedArgs) (argList []Argument, applyDefaultFilter bool, err error) { topSupplied := false okSupplied := false - applyDefaultFilter = initialApplyDefaultFilter + applyDefaultFilter = pre.applyDefaultFilter - for _, arg := range sanitized { + for _, arg := range pre.sanitized { keyword := arg.key argValue := arg.value argExpr := arg.raw @@ -708,30 +780,51 @@ func (cd *CheckData) processArgs(sanitized []Argument, defaultWarning, defaultCr } cd.okThreshold = append(cd.okThreshold, cond) case "warn+", "warning+": - warn, err2 := cd.appendDefaultThreshold(keyword, argValue, defaultWarning, cd.warnThreshold) + warn, err2 := cd.appendDefaultThreshold(keyword, argValue, pre.defaultWarning, cd.warnThreshold) if err2 != nil { return nil, false, err2 } cd.warnThreshold = warn + cd.markCheckMultiThresholdSupplied(keyword) case "warn", "warning": cond, err2 := NewCondition(argValue, &cd.attributes) if err2 != nil { return nil, false, err2 } cd.warnThreshold = append(cd.warnThreshold, cond) + cd.markCheckMultiThresholdSupplied(keyword) case "crit+", "critical+": - crit, err2 := cd.appendDefaultThreshold(keyword, argValue, defaultCritical, cd.critThreshold) + crit, err2 := cd.appendDefaultThreshold(keyword, argValue, pre.defaultCritical, cd.critThreshold) if err2 != nil { return nil, false, err2 } cd.critThreshold = crit + cd.markCheckMultiThresholdSupplied(keyword) case "crit", "critical": cond, err2 := NewCondition(argValue, &cd.attributes) if err2 != nil { return nil, false, err2 } cd.critThreshold = append(cd.critThreshold, cond) + cd.markCheckMultiThresholdSupplied(keyword) + case "unknown+": + unknown, err2 := cd.appendDefaultThreshold(keyword, argValue, pre.defaultUnknown, cd.unknownThreshold) + if err2 != nil { + return nil, false, err2 + } + cd.unknownThreshold = unknown + cd.markCheckMultiThresholdSupplied(keyword) + case "unknown": + cond, err2 := NewCondition(argValue, &cd.attributes) + if err2 != nil { + return nil, false, err2 + } + cd.unknownThreshold = append(cd.unknownThreshold, cond) + cd.markCheckMultiThresholdSupplied(keyword) case "filter+": + if cd.disableFilter { + return nil, false, fmt.Errorf("%s is disabled for this check", keyword) + } applyDefaultFilter = false filter, err2 := cd.appendDefaultThreshold(keyword, argValue, cd.defaultFilter, cd.filter) if err2 != nil { @@ -739,6 +832,9 @@ func (cd *CheckData) processArgs(sanitized []Argument, defaultWarning, defaultCr } cd.filter = filter case "filter": + if cd.disableFilter { + return nil, false, fmt.Errorf("%s is disabled for this check", keyword) + } applyDefaultFilter = false cond, err2 := NewCondition(argValue, &cd.attributes) if err2 != nil { @@ -826,12 +922,23 @@ func (cd *CheckData) processArgs(sanitized []Argument, defaultWarning, defaultCr return argList, applyDefaultFilter, nil } -func (cd *CheckData) preParseArgs(args []string) (sanitized []Argument, defaultWarning, defaultCritical string, hasArgsFilter bool, err error) { - sanitized = make([]Argument, 0) +type preParsedArgs struct { + sanitized []Argument + defaultWarning string + defaultCritical string + defaultUnknown string + applyDefaultFilter bool +} + +func (cd *CheckData) preParseArgs(args []string) (pre *preParsedArgs, err error) { + pre = &preParsedArgs{ + sanitized: make([]Argument, 0), + applyDefaultFilter: true, + defaultWarning: cd.defaultWarning, + defaultCritical: cd.defaultCritical, + defaultUnknown: cd.defaultUnknown, + } numArgs := len(args) - applyDefaultFilter := true - defaultWarning = cd.defaultWarning - defaultCritical = cd.defaultCritical for idx := 0; idx < numArgs; idx++ { argExpr := cd.removeQuotes(args[idx]) @@ -839,7 +946,7 @@ func (cd *CheckData) preParseArgs(args []string) (sanitized []Argument, defaultW keyword := cd.removeQuotes(split[0]) argValue, newIdx, err2 := cd.fetchNextArg(args, split, keyword, idx, numArgs) if err2 != nil { - return nil, "", "", false, err2 + return pre, err2 } idx = newIdx argValue = cd.removeQuotes(argValue) @@ -851,19 +958,22 @@ func (cd *CheckData) preParseArgs(args []string) (sanitized []Argument, defaultW chkArg = &a } if chkArg != nil { - applyDefaultFilter = false + pre.applyDefaultFilter = false cd.hasArgsFilter = true if chkArg.defaultWarning != "" { - defaultWarning = chkArg.defaultWarning + pre.defaultWarning = chkArg.defaultWarning } if chkArg.defaultCritical != "" { - defaultCritical = chkArg.defaultCritical + pre.defaultCritical = chkArg.defaultCritical + } + if chkArg.defaultUnknown != "" { + pre.defaultUnknown = chkArg.defaultUnknown } } - sanitized = append(sanitized, Argument{key: keyword, value: argValue, raw: argExpr}) + pre.sanitized = append(pre.sanitized, Argument{key: keyword, value: argValue, raw: argExpr}) } - return sanitized, defaultWarning, defaultCritical, applyDefaultFilter, nil + return pre, nil } // Threshold keywords do not necessarily have to match an attribute name. @@ -889,6 +999,14 @@ func (cd *CheckData) checkThresholdKeywordsAgainstAttributeNames() { } } + unknownKeywords, err := cd.unknownThreshold.GetListOfKeywords() + if err == nil && len(unknownKeywords) > 0 { + unknownKeywordsExtra := utils.SubtractSlice(unknownKeywords, attributeNames) + if len(unknownKeywordsExtra) > 0 { + log.Tracef("Unknown condition uses keyword(s) not present in the attributes, run with --help to get a list of attributes, extra keywords: %v", unknownKeywordsExtra) + } + } + okKeywords, err := cd.okThreshold.GetListOfKeywords() if err == nil && len(okKeywords) > 0 { okKeywordsExtra := utils.SubtractSlice(okKeywords, attributeNames) @@ -947,9 +1065,13 @@ func (cd *CheckData) fetchNextArg(args, split []string, keyword string, idx, num if len(split) == 2 { return split[1], idx, nil } - arg, ok := cd.args[keyword] + lookupKey := keyword + if before, _, found := strings.Cut(keyword, "["); found && strings.HasSuffix(keyword, "]") { + lookupKey = before + } + arg, ok := cd.args[lookupKey] if !ok { - arg, ok = cd.extraArgs[keyword] + arg, ok = cd.extraArgs[lookupKey] if !ok { return "", idx, nil } @@ -969,29 +1091,79 @@ func (cd *CheckData) fetchNextArg(args, split []string, keyword string, idx, num return args[idx], idx, nil } +// parseTaggedCommand handles parsing a TaggedCommandList argument (command[tag]=...). +func (cd *CheckData) parseTaggedCommand(argRef *TaggedCommandList, tag, argValue string) error { + tag = strings.TrimSpace(tag) + if tag == "" { + return fmt.Errorf("command argument requires a unique tag, e.g. command[tag]=") + } + + if strings.ContainsAny(tag, DefaultNastyCharacters+"=") { + return fmt.Errorf("command tag contains invalid characters: %s", tag) + } + + for _, existing := range *argRef { + if existing.Tag == tag { + return fmt.Errorf("duplicate command tag: %s", tag) + } + } + + *argRef = append(*argRef, TaggedCommand{ + Tag: tag, + Command: strings.TrimSpace(argValue), + }) + + return nil +} + // parseAnyArg parses args into the args map with custom arguments func (cd *CheckData) parseAnyArg(argExpr, keyword, argValue string) (bool, error) { - arg, ok := cd.args[keyword] + lookupKey := keyword + tag := "" + hasTag := false + if before, rest, found := strings.Cut(keyword, "["); found && strings.HasSuffix(keyword, "]") { + lookupKey = before + tag = rest[:len(rest)-1] + hasTag = true + } + + arg, ok := cd.args[lookupKey] if !ok { - arg, ok = cd.extraArgs[keyword] + arg, ok = cd.extraArgs[lookupKey] if !ok { return false, nil } } + if hasTag { + if _, tagged := arg.value.(*TaggedCommandList); !tagged { + return false, fmt.Errorf("argument %s does not support tags", lookupKey) + } + } + + if err := cd.parseArgValue(argExpr, keyword, argValue, tag, &arg); err != nil { + return true, err + } + + cd.hasArgsSupplied[keyword] = true + + return true, nil +} +// parseArgValue dispatches an argument value into the correct typed storage reference. +func (cd *CheckData) parseArgValue(argExpr, keyword, argValue, tag string, arg *CheckArgument) error { //nolint:cyclop // many type cases are required here switch argRef := arg.value.(type) { + case *TaggedCommandList: + return cd.parseTaggedCommand(argRef, tag, argValue) case *[]string: if _, ok := cd.hasArgsSupplied[keyword]; !ok { // first time this arg occurs, empty default lists - empty := make([]string, 0) - *argRef = empty + *argRef = make([]string, 0) } *argRef = append(*argRef, argValue) case *CommaStringList: if _, ok := cd.hasArgsSupplied[keyword]; !ok { // first time this arg occurs, empty default lists - empty := make([]string, 0) - *argRef = empty + *argRef = make([]string, 0) } *argRef = append(*argRef, strings.Split(argValue, ",")...) case *string: @@ -999,29 +1171,28 @@ func (cd *CheckData) parseAnyArg(argExpr, keyword, argValue string) (bool, error case *float64: f, err := strconv.ParseFloat(argValue, 64) if err != nil { - return true, fmt.Errorf("parseFloat %s: %s", argExpr, err.Error()) + return fmt.Errorf("parseFloat %s: %s", argExpr, err.Error()) } *argRef = f case *int64: i, err := strconv.ParseInt(argValue, 10, 64) if err != nil { - return true, fmt.Errorf("parseInt %s: %s", argExpr, err.Error()) + return fmt.Errorf("parseInt %s: %s", argExpr, err.Error()) } *argRef = i case *int: i, err := strconv.ParseInt(argValue, 10, 32) if err != nil { - return true, fmt.Errorf("parseInt %s: %s", argExpr, err.Error()) + return fmt.Errorf("parseInt %s: %s", argExpr, err.Error()) } *argRef = int(i) case *bool: if argValue == "" { - b := true - *argRef = b + *argRef = true } else { b, err := convert.BoolE(argValue) if err != nil { - return true, fmt.Errorf("parseBool %s: %s", argValue, err.Error()) + return fmt.Errorf("parseBool %s: %s", argValue, err.Error()) } *argRef = b } @@ -1029,9 +1200,7 @@ func (cd *CheckData) parseAnyArg(argExpr, keyword, argValue string) (bool, error log.Errorf("unsupported args type: %T in %s", argRef, argExpr) } - cd.hasArgsSupplied[keyword] = true - - return true, nil + return nil } // removeQuotes remove single/double quotes around string @@ -1053,8 +1222,8 @@ func (cd *CheckData) removeQuotes(str string) string { return str } -// setFallbacks sets default filter/warn/crit thresholds unless already set. -func (cd *CheckData) setFallbacks(applyDefaultFilter bool, defaultWarning, defaultCritical string) error { +// setFallbacks sets default filter/warn/crit/unknown thresholds unless already set. +func (cd *CheckData) setFallbacks(applyDefaultFilter bool, defaultWarning, defaultCritical, defaultUnknown string) error { if applyDefaultFilter && cd.defaultFilter != "" { cond, err := NewCondition(cd.defaultFilter, &cd.attributes) if err != nil { @@ -1063,16 +1232,20 @@ func (cd *CheckData) setFallbacks(applyDefaultFilter bool, defaultWarning, defau cd.filter = append(cd.filter, cond) } - // default warning/critical overridden from check arguments, ex. check_service + // default warning/critical/unknown overridden from check arguments, ex. check_service if defaultWarning != "" { cd.defaultWarning = defaultWarning } if defaultCritical != "" { cd.defaultCritical = defaultCritical } + if defaultUnknown != "" { + cd.defaultUnknown = defaultUnknown + } cd.warnThreshold = cd.applyDefaultThreshold(cd.defaultWarning, cd.warnThreshold) cd.critThreshold = cd.applyDefaultThreshold(cd.defaultCritical, cd.critThreshold) + cd.unknownThreshold = cd.applyDefaultThreshold(cd.defaultUnknown, cd.unknownThreshold) if cd.timeout == 0 { cd.timeout = DefaultCheckTimeout.Seconds() @@ -1122,6 +1295,7 @@ func (cd *CheckData) applyConditionColAlias() { cd.applyConditionColAliasList(cd.filter) cd.applyConditionColAliasList(cd.warnThreshold) cd.applyConditionColAliasList(cd.critThreshold) + cd.applyConditionColAliasList(cd.unknownThreshold) cd.applyConditionColAliasList(cd.okThreshold) } @@ -1167,6 +1341,7 @@ func (cd *CheckData) applyConditionAlias() { cd.applyConditionAliasList(cd.filter) cd.applyConditionAliasList(cd.warnThreshold) cd.applyConditionAliasList(cd.critThreshold) + cd.applyConditionAliasList(cd.unknownThreshold) cd.applyConditionAliasList(cd.okThreshold) } @@ -1203,6 +1378,9 @@ func (cd *CheckData) HasThreshold(name string) bool { if cd.hasThresholdCond(cd.critThreshold, name) { return true } + if cd.hasThresholdCond(cd.unknownThreshold, name) { + return true + } if cd.hasThresholdCond(cd.okThreshold, name) { return true } @@ -1210,16 +1388,18 @@ func (cd *CheckData) HasThreshold(name string) bool { return false } -// GetAllThresholdKeywords returns a list of all keywords used in warn/crit/ok thresholds. +// GetAllThresholdKeywords returns a list of all keywords used in warn/crit/unknown/ok thresholds. func (cd *CheckData) GetAllThresholdKeywords() []string { - keywords := make([]string, 0, len(cd.warnThreshold)+len(cd.critThreshold)+len(cd.okThreshold)) + keywords := make([]string, 0, len(cd.warnThreshold)+len(cd.critThreshold)+len(cd.unknownThreshold)+len(cd.okThreshold)) warnThresholdKeywords, _ := cd.warnThreshold.GetListOfKeywords() critThresholdKeywords, _ := cd.critThreshold.GetListOfKeywords() + unknownThresholdKeywords, _ := cd.unknownThreshold.GetListOfKeywords() okThresholdKeywords, _ := cd.okThreshold.GetListOfKeywords() keywords = append(keywords, warnThresholdKeywords...) keywords = append(keywords, critThresholdKeywords...) + keywords = append(keywords, unknownThresholdKeywords...) keywords = append(keywords, okThresholdKeywords...) utils.Deduplicate(keywords) @@ -1259,6 +1439,7 @@ func (cd *CheckData) SetDefaultThresholdUnit(defaultUnit string, names []string) } cd.VisitAll(cd.warnThreshold, setDefault) cd.VisitAll(cd.critThreshold, setDefault) + cd.VisitAll(cd.unknownThreshold, setDefault) cd.VisitAll(cd.okThreshold, setDefault) cd.VisitAll(cd.filter, setDefault) } @@ -1660,6 +1841,9 @@ func (cd *CheckData) helpDefaultArguments(format ShowHelp) string { if cd.defaultCritical != "" { defaultArgs = append(defaultArgs, defaultArg{name: "critical", defaults: cd.defaultCritical}) } + if cd.defaultUnknown != "" { + defaultArgs = append(defaultArgs, defaultArg{name: "unknown", defaults: cd.defaultUnknown}) + } defaultArgs = append( defaultArgs, defaultArg{name: "empty-state", defaults: fmt.Sprintf("%d (%s)", cd.emptyState, convert.StateString(cd.emptyState))}, diff --git a/pkg/snclient/checkmetric.go b/pkg/snclient/checkmetric.go index c67f4f20..6398d177 100644 --- a/pkg/snclient/checkmetric.go +++ b/pkg/snclient/checkmetric.go @@ -14,18 +14,19 @@ import ( // CheckMetric contains a single performance value. type CheckMetric struct { - Name string // Name as used in the perf data string - Unit string // Unit of the value - Value any // Current value - ThresholdName string // if set, this will be added to the data before checking a conditions - Warning ConditionList // threshold used for warnings - WarningStr *string // set warnings from string - Critical ConditionList // threshold used for critical - CriticalStr *string // set critical from string - Min *float64 - Max *float64 - PerfConfig *PerfConfig // apply perf tweaks - Entry map[string]string // entry that this metric is generated from + Name string // Name as used in the perf data string + Unit string // Unit of the value + Value any // Current value + ThresholdName string // if set, this will be added to the data before checking a conditions + Warning ConditionList // threshold used for warnings + WarningStr *string // set warnings from string + Critical ConditionList // threshold used for critical + CriticalStr *string // set critical from string + SkipStateCheck bool // do not use warning or critical conditions for state + Min *float64 + Max *float64 + PerfConfig *PerfConfig // apply perf tweaks + Entry map[string]string // entry that this metric is generated from } // generates a naemon like string, including the perfdata diff --git a/pkg/snclient/checkmetric_test.go b/pkg/snclient/checkmetric_test.go index 8f17c69c..a1823b9f 100644 --- a/pkg/snclient/checkmetric_test.go +++ b/pkg/snclient/checkmetric_test.go @@ -19,3 +19,22 @@ func TestCheckMetricsString(t *testing.T) { assert.Equalf(t, check.expect, res, "CheckMetric.String() ->> %s", res) } } + +func TestCheckMetricsSkipStateCheck(t *testing.T) { + metric := &CheckMetric{ + Name: "value", + Value: 1, + Warning: ConditionList{{ + keyword: "value", + operator: Greater, + value: float64(0), + }}, + SkipStateCheck: true, + } + check := &CheckData{result: &CheckResult{Metrics: []*CheckMetric{metric}}} + + check.CheckMetrics(nil) + + assert.Equal(t, CheckExitOK, check.result.State) + assert.Equal(t, "'value'=1;0", metric.String()) +} diff --git a/pkg/snclient/checkresult.go b/pkg/snclient/checkresult.go index bd5cba5b..f926c75d 100644 --- a/pkg/snclient/checkresult.go +++ b/pkg/snclient/checkresult.go @@ -2,6 +2,7 @@ package snclient import ( "bytes" + "fmt" "regexp" "strconv" "strings" @@ -29,11 +30,12 @@ var reValuesUnit = regexp.MustCompile(`^([0-9.]+)(.*?)$`) // CheckResult is the result of a single check run. type CheckResult struct { - State int64 // naemon exit code: OK=0, Warning=1, Critical=2, Unknown=3 - Output string // plugin output, should be human readable - Metrics []*CheckMetric // performance data metrics - Raw *CheckData // reference to the original check data, for use in inventory and other checks - Details string // additional details that should be printed on a new line after the main output, e.g. for showing top consuming processes + State int64 // naemon exit code: OK=0, Warning=1, Critical=2, Unknown=3 + Output string // plugin output, should be human readable + Metrics []*CheckMetric // performance data metrics + Raw *CheckData // reference to the original check data, for use in inventory and other checks + Details string // additional details that should be printed on a new line after the main output, e.g. for showing top consuming processes + literalDetails map[string]string } func (cr *CheckResult) Finalize(timezone *time.Location, macros ...map[string]string) { @@ -59,6 +61,20 @@ func (cr *CheckResult) Finalize(timezone *time.Location, macros ...map[string]st cr.Output = ReplaceMacros(cr.Output, timezone, macroSet...) } cr.Details = ReplaceMacros(cr.Details, timezone, macroSet...) + for placeholder, literal := range cr.literalDetails { + cr.Details = strings.ReplaceAll(cr.Details, placeholder, literal) + } + cr.literalDetails = nil +} + +func (cr *CheckResult) LiteralizeDetails(value string) string { + if cr.literalDetails == nil { + cr.literalDetails = make(map[string]string) + } + placeholder := fmt.Sprintf("\x00snclient-literal-%d\x00", len(cr.literalDetails)) + cr.literalDetails[placeholder] = value + + return placeholder } func (cr *CheckResult) ApplyPerfConfig(perfCfg []PerfConfig) error { diff --git a/pkg/snclient/config.go b/pkg/snclient/config.go index 1b24f028..4d577d55 100644 --- a/pkg/snclient/config.go +++ b/pkg/snclient/config.go @@ -48,6 +48,7 @@ var DefaultConfig = map[string]ConfigData{ "CheckSystem": "enabled", "CheckSystemUnix": "enabled", "CheckAlias": "enabled", + "CheckMulti": "enabled", "CheckExternalScripts": "enabled", "CheckDisk": "enabled", "CheckDriveIO": "enabled", @@ -325,6 +326,7 @@ func (config *Config) ParseINI(configData, iniPath string, snc *Agent) error { // parse key and value val := strings.SplitN(line, "=", 2) + if len(val) < 2 { parseErrors = append(parseErrors, fmt.Errorf("parse error in %s:%d: found key without '='", iniPath, lineNr))