diff --git a/.gitignore b/.gitignore index 18d343398..8c6eaa7df 100644 --- a/.gitignore +++ b/.gitignore @@ -7,6 +7,7 @@ web/screenshots/ web/cypress/videos/ web/dist/ web/node_modules/ +e2e.test plugin-backend # Leave these permanently for backwards compatability for backporting node_modules/ diff --git a/Makefile b/Makefile index 159128c23..4a7bf4aaf 100644 --- a/Makefile +++ b/Makefile @@ -62,6 +62,10 @@ start-backend: test-backend: go test ./pkg/... ./internal/... -v +.PHONY: build-e2e +build-e2e: + go test -c -tags e2e ./test/e2e + .PHONY: test-e2e test-e2e: go test -tags e2e -v -timeout=150m -count=1 ./test/e2e diff --git a/test/e2e/create_alert_rule_test.go b/test/e2e/create_alert_rule_test.go index 0e2ace5cf..bcedc7dbb 100644 --- a/test/e2e/create_alert_rule_test.go +++ b/test/e2e/create_alert_rule_test.go @@ -4,8 +4,12 @@ package e2e import ( "context" + "errors" + "fmt" "testing" + "time" + "github.com/stretchr/testify/require" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "github.com/openshift/monitoring-plugin/internal/managementrouter" @@ -27,7 +31,7 @@ func TestCreateUserDefinedAlertRule(t *testing.T) { defer cleanup() createExpr := "vector(1) or vector(0)" - id, err := createRuleViaAPI(ctx, f, managementrouter.CreateAlertRuleRequest{ + createAlertRuleRequest := managementrouter.CreateAlertRuleRequest{ AlertingRule: &managementrouter.AlertRuleSpec{ Alert: new("E2ECreateAlert"), Expr: &createExpr, @@ -43,47 +47,43 @@ func TestCreateUserDefinedAlertRule(t *testing.T) { PrometheusRuleName: "e2e-create-pr", PrometheusRuleNamespace: testNamespace, }, - }) - if err != nil { - t.Fatalf("Failed to create alert rule: %v", err) } + id, err := createRuleViaAPIWithRetry(ctx, f, createAlertRuleRequest) + require.NoError(t, err) + require.NotEmpty(t, id) + t.Logf("Created rule with ID: %s", id) - promRule, err := f.Monitoringv1clientset.MonitoringV1().PrometheusRules(testNamespace).Get( - ctx, "e2e-create-pr", metav1.GetOptions{}, - ) - if err != nil { - t.Fatalf("Failed to get PrometheusRule: %v", err) - } + err = poll(time.Second, time.Minute, func() error { + promRule, err := f.Monitoringv1clientset.MonitoringV1().PrometheusRules(testNamespace).Get( + ctx, "e2e-create-pr", metav1.GetOptions{}, + ) + if err != nil { + return fmt.Errorf("failed to get PrometheusRule: %w", err) + } - if len(promRule.Spec.Groups) == 0 { - t.Fatal("Expected at least one rule group in PrometheusRule") - } + for _, group := range promRule.Spec.Groups { + for _, rule := range group.Rules { + if rule.Alert == "E2ECreateAlert" { + if rule.Expr.String() != createExpr { + return fmt.Errorf("expected expr %q, got %q", createExpr, rule.Expr.String()) + } + if rule.For == nil || string(*rule.For) != "1m" { + return fmt.Errorf("expected for '1m', got %v", rule.For) + } + if rule.Labels["severity"] != "info" { + return fmt.Errorf("expected severity=info, got %q", rule.Labels["severity"]) + } + if rule.Annotations["summary"] != "E2E test alert for create-rule" { + return fmt.Errorf("expected summary annotation, got %q", rule.Annotations["summary"]) + } - var foundAlert bool - for _, group := range promRule.Spec.Groups { - for _, rule := range group.Rules { - if rule.Alert == "E2ECreateAlert" { - foundAlert = true - if rule.Expr.String() != createExpr { - t.Errorf("Expected expr %q, got %q", createExpr, rule.Expr.String()) - } - if rule.For == nil || string(*rule.For) != "1m" { - t.Errorf("Expected for '1m', got %v", rule.For) - } - if rule.Labels["severity"] != "info" { - t.Errorf("Expected severity=info, got %q", rule.Labels["severity"]) - } - if rule.Annotations["summary"] != "E2E test alert for create-rule" { - t.Errorf("Expected summary annotation, got %q", rule.Annotations["summary"]) + return nil } } } - } - if !foundAlert { - t.Fatal("Alert 'E2ECreateAlert' not found in PrometheusRule") - } - - t.Log("Create alert rule e2e test passed successfully") + return errors.New("alerting rule 'E2ECreateAlert' not found in PrometheusRule") + }) + require.NoError(t, err) } diff --git a/test/e2e/delete_alert_rule_test.go b/test/e2e/delete_alert_rule_test.go index a806ca35c..cac1e28ad 100644 --- a/test/e2e/delete_alert_rule_test.go +++ b/test/e2e/delete_alert_rule_test.go @@ -12,6 +12,7 @@ import ( "testing" "time" + "github.com/stretchr/testify/require" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "github.com/openshift/monitoring-plugin/internal/managementrouter" @@ -37,7 +38,7 @@ func TestDeleteAlertRule(t *testing.T) { for _, name := range ruleNames { expr := fmt.Sprintf("absent(nonexistent{e2e_rule=%q})", name) - id, err := createRuleViaAPI(ctx, f, managementrouter.CreateAlertRuleRequest{ + alertRuleRequest := managementrouter.CreateAlertRuleRequest{ AlertingRule: &managementrouter.AlertRuleSpec{ Alert: new(name), Expr: &expr, @@ -50,17 +51,18 @@ func TestDeleteAlertRule(t *testing.T) { PrometheusRuleName: "e2e-delete-pr", PrometheusRuleNamespace: testNamespace, }, - }) + } + + id, err := createRuleViaAPIWithRetry(ctx, f, alertRuleRequest) if err != nil { - t.Fatalf("Failed to create alert rule %s: %v", name, err) + t.Fatalf("failed to create alert rule %s: %v", name, err) } ruleIDs = append(ruleIDs, id) + } t.Logf("Created 3 rules with IDs: %v", ruleIDs) - time.Sleep(2 * time.Second) - deleteReq := managementrouter.BulkDeleteAlertRulesRequest{ RuleIds: []string{ruleIDs[0], ruleIDs[1]}, } @@ -69,61 +71,73 @@ func TestDeleteAlertRule(t *testing.T) { t.Fatalf("Failed to marshal delete request: %v", err) } - deleteURL := f.PluginURL + "/api/v1/alerting/rules" - req, err := http.NewRequestWithContext(ctx, http.MethodDelete, deleteURL, bytes.NewBuffer(reqBody)) - if err != nil { - t.Fatalf("Failed to create delete request: %v", err) - } - req.Header.Set("Content-Type", "application/json") - if f.BearerToken != "" { - req.Header.Set("Authorization", "Bearer "+f.BearerToken) - } + err = poll(time.Second, time.Minute, func() error { + deleteURL := f.PluginURL + "/api/v1/alerting/rules" + req, err := http.NewRequestWithContext(ctx, http.MethodDelete, deleteURL, bytes.NewBuffer(reqBody)) + if err != nil { + return fmt.Errorf("failed to create delete request: %w", err) + } + req.Header.Set("Content-Type", "application/json") + if f.BearerToken != "" { + req.Header.Set("Authorization", "Bearer "+f.BearerToken) + } - resp, err := f.HTTPClient().Do(req) - if err != nil { - t.Fatalf("Failed to make delete request: %v", err) - } - defer resp.Body.Close() + resp, err := f.HTTPClient().Do(req) + if err != nil { + return fmt.Errorf("failed to make delete request: %w", err) + } + defer resp.Body.Close() + + if resp.StatusCode != http.StatusOK { + body, err := io.ReadAll(resp.Body) + if err != nil { + return fmt.Errorf("failed to read body: %w", err) + } + return fmt.Errorf("expected status 200, got %d (body: %s)", resp.StatusCode, string(body)) + } - if resp.StatusCode != http.StatusOK { - body, _ := io.ReadAll(resp.Body) - t.Fatalf("Expected status 200, got %d. Body: %s", resp.StatusCode, string(body)) - } + var deleteResp managementrouter.BulkDeleteAlertRulesResponse + if err := json.NewDecoder(resp.Body).Decode(&deleteResp); err != nil { + return fmt.Errorf("failed to decode delete response: %w", err) + } - var deleteResp managementrouter.BulkDeleteAlertRulesResponse - if err := json.NewDecoder(resp.Body).Decode(&deleteResp); err != nil { - t.Fatalf("Failed to decode delete response: %v", err) - } + if len(deleteResp.Rules) != 2 { + return fmt.Errorf("expected 2 results, got %d", len(deleteResp.Rules)) + } + for _, result := range deleteResp.Rules { + if result.StatusCode != http.StatusNoContent { + return fmt.Errorf("rule %s deletion failed with status %d: %v", result.Id, result.StatusCode, result.Message) + } + } - if len(deleteResp.Rules) != 2 { - t.Fatalf("Expected 2 results, got %d", len(deleteResp.Rules)) - } - for _, result := range deleteResp.Rules { - if result.StatusCode != http.StatusNoContent { - t.Errorf("Rule %s deletion failed with status %d: %v", result.Id, result.StatusCode, result.Message) + return nil + }) + require.NoError(t, err) + + err = poll(time.Second, 20*time.Second, func() error { + promRule, err := f.Monitoringv1clientset.MonitoringV1().PrometheusRules(testNamespace).Get( + ctx, "e2e-delete-pr", metav1.GetOptions{}, + ) + if err != nil { + return fmt.Errorf("failed to get PrometheusRule after deletion: %w", err) } - } - promRule, err := f.Monitoringv1clientset.MonitoringV1().PrometheusRules(testNamespace).Get( - ctx, "e2e-delete-pr", metav1.GetOptions{}, - ) - if err != nil { - t.Fatalf("Failed to get PrometheusRule after deletion: %v", err) - } + var remainingAlerts []string + for _, group := range promRule.Spec.Groups { + for _, rule := range group.Rules { + remainingAlerts = append(remainingAlerts, rule.Alert) + } + } - var remainingAlerts []string - for _, group := range promRule.Spec.Groups { - for _, rule := range group.Rules { - remainingAlerts = append(remainingAlerts, rule.Alert) + if len(remainingAlerts) != 1 { + return fmt.Errorf("expected 1 remaining rule, got %d: %v", len(remainingAlerts), remainingAlerts) } - } - if len(remainingAlerts) != 1 { - t.Fatalf("Expected 1 remaining rule, got %d: %v", len(remainingAlerts), remainingAlerts) - } - if remainingAlerts[0] != "KeepAlert3" { - t.Errorf("Expected remaining rule 'KeepAlert3', got %q", remainingAlerts[0]) - } + if remainingAlerts[0] != "KeepAlert3" { + return fmt.Errorf("expected remaining rule 'KeepAlert3', got %q", remainingAlerts[0]) + } - t.Log("Delete alert rule e2e test passed successfully") + return nil + }) + require.NoError(t, err) } diff --git a/test/e2e/helpers_test.go b/test/e2e/helpers_test.go index f21784944..481634de8 100644 --- a/test/e2e/helpers_test.go +++ b/test/e2e/helpers_test.go @@ -10,11 +10,47 @@ import ( "io" "net/http" "net/url" + "time" + + "k8s.io/apimachinery/pkg/util/wait" "github.com/openshift/monitoring-plugin/internal/managementrouter" "github.com/openshift/monitoring-plugin/test/e2e/framework" ) +// poll calls the given function f() every given interval +// until it returns no error or the given timeout occurs. +// When a timeout occurs, the last observed error is returned +// wrapped in a wait.ErrWaitTimeout. +func poll(interval, timeout time.Duration, f func() error) error { + var lastErr error + err := wait.PollUntilContextTimeout(context.Background(), interval, timeout, true, func(context.Context) (bool, error) { + if lastErr = f(); lastErr != nil { + return false, nil + } + + return true, nil + }) + if err != nil && lastErr != nil { + return fmt.Errorf("%w: %w", err, lastErr) + } + + return err +} + +func createRuleViaAPIWithRetry(ctx context.Context, f *framework.Framework, createAlertRuleRequest managementrouter.CreateAlertRuleRequest) (string, error) { + var id string + err := poll(time.Second, 20*time.Second, func() error { + var err error + id, err = createRuleViaAPI(ctx, f, createAlertRuleRequest) + if err != nil { + return fmt.Errorf("failed to create alert rule: %w", err) + } + return nil + }) + return id, err +} + func createRuleViaAPI(ctx context.Context, f *framework.Framework, payload managementrouter.CreateAlertRuleRequest) (string, error) { reqBody, err := json.Marshal(payload) if err != nil {