Skip to content

Introduce tenant uuid map for rules + prevent infinite generation #45

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
wants to merge 1 commit into
base: main
Choose a base branch
from
Open

Introduce tenant uuid map for rules + prevent infinite generation #45

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
34 changes: 34 additions & 0 deletions main.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3,8 +3,10 @@ package main
import (
"context"
"flag"
"fmt"
"net/http"
"os"
"strings"
"syscall"

"github.com/go-kit/log"
Expand Down Expand Up @@ -36,6 +38,7 @@ type cfg struct {
observatoriumURL string
sleepDurationSeconds uint
managedTenants string
tenantUUIDs string
audience string
issuerURL string
logRulesEnabled bool
Expand Down Expand Up @@ -74,6 +77,7 @@ func parseFlags() *cfg {
flag.UintVar(&cfg.configReloadInterval, "config-reload-interval-seconds", defaultConfigReloadIntervalSeconds, "The interval in seconds for reloading configuration.")
flag.StringVar(&cfg.observatoriumURL, "observatorium-api-url", "", "The URL of the Observatorium API to which rules will be synced.")
flag.StringVar(&cfg.managedTenants, "managed-tenants", "", "The name of the tenants whose rules should be synced. If there are multiple tenants, ensure they are comma-separated.")
flag.StringVar(&cfg.tenantUUIDs, "tenant-uuids", "", "Mapping of tenant names to UUIDs in format 'tenant1=uuid1,tenant2=uuid2'")
flag.StringVar(&cfg.issuerURL, "issuer-url", "", "The OIDC issuer URL, see https://openid.net/specs/openid-connect-discovery-1_0.html#IssuerDiscovery.")
flag.StringVar(&cfg.audience, "audience", "", "The audience for whom the access token is intended, see https://openid.net/specs/openid-connect-core-1_0.html#IDToken.")
flag.BoolVar(&cfg.logRulesEnabled, "log-rules-enabled", false, "Enable syncing Loki logging rules.")
Expand All @@ -85,6 +89,28 @@ func parseFlags() *cfg {
return cfg
}

func parseTenantUUIDs(tenantUUIDsStr string) (map[string]string, error) {
if tenantUUIDsStr == "" {
return nil, nil
}

tenantUUIDs := make(map[string]string)
pairs := strings.Split(tenantUUIDsStr, ",")
for _, pair := range pairs {
parts := strings.Split(pair, "=")
if len(parts) != 2 {
return nil, fmt.Errorf("invalid tenant UUID mapping format: %s", pair)
}
tenant := strings.TrimSpace(parts[0])
uuid := strings.TrimSpace(parts[1])
if tenant == "" || uuid == "" {
return nil, fmt.Errorf("empty tenant or UUID in mapping: %s", pair)
}
tenantUUIDs[tenant] = uuid
}
return tenantUUIDs, nil
}

func main() {
cfg := parseFlags()

Expand All @@ -98,6 +124,13 @@ func main() {
logger := setupLogger(cfg.logLevel)
defer level.Info(logger).Log("msg", "exiting")

// After parsing flags, parse the tenant UUIDs
tenantUUIDs, err := parseTenantUUIDs(cfg.tenantUUIDs)
if err != nil {
level.Error(logger).Log("msg", "error parsing tenant UUIDs", "error", err)
os.Exit(1)
}

// Create kubernetes client for deployments
k8sCfg, err := k8sconfig.GetConfig()
if err != nil {
Expand Down Expand Up @@ -150,6 +183,7 @@ func main() {
cfg.audience,
cfg.issuerURL,
cfg.managedTenants,
tenantUUIDs,
reg,
)
if err := o.InitOrReloadObsctlConfig(); err != nil {
Expand Down
42 changes: 25 additions & 17 deletions pkg/syncer/obsctlsyncer.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -41,11 +41,11 @@ type ObsctlRulesSyncer struct {
skipClientCheck bool
k8s client.Client
namespace string

apiURL string
audience string
issuerURL string
managedTenants string
tenantUUIDs map[string]string
apiURL string
audience string
issuerURL string
managedTenants string

autoDetectSecretsFn func(ctx context.Context,
k8s client.Client,
Expand All @@ -66,18 +66,19 @@ func NewObsctlRulesSyncer(
logger log.Logger,
kc client.Client,
namespace, apiURL, audience, issuerURL, managedTenants string,
tenantUUIDs map[string]string,
reg prometheus.Registerer,
) *ObsctlRulesSyncer {
return &ObsctlRulesSyncer{
ctx: ctx,
logger: logger,
k8s: kc,
apiURL: apiURL,
namespace: namespace,
audience: audience,
issuerURL: issuerURL,
managedTenants: managedTenants,

ctx: ctx,
logger: logger,
k8s: kc,
apiURL: apiURL,
namespace: namespace,
audience: audience,
issuerURL: issuerURL,
managedTenants: managedTenants,
tenantUUIDs: tenantUUIDs,
autoDetectSecretsFn: AutoDetectTenantSecrets,

lokiRulesSetOps: promauto.With(reg).NewCounterVec(prometheus.CounterOpts{
Expand Down Expand Up @@ -352,17 +353,24 @@ func (o *ObsctlRulesSyncer) MetricsSet(tenant string, rules monitoringv1.Prometh
level.Debug(o.logger).Log("msg", "setting metrics for tenant")
o.promRulesSetOps.WithLabelValues(string(tenant)).Inc()

tenantUUID, ok := o.tenantUUIDs[tenant]
if !ok {
level.Error(o.logger).Log("msg", "tenant UUID not found", "tenant", tenant)
return errors.Newf("tenant UUID not found: %s", tenant)
}

enforcer := injectproxy.NewEnforcer([]*labels.Matcher{{
Name: "tenant",
Type: labels.MatchEqual,
Value: tenant,
Value: tenantUUID,
}}...)

newRule := &monitoringv1.PrometheusRule{
ObjectMeta: metav1.ObjectMeta{
Name: fmt.Sprintf("prometheus-rules-%s-%d", tenant, time.Now().Unix()),
Labels: map[string]string{
"tenant": tenant,
"obsctl-reloader-generated-tenant": tenant,
"obsctl-reloader-generated-uuid": tenantUUID,
"operator.thanos.io/prometheus-rule": "true",
},
},
Expand All @@ -385,7 +393,7 @@ func (o *ObsctlRulesSyncer) MetricsSet(tenant string, rules monitoringv1.Prometh
if newRule.Labels == nil {
newRule.Labels = make(map[string]string)
}
newRule.Labels["tenant"] = string(tenant)
newRule.Labels["tenant"] = tenantUUID

// Modify PromQL expressions to include tenant label using prom-label-proxy
if rule.Record != "" {
Expand Down
26 changes: 18 additions & 8 deletions pkg/syncer/obsctlsyncer_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -51,10 +51,12 @@ func TestMetricsSet(t *testing.T) {
require.Len(t, rules.Items, 1)

rule := rules.Items[0]
require.Equal(t, "team-a", rule.Labels["tenant"])
require.Equal(t, "team-a", rule.Labels["obsctl-reloader-generated-tenant"])
require.Equal(t, "123", rule.Labels["obsctl-reloader-generated-uuid"])
require.Equal(t, "true", rule.Labels["operator.thanos.io/prometheus-rule"])
require.Equal(t, "metric:recording", rule.Spec.Groups[0].Rules[0].Record)
require.Equal(t, `sum(http_requests_total{tenant="team-a"})`, rule.Spec.Groups[0].Rules[0].Expr.String())
require.Equal(t, "123", rule.Spec.Groups[0].Rules[0].Labels["tenant"])
require.Equal(t, `sum(http_requests_total{tenant="123"})`, rule.Spec.Groups[0].Rules[0].Expr.String())
},
},
{
Expand Down Expand Up @@ -84,11 +86,12 @@ func TestMetricsSet(t *testing.T) {
require.Len(t, rules.Items, 1)

rule := rules.Items[0]
require.Equal(t, "team-b", rule.Labels["tenant"])
require.Equal(t, "team-b", rule.Labels["obsctl-reloader-generated-tenant"])
require.Equal(t, "456", rule.Labels["obsctl-reloader-generated-uuid"])
require.Equal(t, "HighErrorRate", rule.Spec.Groups[0].Rules[0].Alert)
require.Equal(t, "warning", rule.Spec.Groups[0].Rules[0].Labels["severity"])
require.Equal(t, "team-b", rule.Spec.Groups[0].Rules[0].Labels["tenant"])
require.Equal(t, `rate(errors_total{tenant="team-b"}[5m]) > 0.1`, rule.Spec.Groups[0].Rules[0].Expr.String())
require.Equal(t, "456", rule.Spec.Groups[0].Rules[0].Labels["tenant"])
require.Equal(t, `rate(errors_total{tenant="456"}[5m]) > 0.1`, rule.Spec.Groups[0].Rules[0].Expr.String())
},
},
{
Expand Down Expand Up @@ -118,10 +121,12 @@ func TestMetricsSet(t *testing.T) {
require.Len(t, rules.Items, 1)

rule := rules.Items[0]
require.Equal(t, "team-c", rule.Labels["tenant"])
require.Equal(t, "team-c", rule.Labels["obsctl-reloader-generated-tenant"])
require.Equal(t, "789", rule.Labels["obsctl-reloader-generated-uuid"])
require.Len(t, rule.Spec.Groups[0].Rules, 2)
require.Equal(t, `sum by(job) (rate(http_requests_total{tenant="team-c"}[5m]))`, rule.Spec.Groups[0].Rules[0].Expr.String())
require.Equal(t, `http_request_duration_seconds{tenant="team-c"} > 2`, rule.Spec.Groups[0].Rules[1].Expr.String())
require.Equal(t, "789", rule.Spec.Groups[0].Rules[0].Labels["tenant"])
require.Equal(t, `sum by(job) (rate(http_requests_total{tenant="789"}[5m]))`, rule.Spec.Groups[0].Rules[0].Expr.String())
require.Equal(t, `http_request_duration_seconds{tenant="789"} > 2`, rule.Spec.Groups[0].Rules[1].Expr.String())
},
},
}
Expand All @@ -141,6 +146,11 @@ func TestMetricsSet(t *testing.T) {
"test-audience",
"test-issuer",
"team-a,team-b,team-c",
map[string]string{
"team-a": "123",
"team-b": "456",
"team-c": "789",
},
reg,
)

Expand Down
Loading