From 4917f2de3c33a387acc7cb4a7a20e83d941a50a4 Mon Sep 17 00:00:00 2001 From: Pranshu Srivastava Date: Mon, 6 Jan 2025 11:48:51 +0530 Subject: [PATCH 1/4] bugfix: refactor alerts to accomodate for single-node clusters For the sake of brevity, let: Q: kube_node_status_allocatable{job="kube-state-metrics",resource="cpu"} (allocable), and, QQ: namespace_cpu:kube_pod_container_resource_requests:sum{} (requested), thus, both quota alerts relevant here exist in the form: sum(QQ) by (cluster) - (sum(Q) by (cluster) - max(Q) by (cluster)) > 0 and (sum(Q) by (cluster) - max(Q) by (cluster)) > 0, which, in case of a single-node cluster (sum(Q) by (cluster) = max(Q) by (cluster)), is reduced to, sum(QQ) by (cluster) > 0, i.e., the alert will fire if *any* request limits exist. To address this, drop the "max(Q) by (cluster)" buffer assumed in non-SNO clusters from SNO, reducing the expression to: sum(QQ) by (cluster) - sum(Q) by (cluster) > 0 (total requeted - total allocable > 0 to trigger alert), since there is only a single node, so a buffer of the same sort does not make sense. Signed-off-by: Pranshu Srivastava --- alerts/resource_alerts.libsonnet | 63 ++++++++++++++++++++++++-------- 1 file changed, 47 insertions(+), 16 deletions(-) diff --git a/alerts/resource_alerts.libsonnet b/alerts/resource_alerts.libsonnet index 75abb51af..d08c95471 100644 --- a/alerts/resource_alerts.libsonnet +++ b/alerts/resource_alerts.libsonnet @@ -36,18 +36,34 @@ local utils = import '../lib/utils.libsonnet'; } + if $._config.showMultiCluster then { expr: ||| - sum(namespace_cpu:kube_pod_container_resource_requests:sum{%(ignoringOverprovisionedWorkloadSelector)s}) by (%(clusterLabel)s) - (sum(kube_node_status_allocatable{%(kubeStateMetricsSelector)s,resource="cpu"}) by (%(clusterLabel)s) - max(kube_node_status_allocatable{%(kubeStateMetricsSelector)s,resource="cpu"}) by (%(clusterLabel)s)) > 0 + (count(kube_node_info) == 1 and - (sum(kube_node_status_allocatable{%(kubeStateMetricsSelector)s,resource="cpu"}) by (%(clusterLabel)s) - max(kube_node_status_allocatable{%(kubeStateMetricsSelector)s,resource="cpu"}) by (%(clusterLabel)s)) > 0 + sum(namespace_cpu:kube_pod_container_resource_requests:sum{%(ignoringOverprovisionedWorkloadSelector)s}) by (%(clusterLabel)s) - + sum(kube_node_status_allocatable{%(kubeStateMetricsSelector)s,resource="cpu"}) by (%(clusterLabel)s) > 0) + or + (sum(namespace_cpu:kube_pod_container_resource_requests:sum{%(ignoringOverprovisionedWorkloadSelector)s}) by (%(clusterLabel)s) - + (sum(kube_node_status_allocatable{%(kubeStateMetricsSelector)s,resource="cpu"}) by (%(clusterLabel)s) - + max(kube_node_status_allocatable{%(kubeStateMetricsSelector)s,resource="cpu"}) by (%(clusterLabel)s)) > 0 + and + (sum(kube_node_status_allocatable{%(kubeStateMetricsSelector)s,resource="cpu"}) by (%(clusterLabel)s) - + max(kube_node_status_allocatable{%(kubeStateMetricsSelector)s,resource="cpu"}) by (%(clusterLabel)s)) > 0) ||| % $._config, annotations+: { description: 'Cluster {{ $labels.%(clusterLabel)s }} has overcommitted CPU resource requests for Pods by {{ $value }} CPU shares and cannot tolerate node failure.' % $._config, }, } else { expr: ||| - sum(namespace_cpu:kube_pod_container_resource_requests:sum{%(ignoringOverprovisionedWorkloadSelector)s}) - (sum(kube_node_status_allocatable{resource="cpu", %(kubeStateMetricsSelector)s}) - max(kube_node_status_allocatable{resource="cpu", %(kubeStateMetricsSelector)s})) > 0 + (count(kube_node_info) == 1 + and + sum(namespace_cpu:kube_pod_container_resource_requests:sum{%(ignoringOverprovisionedWorkloadSelector)s}) - + sum(kube_node_status_allocatable{resource="cpu", %(kubeStateMetricsSelector)s}) > 0) + or + (sum(namespace_cpu:kube_pod_container_resource_requests:sum{%(ignoringOverprovisionedWorkloadSelector)s}) - + (sum(kube_node_status_allocatable{resource="cpu", %(kubeStateMetricsSelector)s}) - + max(kube_node_status_allocatable{resource="cpu", %(kubeStateMetricsSelector)s})) > 0 and - (sum(kube_node_status_allocatable{resource="cpu", %(kubeStateMetricsSelector)s}) - max(kube_node_status_allocatable{resource="cpu", %(kubeStateMetricsSelector)s})) > 0 + (sum(kube_node_status_allocatable{resource="cpu", %(kubeStateMetricsSelector)s}) - + max(kube_node_status_allocatable{resource="cpu", %(kubeStateMetricsSelector)s})) > 0) ||| % $._config, annotations+: { description: 'Cluster has overcommitted CPU resource requests for Pods by {{ $value }} CPU shares and cannot tolerate node failure.' % $._config, @@ -65,24 +81,39 @@ local utils = import '../lib/utils.libsonnet'; } + if $._config.showMultiCluster then { expr: ||| - sum(namespace_memory:kube_pod_container_resource_requests:sum{%(ignoringOverprovisionedWorkloadSelector)s}) by (%(clusterLabel)s) - (sum(kube_node_status_allocatable{resource="memory", %(kubeStateMetricsSelector)s}) by (%(clusterLabel)s) - max(kube_node_status_allocatable{resource="memory", %(kubeStateMetricsSelector)s}) by (%(clusterLabel)s)) > 0 + (count(kube_node_info) == 1 and - (sum(kube_node_status_allocatable{resource="memory", %(kubeStateMetricsSelector)s}) by (%(clusterLabel)s) - max(kube_node_status_allocatable{resource="memory", %(kubeStateMetricsSelector)s}) by (%(clusterLabel)s)) > 0 + sum(namespace_memory:kube_pod_container_resource_requests:sum{%(ignoringOverprovisionedWorkloadSelector)s}) by (%(clusterLabel)s) - + sum(kube_node_status_allocatable{resource="memory", %(kubeStateMetricsSelector)s}) by (%(clusterLabel)s) > 0) + or + (sum(namespace_memory:kube_pod_container_resource_requests:sum{%(ignoringOverprovisionedWorkloadSelector)s}) by (%(clusterLabel)s) - + (sum(kube_node_status_allocatable{resource="memory", %(kubeStateMetricsSelector)s}) by (%(clusterLabel)s) - + max(kube_node_status_allocatable{resource="memory", %(kubeStateMetricsSelector)s}) by (%(clusterLabel)s)) > 0 + and + (sum(kube_node_status_allocatable{resource="memory", %(kubeStateMetricsSelector)s}) by (%(clusterLabel)s) - + max(kube_node_status_allocatable{resource="memory", %(kubeStateMetricsSelector)s}) by (%(clusterLabel)s)) > 0) ||| % $._config, annotations+: { description: 'Cluster {{ $labels.%(clusterLabel)s }} has overcommitted memory resource requests for Pods by {{ $value | humanize }} bytes and cannot tolerate node failure.' % $._config, }, - } else - { - expr: ||| - sum(namespace_memory:kube_pod_container_resource_requests:sum{%(ignoringOverprovisionedWorkloadSelector)s}) - (sum(kube_node_status_allocatable{resource="memory", %(kubeStateMetricsSelector)s}) - max(kube_node_status_allocatable{resource="memory", %(kubeStateMetricsSelector)s})) > 0 - and - (sum(kube_node_status_allocatable{resource="memory", %(kubeStateMetricsSelector)s}) - max(kube_node_status_allocatable{resource="memory", %(kubeStateMetricsSelector)s})) > 0 - ||| % $._config, - annotations+: { - description: 'Cluster has overcommitted memory resource requests for Pods by {{ $value | humanize }} bytes and cannot tolerate node failure.', - }, + } else { + expr: ||| + (count(kube_node_info) == 1 + and + sum(namespace_memory:kube_pod_container_resource_requests:sum{%(ignoringOverprovisionedWorkloadSelector)s}) - + sum(kube_node_status_allocatable{resource="memory", %(kubeStateMetricsSelector)s}) > 0) + or + (sum(namespace_memory:kube_pod_container_resource_requests:sum{%(ignoringOverprovisionedWorkloadSelector)s}) - + (sum(kube_node_status_allocatable{resource="memory", %(kubeStateMetricsSelector)s}) - + max(kube_node_status_allocatable{resource="memory", %(kubeStateMetricsSelector)s})) > 0 + and + (sum(kube_node_status_allocatable{resource="memory", %(kubeStateMetricsSelector)s}) - + max(kube_node_status_allocatable{resource="memory", %(kubeStateMetricsSelector)s})) > 0) + ||| % $._config, + annotations+: { + description: 'Cluster has overcommitted memory resource requests for Pods by {{ $value | humanize }} bytes and cannot tolerate node failure.', }, + }, { alert: 'KubeCPUQuotaOvercommit', labels: { From 5788ff9166906a8b7c8a6790f4b7ebe4235b8660 Mon Sep 17 00:00:00 2001 From: Pranshu Srivastava Date: Tue, 21 Jan 2025 19:04:47 +0530 Subject: [PATCH 2/4] fixup! bugfix: refactor alerts to accomodate for single-node clusters --- alerts/resource_alerts.libsonnet | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/alerts/resource_alerts.libsonnet b/alerts/resource_alerts.libsonnet index d08c95471..d583e94bc 100644 --- a/alerts/resource_alerts.libsonnet +++ b/alerts/resource_alerts.libsonnet @@ -36,10 +36,10 @@ local utils = import '../lib/utils.libsonnet'; } + if $._config.showMultiCluster then { expr: ||| - (count(kube_node_info) == 1 + (sum(namespace_cpu:kube_pod_container_resource_requests:sum{%(ignoringOverprovisionedWorkloadSelector)s}) by (%(clusterLabel)s) - + 0.85 * sum(kube_node_status_allocatable{%(kubeStateMetricsSelector)s,resource="cpu"}) by (%(clusterLabel)s) > 0 and - sum(namespace_cpu:kube_pod_container_resource_requests:sum{%(ignoringOverprovisionedWorkloadSelector)s}) by (%(clusterLabel)s) - - sum(kube_node_status_allocatable{%(kubeStateMetricsSelector)s,resource="cpu"}) by (%(clusterLabel)s) > 0) + count by (cluster) (max by (cluster, node) (kube_node_info)) == 1) or (sum(namespace_cpu:kube_pod_container_resource_requests:sum{%(ignoringOverprovisionedWorkloadSelector)s}) by (%(clusterLabel)s) - (sum(kube_node_status_allocatable{%(kubeStateMetricsSelector)s,resource="cpu"}) by (%(clusterLabel)s) - @@ -53,10 +53,10 @@ local utils = import '../lib/utils.libsonnet'; }, } else { expr: ||| - (count(kube_node_info) == 1 + (sum(namespace_cpu:kube_pod_container_resource_requests:sum{%(ignoringOverprovisionedWorkloadSelector)s}) - + 0.85 * sum(kube_node_status_allocatable{resource="cpu", %(kubeStateMetricsSelector)s}) > 0 and - sum(namespace_cpu:kube_pod_container_resource_requests:sum{%(ignoringOverprovisionedWorkloadSelector)s}) - - sum(kube_node_status_allocatable{resource="cpu", %(kubeStateMetricsSelector)s}) > 0) + count(max by (node) (kube_node_info)) == 1) or (sum(namespace_cpu:kube_pod_container_resource_requests:sum{%(ignoringOverprovisionedWorkloadSelector)s}) - (sum(kube_node_status_allocatable{resource="cpu", %(kubeStateMetricsSelector)s}) - @@ -81,10 +81,10 @@ local utils = import '../lib/utils.libsonnet'; } + if $._config.showMultiCluster then { expr: ||| - (count(kube_node_info) == 1 + (sum(namespace_memory:kube_pod_container_resource_requests:sum{%(ignoringOverprovisionedWorkloadSelector)s}) by (%(clusterLabel)s) - + 0.85 * sum(kube_node_status_allocatable{resource="memory", %(kubeStateMetricsSelector)s}) by (%(clusterLabel)s) > 0 and - sum(namespace_memory:kube_pod_container_resource_requests:sum{%(ignoringOverprovisionedWorkloadSelector)s}) by (%(clusterLabel)s) - - sum(kube_node_status_allocatable{resource="memory", %(kubeStateMetricsSelector)s}) by (%(clusterLabel)s) > 0) + count by (cluster) (max by (cluster, node) (kube_node_info)) == 1) or (sum(namespace_memory:kube_pod_container_resource_requests:sum{%(ignoringOverprovisionedWorkloadSelector)s}) by (%(clusterLabel)s) - (sum(kube_node_status_allocatable{resource="memory", %(kubeStateMetricsSelector)s}) by (%(clusterLabel)s) - @@ -98,10 +98,10 @@ local utils = import '../lib/utils.libsonnet'; }, } else { expr: ||| - (count(kube_node_info) == 1 + (sum(namespace_memory:kube_pod_container_resource_requests:sum{%(ignoringOverprovisionedWorkloadSelector)s}) - + 0.85 * sum(kube_node_status_allocatable{resource="memory", %(kubeStateMetricsSelector)s}) > 0 and - sum(namespace_memory:kube_pod_container_resource_requests:sum{%(ignoringOverprovisionedWorkloadSelector)s}) - - sum(kube_node_status_allocatable{resource="memory", %(kubeStateMetricsSelector)s}) > 0) + count(max by (node) (kube_node_info)) == 1) or (sum(namespace_memory:kube_pod_container_resource_requests:sum{%(ignoringOverprovisionedWorkloadSelector)s}) - (sum(kube_node_status_allocatable{resource="memory", %(kubeStateMetricsSelector)s}) - From 41645e2ce61de4dcf0ac894ee8529a3b49678f9e Mon Sep 17 00:00:00 2001 From: Stephen Lang Date: Thu, 23 Jan 2025 19:16:20 +0000 Subject: [PATCH 3/4] test: Add tests for KubeCPUOvercommit and KubeMemoryOvercommit --- tests/tests.yaml | 104 +++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 104 insertions(+) diff --git a/tests/tests.yaml b/tests/tests.yaml index 49a81c730..55f79c090 100644 --- a/tests/tests.yaml +++ b/tests/tests.yaml @@ -1323,3 +1323,107 @@ tests: description: 'Cluster has overcommitted memory resource requests for Namespaces.' runbook_url: "https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubememoryquotaovercommit" summary: "Cluster has overcommitted memory resource requests." + +- name: KubeCPUOvercommit alert (single-node) +- interval: 1m + input_series: + - series: 'namespace_cpu:kube_pod_container_resource_requests:sum{cluster="kubernetes", namespace="default"}' + values: '1x10' + - series: 'namespace_cpu:kube_pod_container_resource_requests:sum{cluster="kubernetes", namespace="kube-system"}' + values: '1x10' + - series: 'kube_node_status_allocatable{cluster="kubernetes", node="n1", resource="cpu", job="kube-state-metrics"}' + values: '1.9x10' # This value was seen on a 2x vCPU node + - series: 'kube_node_info{cluster="kubernetes", node="n1", job="kube-state-metrics"}' + values: '1x10' + alert_rule_test: + - eval_time: 9m + alertname: KubeCPUOvercommit + - eval_time: 10m + alertname: KubeCPUOvercommit + exp_alerts: + - exp_labels: + severity: warning + exp_annotations: + description: Cluster has overcommitted CPU resource requests for Pods by 0.385 CPU shares and cannot tolerate node failure. + runbook_url: https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubecpuovercommit + summary: Cluster has overcommitted CPU resource requests. + +- name: KubeCPUOvercommit alert (multi-node) +- interval: 1m + input_series: + - series: 'namespace_cpu:kube_pod_container_resource_requests:sum{cluster="kubernetes", namespace="default"}' + values: '2x10' + - series: 'namespace_cpu:kube_pod_container_resource_requests:sum{cluster="kubernetes", namespace="kube-system"}' + values: '2x10' + - series: 'kube_node_status_allocatable{cluster="kubernetes", node="n1", resource="cpu", job="kube-state-metrics"}' + values: '1.9x10' # This value was seen on a 2x vCPU node + - series: 'kube_node_status_allocatable{cluster="kubernetes", node="n2", resource="cpu", job="kube-state-metrics"}' + values: '1.9x10' + - series: 'kube_node_info{cluster="kubernetes", node="n1", job="kube-state-metrics"}' + values: '1x10' + - series: 'kube_node_info{cluster="kubernetes", node="n2", job="kube-state-metrics"}' + values: '1x10' + alert_rule_test: + - eval_time: 9m + alertname: KubeCPUOvercommit + - eval_time: 10m + alertname: KubeCPUOvercommit + exp_alerts: + - exp_labels: + severity: warning + exp_annotations: + description: Cluster has overcommitted CPU resource requests for Pods by 2.1 CPU shares and cannot tolerate node failure. + runbook_url: https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubecpuovercommit + summary: Cluster has overcommitted CPU resource requests. + +- name: KubeMemoryOvercommit alert (single-node) +- interval: 1m + input_series: + - series: 'namespace_memory:kube_pod_container_resource_requests:sum{cluster="kubernetes", namespace="default"}' + values: '1000000000x10' # 1 GB + - series: 'namespace_memory:kube_pod_container_resource_requests:sum{cluster="kubernetes", namespace="kube-system"}' + values: '1000000000x10' + - series: 'kube_node_status_allocatable{cluster="kubernetes", node="n1", resource="memory", job="kube-state-metrics"}' + values: '1000000000x10' + - series: 'kube_node_info{cluster="kubernetes", node="n1", job="kube-state-metrics"}' + values: '1x10' + alert_rule_test: + - eval_time: 9m + alertname: KubeMemoryOvercommit + - eval_time: 10m + alertname: KubeMemoryOvercommit + exp_alerts: + - exp_labels: + severity: warning + exp_annotations: + description: Cluster has overcommitted memory resource requests for Pods by 1.15G bytes and cannot tolerate node failure. + runbook_url: https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubememoryovercommit + summary: Cluster has overcommitted memory resource requests. + +- name: KubeMemoryOvercommit alert (multi-node) +- interval: 1m + input_series: + - series: 'namespace_memory:kube_pod_container_resource_requests:sum{cluster="kubernetes", namespace="default"}' + values: '2000000000x10' # 2 GB + - series: 'namespace_memory:kube_pod_container_resource_requests:sum{cluster="kubernetes", namespace="kube-system"}' + values: '2000000000x10' + - series: 'kube_node_status_allocatable{cluster="kubernetes", node="n1", resource="memory", job="kube-state-metrics"}' + values: '1000000000x10' + - series: 'kube_node_status_allocatable{cluster="kubernetes", node="n2", resource="memory", job="kube-state-metrics"}' + values: '1000000000x10' + - series: 'kube_node_info{cluster="kubernetes", node="n1", job="kube-state-metrics"}' + values: '1x10' + - series: 'kube_node_info{cluster="kubernetes", node="n2", job="kube-state-metrics"}' + values: '1x10' + alert_rule_test: + - eval_time: 9m + alertname: KubeMemoryOvercommit + - eval_time: 10m + alertname: KubeMemoryOvercommit + exp_alerts: + - exp_labels: + severity: warning + exp_annotations: + description: Cluster has overcommitted memory resource requests for Pods by 3G bytes and cannot tolerate node failure. + runbook_url: https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubememoryovercommit + summary: Cluster has overcommitted memory resource requests. From fc9fe6a6cf20630824b663cc3ede2125194c2ba6 Mon Sep 17 00:00:00 2001 From: Pranshu Srivastava Date: Mon, 10 Feb 2025 19:16:33 +0530 Subject: [PATCH 4/4] `s/kube_node_info/kube_node_role` Use kube_node_role{role="control-plane"} (see [1] and [2]) to estimate if the cluster is HA or not. * [1]: https://github.com/search?q=repo%3Akubernetes%2Fkube-state-metrics%20kube_node_role&type=code * [2]: https://kubernetes.io/docs/reference/labels-annotations-taints/#node-role-kubernetes-io-control-plane Also drop any thresholds as they would lead to false positives. --- alerts/resource_alerts.libsonnet | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/alerts/resource_alerts.libsonnet b/alerts/resource_alerts.libsonnet index d583e94bc..58e5b4e8c 100644 --- a/alerts/resource_alerts.libsonnet +++ b/alerts/resource_alerts.libsonnet @@ -37,9 +37,9 @@ local utils = import '../lib/utils.libsonnet'; if $._config.showMultiCluster then { expr: ||| (sum(namespace_cpu:kube_pod_container_resource_requests:sum{%(ignoringOverprovisionedWorkloadSelector)s}) by (%(clusterLabel)s) - - 0.85 * sum(kube_node_status_allocatable{%(kubeStateMetricsSelector)s,resource="cpu"}) by (%(clusterLabel)s) > 0 + sum(kube_node_status_allocatable{%(kubeStateMetricsSelector)s,resource="cpu"}) by (%(clusterLabel)s) > 0 and - count by (cluster) (max by (cluster, node) (kube_node_info)) == 1) + count by (cluster) (max by (cluster, node) (kube_node_role{role="control-plane"})) < 3) or (sum(namespace_cpu:kube_pod_container_resource_requests:sum{%(ignoringOverprovisionedWorkloadSelector)s}) by (%(clusterLabel)s) - (sum(kube_node_status_allocatable{%(kubeStateMetricsSelector)s,resource="cpu"}) by (%(clusterLabel)s) - @@ -54,9 +54,9 @@ local utils = import '../lib/utils.libsonnet'; } else { expr: ||| (sum(namespace_cpu:kube_pod_container_resource_requests:sum{%(ignoringOverprovisionedWorkloadSelector)s}) - - 0.85 * sum(kube_node_status_allocatable{resource="cpu", %(kubeStateMetricsSelector)s}) > 0 + sum(kube_node_status_allocatable{resource="cpu", %(kubeStateMetricsSelector)s}) > 0 and - count(max by (node) (kube_node_info)) == 1) + count(max by (node) (kube_node_role{role="control-plane"})) < 3) or (sum(namespace_cpu:kube_pod_container_resource_requests:sum{%(ignoringOverprovisionedWorkloadSelector)s}) - (sum(kube_node_status_allocatable{resource="cpu", %(kubeStateMetricsSelector)s}) - @@ -82,9 +82,9 @@ local utils = import '../lib/utils.libsonnet'; if $._config.showMultiCluster then { expr: ||| (sum(namespace_memory:kube_pod_container_resource_requests:sum{%(ignoringOverprovisionedWorkloadSelector)s}) by (%(clusterLabel)s) - - 0.85 * sum(kube_node_status_allocatable{resource="memory", %(kubeStateMetricsSelector)s}) by (%(clusterLabel)s) > 0 + sum(kube_node_status_allocatable{resource="memory", %(kubeStateMetricsSelector)s}) by (%(clusterLabel)s) > 0 and - count by (cluster) (max by (cluster, node) (kube_node_info)) == 1) + count by (cluster) (max by (cluster, node) (kube_node_role{role="control-plane"})) < 3) or (sum(namespace_memory:kube_pod_container_resource_requests:sum{%(ignoringOverprovisionedWorkloadSelector)s}) by (%(clusterLabel)s) - (sum(kube_node_status_allocatable{resource="memory", %(kubeStateMetricsSelector)s}) by (%(clusterLabel)s) - @@ -99,9 +99,9 @@ local utils = import '../lib/utils.libsonnet'; } else { expr: ||| (sum(namespace_memory:kube_pod_container_resource_requests:sum{%(ignoringOverprovisionedWorkloadSelector)s}) - - 0.85 * sum(kube_node_status_allocatable{resource="memory", %(kubeStateMetricsSelector)s}) > 0 + sum(kube_node_status_allocatable{resource="memory", %(kubeStateMetricsSelector)s}) > 0 and - count(max by (node) (kube_node_info)) == 1) + count(max by (node) (kube_node_role{role="control-plane"})) < 3) or (sum(namespace_memory:kube_pod_container_resource_requests:sum{%(ignoringOverprovisionedWorkloadSelector)s}) - (sum(kube_node_status_allocatable{resource="memory", %(kubeStateMetricsSelector)s}) -