| { "annotations": { "list": [ { "builtIn": 1, "datasource": { "type": "grafana", "uid": "-- Grafana --" }, "enable": true, "hide": true, "iconColor": "rgba(0, 211, 255, 1)", "name": "Annotations & Alerts", "type": "dashboard" } ] }, "editable": true, "fiscalYearStartMonth": 0, "graphTooltip": 0, "id": 1037, "links": [], "panels": [ { "collapsed": false, "gridPos": { "h": 1, "w": 24, "x": 0, "y": 0 }, "id": 10, "panels": [], "title": "k8s集群资源容量预估", "type": "row" }, { "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, "fieldConfig": { "defaults": { "color": { "mode": "thresholds" }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null }, { "color": "red", "value": 0.8 }, { "color": "dark-red", "value": 0.9 } ] }, "unit": "percentunit" }, "overrides": [] }, "gridPos": { "h": 6, "w": 8, "x": 0, "y": 1 }, "id": 18, "options": { "colorMode": "value", "graphMode": "area", "justifyMode": "auto", "orientation": "auto", "reduceOptions": { "calcs": [ "lastNotNull" ], "fields": "", "values": false }, "showPercentChange": false, "textMode": "auto", "wideLayout": true }, "pluginVersion": "10.4.1", "targets": [ { "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, "editorMode": "code", "expr": "sum by(cluster,resource)(kube_pod_container_resource_requests{job=\"kube-state-metrics\",cluster=\"$cluster\",resource=~\".*gpu.*|cpu|memory\"})\r\n/sum by(cluster,resource)(kube_node_status_capacity{job=\"kube-state-metrics\",cluster=\"$cluster\",resource=~\".*gpu.*|cpu|memory\"})>0", "instant": false, "legendFormat": "{{resource}}", "range": true, "refId": "A" } ], "title": "$cluster集群资源requests使用率", "type": "stat" }, { "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, "fieldConfig": { "defaults": { "color": { "mode": "thresholds" }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null }, { "color": "red", "value": 0.9 }, { "color": "dark-red", "value": 1 } ] }, "unit": "percentunit" }, "overrides": [] }, "gridPos": { "h": 6, "w": 8, "x": 8, "y": 1 }, "id": 19, "options": { "colorMode": "value", "graphMode": "area", "justifyMode": "auto", "orientation": "auto", "reduceOptions": { "calcs": [ "lastNotNull" ], "fields": "", "values": false }, "showPercentChange": false, "textMode": "auto", "wideLayout": true }, "pluginVersion": "10.4.1", "targets": [ { "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, "editorMode": "code", "expr": "sum by(cluster,resource)(kube_pod_container_resource_limits{job=\"kube-state-metrics\",cluster=\"$cluster\",resource=~\".*gpu.*|cpu|memory\"})\r\n/sum by(cluster,resource)(kube_node_status_capacity{job=\"kube-state-metrics\",cluster=\"$cluster\",resource=~\".*gpu.*|cpu|memory\"})>0", "instant": false, "legendFormat": "{{resource}}", "range": true, "refId": "A" } ], "title": "$cluster集群资源limits使用率", "type": "stat" }, { "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, "fieldConfig": { "defaults": { "color": { "mode": "thresholds" }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null }, { "color": "red", "value": 0.9 }, { "color": "dark-red", "value": 1 } ] }, "unit": "percentunit" }, "overrides": [] }, "gridPos": { "h": 3, "w": 5, "x": 16, "y": 1 }, "id": 12, "options": { "colorMode": "value", "graphMode": "area", "justifyMode": "auto", "orientation": "auto", "reduceOptions": { "calcs": [ "lastNotNull" ], "fields": "", "values": false }, "showPercentChange": false, "textMode": "auto", "wideLayout": true }, "pluginVersion": "10.4.1", "targets": [ { "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, "editorMode": "code", "expr": "sum(\r\n max_over_time(node_namespace_pod_container:container_cpu_usage_seconds_total:sum_irate{cluster=\"$cluster\",container!=\"POD\"}[$__range])\r\n * on(namespace,pod)\r\n group_left(workload, workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\"}\r\n) by (cluster)\r\n/(sum by(cluster)(kube_node_status_capacity{job=\"kube-state-metrics\",cluster=\"$cluster\",resource=\"memory\"})/1024/1024/1024)", "instant": false, "legendFormat": "__auto", "range": true, "refId": "A" } ], "title": "$cluster集群cpu使用率", "type": "stat" }, { "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, "fieldConfig": { "defaults": { "color": { "mode": "thresholds" }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null }, { "color": "red", "value": 80 } ] }, "unit": "percentunit" }, "overrides": [] }, "gridPos": { "h": 3, "w": 5, "x": 16, "y": 4 }, "id": 16, "options": { "colorMode": "value", "graphMode": "area", "justifyMode": "auto", "orientation": "auto", "reduceOptions": { "calcs": [ "lastNotNull" ], "fields": "", "values": false }, "showPercentChange": false, "textMode": "auto", "wideLayout": true }, "pluginVersion": "10.4.1", "targets": [ { "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, "editorMode": "code", "expr": "sum(\r\n max_over_time(container_memory_working_set_bytes{cluster=\"$cluster\", container!=\"\", image!=\"\"}[$__range])\r\n * on(namespace,pod)\r\n group_left(workload, workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\"}\r\n) by (cluster)/1024/1024/1024\r\n/(sum by(cluster)(kube_node_status_capacity{job=\"kube-state-metrics\",cluster=\"$cluster\",resource=\"memory\"})/1024/1024/1024)", "instant": false, "legendFormat": "__auto", "range": true, "refId": "A" } ], "title": "$cluster集群内存使用率", "type": "stat" }, { "datasource": { "type": "datasource", "uid": "-- Mixed --" }, "fieldConfig": { "defaults": { "color": { "mode": "thresholds" }, "custom": { "align": "left", "cellOptions": { "type": "auto" }, "filterable": true, "inspect": false }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null }, { "color": "red", "value": 80 } ] }, "unit": "none" }, "overrides": [ { "matcher": { "id": "byName", "options": "node" }, "properties": [ { "id": "custom.width", "value": 250 } ] }, { "matcher": { "id": "byName", "options": "gpu_mem_request可分配数" }, "properties": [ { "id": "custom.width", "value": 178 } ] }, { "matcher": { "id": "byName", "options": "cpu实际可分配数" }, "properties": [ { "id": "custom.width", "value": 123 } ] }, { "matcher": { "id": "byName", "options": "mem实际可分配数" }, "properties": [ { "id": "custom.width", "value": 131 } ] }, { "matcher": { "id": "byName", "options": "gpu_type" }, "properties": [ { "id": "custom.width", "value": 81 } ] }, { "matcher": { "id": "byName", "options": "cpu_limit可分配数" }, "properties": [ { "id": "custom.width", "value": 132 } ] }, { "matcher": { "id": "byName", "options": "mem_limit可分配数" }, "properties": [ { "id": "custom.width", "value": 138 } ] }, { "matcher": { "id": "byName", "options": "gpu_limt可分配数" }, "properties": [ { "id": "custom.width", "value": 134 } ] }, { "matcher": { "id": "byName", "options": "gpu_mem_type" }, "properties": [ { "id": "custom.width", "value": 127 } ] }, { "matcher": { "id": "byName", "options": "可预分配数" }, "properties": [ { "id": "custom.width", "value": 94 } ] }, { "matcher": { "id": "byName", "options": "可超限分配数" }, "properties": [ { "id": "custom.width", "value": 99 } ] } ] }, "gridPos": { "h": 13, "w": 24, "x": 0, "y": 7 }, "id": 8, "options": { "cellHeight": "sm", "footer": { "countRows": false, "fields": "", "reducer": [ "sum" ], "show": true }, "showHeader": true, "sortBy": [] }, "pluginVersion": "10.4.1", "targets": [ { "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, "editorMode": "code", "expr": "floor((sum by(node)(kube_node_status_capacity{job=\"kube-state-metrics\",cluster=\"$cluster\",resource=\"cpu\"})\r\n-\r\nsum(\r\n max_over_time(node_namespace_pod_container:container_cpu_usage_seconds_total:sum_irate{cluster=\"$cluster\",container!=\"POD\"}[$__range])\r\n * on(namespace,pod)\r\n group_left(workload, workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\"}\r\n) by (node)\r\n)\r\n/$cpu_core/$workload_count)\r\n", "format": "table", "hide": false, "legendFormat": "{{node}}节点cpu实际可分配数", "range": true, "refId": "B" }, { "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, "editorMode": "code", "expr": "floor(((sum by(node)(kube_node_status_capacity{job=\"kube-state-metrics\",cluster=\"$cluster\",resource=\"memory\"})/1024/1024/1024)\r\n-\r\nsum(\r\n max_over_time(container_memory_working_set_bytes{cluster=\"$cluster\", container!=\"\", image!=\"\"}[$__range])\r\n * on(namespace,pod)\r\n group_left(workload, workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\"}\r\n) by (node)/1024/1024/1024)\r\n/$mem/$workload_count)\r\n\r\n", "format": "table", "hide": false, "legendFormat": "mem实际可分配数:{{node}}", "range": true, "refId": "C" }, { "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, "editorMode": "code", "expr": "floor((\r\n (sum by(node,resource)(kube_node_status_capacity{job=\"kube-state-metrics\",cluster=\"$cluster\",resource=~\".*gpu.*\",resource!~\".*mem.*\"})>0)\r\n-\r\nsum by(node,resource)(kube_pod_container_resource_limits{job=\"kube-state-metrics\",cluster=\"$cluster\",resource=~\".*gpu.*\",resource!~\".*mem.*\"})\r\n)\r\n/$gpu_count/$workload_count)", "format": "table", "hide": false, "legendFormat": "gpu_limits", "range": true, "refId": "F" }, { "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, "editorMode": "code", "expr": "floor((\r\n (sum by(node,resource)(kube_node_status_capacity{job=\"kube-state-metrics\",cluster=\"$cluster\",resource=~\".*gpu.*\",resource!~\".*mem.*\"})>0)\r\n-\r\nsum by(node,resource)(kube_pod_container_resource_requests{job=\"kube-state-metrics\",cluster=\"$cluster\",resource=~\".*gpu.*\",resource!~\".*mem.*\"})\r\n)\r\n/$gpu_count/$workload_count)", "format": "table", "hide": false, "legendFormat": "gpu_requests-{{resource}}", "range": true, "refId": "G" }, { "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, "editorMode": "code", "expr": "floor((\r\n (sum by(node,resource)(kube_node_status_capacity{job=\"kube-state-metrics\",cluster=\"$cluster\",resource=~\".*gpu.*\",resource=~\".*mem.*\"})>0)\r\n-\r\nsum by(node,resource)(kube_pod_container_resource_requests{job=\"kube-state-metrics\",cluster=\"$cluster\",resource=~\".*gpu.*\",resource=~\".*mem.*\"})\r\n)\r\n/$gpu_mem/$workload_count)", "format": "table", "hide": false, "legendFormat": "gpu_mem_request:{{cluster}}", "range": true, "refId": "I" }, { "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, "editorMode": "code", "expr": "floor((\r\n (sum by(node,resource)(kube_node_status_capacity{job=\"kube-state-metrics\",cluster=\"$cluster\",resource=~\".*gpu.*\",resource=~\".*mem.*\"})>0)\r\n-\r\nsum by(node,resource)(kube_pod_container_resource_limits{job=\"kube-state-metrics\",cluster=\"$cluster\",resource=~\".*gpu.*\",resource=~\".*mem.*\"})\r\n)\r\n/$gpu_mem/$workload_count/$workload_count)", "format": "table", "hide": false, "legendFormat": "gpu_mem_limits:{{cluster}}", "range": true, "refId": "L" }, { "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, "editorMode": "code", "expr": "floor((\r\n (sum by(node)(kube_node_status_capacity{job=\"kube-state-metrics\",cluster=\"$cluster\",resource=~\"cpu\"})>0)\r\n-\r\nsum by(node)(kube_pod_container_resource_requests{job=\"kube-state-metrics\",cluster=\"$cluster\",resource=~\"cpu\"})\r\n)\r\n/$cpu_core/$workload_count)", "format": "table", "hide": false, "legendFormat": "cpu_request", "range": true, "refId": "A" }, { "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, "editorMode": "code", "expr": "floor((\r\n (sum by(node)(kube_node_status_capacity{job=\"kube-state-metrics\",cluster=\"$cluster\",resource=~\"cpu\"})>0)\r\n-\r\nsum by(node)(kube_pod_container_resource_limits{job=\"kube-state-metrics\",cluster=\"$cluster\",resource=~\"cpu\"})\r\n)\r\n/$cpu_core/$workload_count)", "format": "table", "hide": false, "legendFormat": "cpu_limit", "range": true, "refId": "D" }, { "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, "editorMode": "code", "expr": "floor((\r\n (sum by(node)(kube_node_status_capacity{job=\"kube-state-metrics\",cluster=\"$cluster\",resource=~\"memory\"})>0)\r\n-\r\nsum by(node)(kube_pod_container_resource_limits{job=\"kube-state-metrics\",cluster=\"$cluster\",resource=~\"memory\"})\r\n)\r\n/$mem/1024/1024/1024/$workload_count)", "format": "table", "hide": false, "legendFormat": "mem_limit", "range": true, "refId": "E" }, { "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, "editorMode": "code", "expr": "floor((\r\n (sum by(node)(kube_node_status_capacity{job=\"kube-state-metrics\",cluster=\"$cluster\",resource=~\"memory\"})>0)\r\n-\r\nsum by(node)(kube_pod_container_resource_requests{job=\"kube-state-metrics\",cluster=\"$cluster\",resource=~\"memory\"})\r\n)\r\n/$mem/1024/1024/1024/$workload_count)", "format": "table", "hide": false, "legendFormat": "mem_request", "range": true, "refId": "H" } ], "title": "$cluster集群资源分配预估", "transformations": [ { "id": "joinByField", "options": { "byField": "node", "mode": "outer" } }, { "id": "organize", "options": { "excludeByName": { "Time 1": true, "Time 10": true, "Time 2": true, "Time 3": true, "Time 4": true, "Time 5": true, "Time 6": true, "Time 7": true, "Time 8": true, "Time 9": true, "resource 2": true, "resource 4": true }, "includeByName": {}, "indexByName": { "Time 1": 1, "Time 10": 24, "Time 2": 5, "Time 3": 9, "Time 4": 12, "Time 5": 15, "Time 6": 18, "Time 7": 21, "Time 8": 22, "Time 9": 23, "Value #A": 3, "Value #B": 2, "Value #C": 6, "Value #D": 4, "Value #E": 8, "Value #F": 11, "Value #G": 14, "Value #H": 7, "Value #I": 17, "Value #L": 20, "node": 0, "resource 1": 10, "resource 2": 13, "resource 3": 16, "resource 4": 19 }, "renameByName": { "Value #A": "cpu_request可分配数", "Value #B": "cpu实际可分配数", "Value #C": "mem实际可分配数", "Value #D": "cpu_limit可分配数", "Value #E": "mem_limit可分配数", "Value #F": "gpu_limt可分配数", "Value #G": "gpu_request可分配数", "Value #H": "mem_request可分配数", "Value #I": "gpu_mem_request可分配数", "Value #L": "gpu_mem_limit可分配数", "resource": "", "resource 1": "gpu_type", "resource 3": "gpu_mem_type" } } }, { "id": "calculateField", "options": { "alias": "可预分配数", "mode": "reduceRow", "reduce": { "include": [ "gpu_request可分配数", "gpu_mem_request可分配数", "cpu_request可分配数", "mem_request可分配数" ], "reducer": "min" } } }, { "id": "calculateField", "options": { "alias": "可超限分配数", "mode": "reduceRow", "reduce": { "include": [ "gpu_limt可分配数", "gpu_mem_limit可分配数", "cpu_limit可分配数", "mem_limit可分配数" ], "reducer": "min" } } }, { "id": "calculateField", "options": { "alias": "可调整分配数", "mode": "reduceRow", "reduce": { "include": [ "cpu实际可分配数", "mem实际可分配数", "gpu_request可分配数", "gpu_mem_request可分配数" ], "reducer": "min" } } } ], "type": "table" }, { "datasource": { "type": "datasource", "uid": "-- Mixed --" }, "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "auto", "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null }, { "color": "red", "value": 80 } ] }, "unit": "percentunit" }, "overrides": [] }, "gridPos": { "h": 15, "w": 24, "x": 0, "y": 20 }, "id": 9, "options": { "legend": { "calcs": [ "last", "max" ], "displayMode": "table", "placement": "bottom", "showLegend": true, "sortBy": "Last", "sortDesc": false }, "tooltip": { "mode": "single", "sort": "none" } }, "targets": [ { "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, "editorMode": "code", "expr": "sum(\r\n max_over_time(node_namespace_pod_container:container_cpu_usage_seconds_total:sum_irate{cluster=\"$cluster\",container!=\"POD\"}[$__range])\r\n * on(namespace,pod)\r\n group_left(workload, workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\"}\r\n) by (cluster)\r\n/(sum by(cluster)(kube_node_status_capacity{job=\"kube-state-metrics\",cluster=\"$cluster\",resource=\"memory\"})/1024/1024/1024)", "hide": false, "legendFormat": "cpu_used:{{cluster}}", "range": true, "refId": "B" }, { "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, "editorMode": "code", "expr": "sum(\r\n max_over_time(container_memory_working_set_bytes{cluster=\"$cluster\", container!=\"\", image!=\"\"}[$__range])\r\n * on(namespace,pod)\r\n group_left(workload, workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\"}\r\n) by (cluster)/1024/1024/1024\r\n/(sum by(cluster)(kube_node_status_capacity{job=\"kube-state-metrics\",cluster=\"$cluster\",resource=\"memory\"})/1024/1024/1024)", "hide": false, "legendFormat": "mem_used:{{cluster}}", "range": true, "refId": "C" }, { "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, "editorMode": "code", "expr": "", "hide": false, "legendFormat": "__auto", "range": true, "refId": "A" }, { "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, "editorMode": "code", "expr": "sum by(cluster,resource)(kube_pod_container_resource_requests{job=\"kube-state-metrics\",cluster=\"$cluster\",resource=~\".*gpu.*\"})\r\n/sum by(cluster,resource)(kube_node_status_capacity{job=\"kube-state-metrics\",cluster=\"$cluster\",resource=~\".*gpu.*\"})>0", "hide": false, "legendFormat": "gpu_requests-{{resource}}:{{cluster}}", "range": true, "refId": "G" }, { "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, "editorMode": "code", "expr": "sum by(cluster,resource)(kube_pod_container_resource_limits{job=\"kube-state-metrics\",cluster=\"$cluster\",resource=~\".*gpu.*\"})\r\n/sum by(cluster,resource)(kube_node_status_capacity{job=\"kube-state-metrics\",cluster=\"$cluster\",resource=~\".*gpu.*\"})>0", "hide": false, "legendFormat": "gpu_limits-{{resource}}:{{cluster}}", "range": true, "refId": "H" }, { "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, "editorMode": "code", "expr": "sum by(cluster)(kube_pod_container_resource_limits{job=\"kube-state-metrics\",cluster=\"$cluster\",resource=~\"cpu\"})/sum by(cluster)(kube_node_status_capacity{job=\"kube-state-metrics\",cluster=\"$cluster\",resource=\"cpu\"})", "hide": false, "legendFormat": "cpu_limits:{{cluster}}", "range": true, "refId": "I" }, { "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, "editorMode": "code", "expr": "sum by(cluster)(kube_pod_container_resource_requests{job=\"kube-state-metrics\",cluster=\"$cluster\",resource=~\"cpu\"})/sum by(cluster)(kube_node_status_capacity{job=\"kube-state-metrics\",cluster=\"$cluster\",resource=\"cpu\"})", "hide": false, "legendFormat": "cpu_requests:{{cluster}}", "range": true, "refId": "J" }, { "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, "editorMode": "code", "expr": "sum by(cluster)(kube_pod_container_resource_requests{job=\"kube-state-metrics\",cluster=\"$cluster\",resource=~\"memory\"})/1024/1024/1024\r\n/(sum by(cluster)(kube_node_status_capacity{job=\"kube-state-metrics\",cluster=\"$cluster\",resource=\"memory\"})/1024/1024/1024)", "hide": false, "legendFormat": "mem_requests:{{cluster}}", "range": true, "refId": "K" }, { "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, "editorMode": "code", "expr": "sum by(cluster)(kube_pod_container_resource_limits{job=\"kube-state-metrics\",cluster=\"$cluster\",resource=~\"memory\"})/1024/1024/1024\r\n/(sum by(cluster)(kube_node_status_capacity{job=\"kube-state-metrics\",cluster=\"$cluster\",resource=\"memory\"})/1024/1024/1024)", "hide": false, "legendFormat": "mem_limits:{{cluster}}", "range": true, "refId": "L" } ], "title": "$cluster集群资源使用率", "type": "timeseries" }, { "datasource": { "type": "datasource", "uid": "-- Mixed --" }, "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "auto", "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null }, { "color": "red", "value": 80 } ] } }, "overrides": [] }, "gridPos": { "h": 15, "w": 24, "x": 0, "y": 35 }, "id": 11, "options": { "legend": { "calcs": [ "last", "max" ], "displayMode": "table", "placement": "bottom", "showLegend": true, "sortBy": "Name", "sortDesc": true }, "tooltip": { "mode": "single", "sort": "none" } }, "targets": [ { "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, "editorMode": "code", "expr": "sum(\r\n max_over_time(node_namespace_pod_container:container_cpu_usage_seconds_total:sum_irate{cluster=\"$cluster\",container!=\"POD\"}[$__range])\r\n * on(namespace,pod)\r\n group_left(workload, workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\"}\r\n) by (cluster)", "hide": false, "legendFormat": "cpu_used:{{cluster}}", "range": true, "refId": "B" }, { "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, "editorMode": "code", "expr": "sum(\r\n max_over_time(container_memory_working_set_bytes{cluster=\"$cluster\", container!=\"\", image!=\"\"}[$__range])\r\n * on(namespace,pod)\r\n group_left(workload, workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\"}\r\n) by (cluster)/1024/1024/1024", "hide": false, "legendFormat": "mem_used(G):{{cluster}}", "range": true, "refId": "C" }, { "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, "editorMode": "code", "expr": "sum by(cluster)(kube_node_status_capacity{job=\"kube-state-metrics\",cluster=\"$cluster\",resource=\"cpu\"})", "hide": false, "legendFormat": "cpu_total:{{cluster}}", "range": true, "refId": "A" }, { "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, "editorMode": "code", "expr": "sum by(cluster)(kube_node_status_capacity{job=\"kube-state-metrics\",cluster=\"$cluster\",resource=\"memory\"})/1024/1024/1024", "hide": false, "legendFormat": "mem_total(G):{{cluster}}", "range": true, "refId": "D" }, { "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, "editorMode": "code", "expr": "sum by(cluster,resource)(kube_node_status_capacity{job=\"kube-state-metrics\",cluster=\"$cluster\",resource=~\".*gpu.*\",resource!~\".*mem.*\"})>0", "hide": false, "legendFormat": "gpu_total-{{resource}}:{{cluster}}", "range": true, "refId": "E" }, { "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, "editorMode": "code", "expr": "sum by(cluster,resource)(kube_node_status_capacity{job=\"kube-state-metrics\",cluster=\"$cluster\",resource=~\".*gpu.*\",resource=~\".*mem.*\"})>0", "hide": false, "legendFormat": "gpu_mem_total(M)-{{resource}}:{{cluster}}", "range": true, "refId": "F" }, { "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, "editorMode": "code", "expr": "sum by(cluster,resource)(kube_pod_container_resource_requests{job=\"kube-state-metrics\",cluster=\"$cluster\",resource=~\".*gpu.*\"})", "hide": false, "legendFormat": "gpu_requests-{{resource}}:{{cluster}}", "range": true, "refId": "G" }, { "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, "editorMode": "code", "expr": "sum by(cluster,resource)(kube_pod_container_resource_limits{job=\"kube-state-metrics\",cluster=\"$cluster\",resource=~\".*gpu.*\"})", "hide": false, "legendFormat": "gpu_limits-{{resource}}:{{cluster}}", "range": true, "refId": "H" }, { "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, "editorMode": "code", "expr": "sum by(cluster)(kube_pod_container_resource_limits{job=\"kube-state-metrics\",cluster=\"$cluster\",resource=~\"cpu\"})", "hide": false, "legendFormat": "cpu_limits:{{cluster}}", "range": true, "refId": "I" }, { "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, "editorMode": "code", "expr": "sum by(cluster)(kube_pod_container_resource_requests{job=\"kube-state-metrics\",cluster=\"$cluster\",resource=~\"cpu\"})", "hide": false, "legendFormat": "cpu_requests:{{cluster}}", "range": true, "refId": "J" }, { "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, "editorMode": "code", "expr": "sum by(cluster)(kube_pod_container_resource_requests{job=\"kube-state-metrics\",cluster=\"$cluster\",resource=~\"memory\"})/1024/1024/1024", "hide": false, "legendFormat": "mem_requests:{{cluster}}", "range": true, "refId": "K" }, { "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, "editorMode": "code", "expr": "sum by(cluster)(kube_pod_container_resource_limits{job=\"kube-state-metrics\",cluster=\"$cluster\",resource=~\"memory\"})/1024/1024/1024", "hide": false, "legendFormat": "mem_limits:{{cluster}}", "range": true, "refId": "L" } ], "title": "$cluster集群资源使用量", "type": "timeseries" }, { "collapsed": true, "gridPos": { "h": 1, "w": 24, "x": 0, "y": 50 }, "id": 7, "panels": [ { "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, "fieldConfig": { "defaults": { "color": { "mode": "thresholds" }, "custom": { "align": "auto", "cellOptions": { "type": "auto" }, "filterable": true, "inspect": false }, "mappings": [], "max": -2, "thresholds": { "mode": "absolute", "steps": [ { "color": "green" }, { "color": "red", "value": 80 } ] }, "unit": "decbytes" }, "overrides": [ { "matcher": { "id": "byName", "options": "CPU" }, "properties": [ { "id": "unit" }, { "id": "custom.width", "value": 72 } ] }, { "matcher": { "id": "byName", "options": "MEMORY" }, "properties": [ { "id": "custom.width", "value": 100 } ] }, { "matcher": { "id": "byName", "options": "Role" }, "properties": [ { "id": "custom.width", "value": 100 } ] }, { "matcher": { "id": "byName", "options": "Cluster" }, "properties": [ { "id": "custom.width", "value": 160 } ] }, { "matcher": { "id": "byName", "options": "Node" }, "properties": [ { "id": "custom.width", "value": 260 } ] }, { "matcher": { "id": "byName", "options": "IP" }, "properties": [ { "id": "custom.width", "value": 150 } ] } ] }, "gridPos": { "h": 15, "w": 24, "x": 0, "y": 17 }, "id": 1, "options": { "cellHeight": "sm", "footer": { "countRows": false, "fields": "", "reducer": [ "sum" ], "show": true }, "showHeader": true, "sortBy": [] }, "pluginVersion": "10.4.1", "targets": [ { "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, "editorMode": "code", "exemplar": false, "expr": "kube_node_info", "format": "table", "instant": true, "legendFormat": "__auto", "range": false, "refId": "A" }, { "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, "editorMode": "code", "exemplar": false, "expr": "kube_node_role", "format": "table", "hide": false, "instant": true, "legendFormat": "__auto", "range": false, "refId": "B" }, { "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, "editorMode": "code", "exemplar": false, "expr": "kube_node_status_capacity{resource=\"cpu\"}", "format": "table", "hide": false, "instant": true, "legendFormat": "__auto", "range": false, "refId": "C" }, { "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, "editorMode": "code", "exemplar": false, "expr": "kube_node_status_capacity{resource=\"memory\"}", "format": "table", "hide": false, "instant": true, "legendFormat": "__auto", "range": false, "refId": "D" }, { "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, "editorMode": "code", "exemplar": false, "expr": "kube_node_status_capacity{resource=\"ephemeral_storage\"}", "format": "table", "hide": false, "instant": true, "legendFormat": "__auto", "range": false, "refId": "E" }, { "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, "editorMode": "code", "exemplar": false, "expr": " (100 - (\r\n (node_memory_MemFree_bytes{job=\"node-exporter\"} + node_memory_Buffers_bytes{job=\"node-exporter\"})+ node_memory_Cached_bytes{job=\"node-exporter\"}+node_memory_SReclaimable_bytes{job=\"node-exporter\"}\r\n-(node_memory_Mapped_bytes{job=\"node-exporter\"}+node_memory_Slab_bytes{job=\"node-exporter\"}+node_memory_Shmem_bytes{job=\"node-exporter\"})\r\n)/node_memory_MemTotal_bytes{job=\"node-exporter\"}*100)\r\n", "format": "table", "hide": false, "instant": true, "legendFormat": "__auto", "range": false, "refId": "F" } ], "title": "主机", "transformations": [ { "id": "groupBy", "options": { "fields": { "Value #C": { "aggregations": [ "lastNotNull" ], "operation": "aggregate" }, "Value #D": { "aggregations": [ "lastNotNull" ], "operation": "aggregate" }, "Value #E": { "aggregations": [ "lastNotNull" ], "operation": "aggregate" }, "cluster": { "aggregations": [ "lastNotNull" ], "operation": "aggregate" }, "instance": { "aggregations": [] }, "internal_ip": { "aggregations": [ "lastNotNull" ], "operation": "aggregate" }, "node": { "aggregations": [ "lastNotNull" ], "operation": "groupby" }, "role": { "aggregations": [ "lastNotNull" ], "operation": "aggregate" } } } }, { "id": "merge", "options": {} }, { "id": "organize", "options": { "excludeByName": {}, "includeByName": {}, "indexByName": {}, "renameByName": { "Value #C (lastNotNull)": "CPU", "Value #D (lastNotNull)": "MEMORY", "Value #E (lastNotNull)": "STORAGE", "cluster (lastNotNull)": "Cluster", "internal_ip (lastNotNull)": "IP", "node": "Node", "role (lastNotNull)": "Role" } } } ], "type": "table" } ], "title": "主机", "type": "row" }, { "collapsed": true, "gridPos": { "h": 1, "w": 24, "x": 0, "y": 51 }, "id": 6, "panels": [ { "datasource": { "type": "datasource", "uid": "-- Mixed --" }, "fieldConfig": { "defaults": { "color": { "mode": "thresholds" }, "custom": { "align": "left", "cellOptions": { "type": "auto" }, "filterable": true, "inspect": false }, "fieldMinMax": true, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null }, { "color": "red", "value": 80 } ] }, "unit": "none" }, "overrides": [ { "matcher": { "id": "byName", "options": "MEM(G) Limits" }, "properties": [ { "id": "links", "value": [] }, { "id": "unit", "value": "bytes" }, { "id": "custom.width", "value": 141 } ] }, { "matcher": { "id": "byName", "options": "POD数量" }, "properties": [ { "id": "custom.width", "value": 81 } ] }, { "matcher": { "id": "byName", "options": "MEM(G) Requests" }, "properties": [ { "id": "custom.width", "value": 156 }, { "id": "unit", "value": "bytes" } ] }, { "matcher": { "id": "byName", "options": "qps/m" }, "properties": [ { "id": "custom.width", "value": 127 } ] }, { "matcher": { "id": "byName", "options": "CPU" }, "properties": [ { "id": "custom.width", "value": 67 } ] }, { "matcher": { "id": "byName", "options": "MEM(G)" }, "properties": [ { "id": "custom.width", "value": 96 } ] }, { "matcher": { "id": "byName", "options": "CPU Requests" }, "properties": [ { "id": "custom.width", "value": 123 } ] }, { "matcher": { "id": "byName", "options": "CPU Limts" }, "properties": [ { "id": "custom.width", "value": 111 } ] }, { "matcher": { "id": "byName", "options": "GPU Limits" }, "properties": [ { "id": "custom.width", "value": 126 } ] }, { "matcher": { "id": "byName", "options": "GPU Requests" }, "properties": [ { "id": "custom.width", "value": 137 } ] }, { "matcher": { "id": "byName", "options": "CPU Type" }, "properties": [ { "id": "custom.width", "value": 164 } ] } ] }, "gridPos": { "h": 7, "w": 24, "x": 0, "y": 11 }, "id": 4, "options": { "cellHeight": "sm", "footer": { "countRows": false, "enablePagination": false, "fields": "", "reducer": [ "sum" ], "show": true }, "showHeader": true, "sortBy": [ { "desc": true, "displayName": "GPU Limits" } ] }, "pluginVersion": "10.4.1", "targets": [ { "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, "editorMode": "code", "expr": "sum(\r\n max_over_time(node_namespace_pod_container:container_cpu_usage_seconds_total:sum_irate{cluster=\"$cluster\",namespace=~\"$namespace\", container!=\"POD\"}[$__range])\r\n * on(namespace,pod)\r\n group_left(workload, workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\",workload_type!=\"job\",namespace=~\"$namespace\",workload=~\"$workload\"}\r\n) by (workload,cluster,namespace)", "format": "table", "instant": false, "legendFormat": "", "range": true, "refId": "A" }, { "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, "editorMode": "code", "expr": "sum(\r\n max_over_time(container_memory_working_set_bytes{cluster=\"$cluster\", container!=\"\", image!=\"\",namespace=~\"$namespace\"}[$__range])\r\n * on(namespace,pod)\r\n group_left(workload, workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\",workload_type!=\"job\",namespace=~\"$namespace\",workload=~\"$workload\"}\r\n) by (workload,cluster,namespace)/1024/1024/1024", "format": "table", "hide": false, "instant": false, "legendFormat": "__auto", "range": true, "refId": "B" }, { "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, "editorMode": "code", "expr": "count(\r\n node_namespace_pod_container:container_cpu_usage_seconds_total:sum_irate{cluster=\"$cluster\", container!=\"POD\",namespace=~\"$namespace\"}\r\n * on(namespace,pod)\r\n group_left(workload, workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\",workload_type!=\"job\",namespace=~\"$namespace\",workload=~\"$workload\"}\r\n) by (workload,cluster,namespace,container)", "format": "table", "hide": false, "legendFormat": "__auto", "range": true, "refId": "C" }, { "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, "editorMode": "code", "expr": "sum(\r\n kube_pod_container_resource_limits{cluster=\"$cluster\", namespace=~\"$namespace\",resource=\"cpu\"}\r\n * on(namespace,pod)\r\n group_left(workload, workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\",workload_type!=\"job\",namespace=~\"$namespace\",workload=~\"$workload\"}\r\n) by (workload,cluster,namespace)", "format": "table", "hide": false, "legendFormat": "__auto", "range": true, "refId": "D" }, { "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, "editorMode": "code", "expr": "sum(\r\n kube_pod_container_resource_limits{cluster=\"$cluster\", namespace=~\"$namespace\",resource=\"memory\"}\r\n * on(namespace,pod)\r\n group_left(workload, workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\",workload_type!=\"job\",namespace=~\"$namespace\",workload=~\"$workload\"}\r\n) by (workload,cluster,namespace)", "format": "table", "hide": false, "legendFormat": "__auto", "range": true, "refId": "E" }, { "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, "editorMode": "code", "expr": "max by(workload) (\r\nlabel_replace(\r\n sum by(job) (increase(http_server_requests_seconds_count{cluster=\"$cluster\",namespace=~\"$namespace\",job=~\"$workload\",uri!~\"/actuator/.*\"}[1m])),\r\n \"workload\", \"$1\", \"job\", \"(.*)\"\r\n)\r\nor on(workload)\r\nlabel_replace(\r\n sum(increase(my_promtail_custom_nginx_request_count{cluster=\"$cluster\",exported_service=~\"$workload\"}[1m])) by (exported_service),\r\n \"workload\", \"$1\", \"exported_service\", \"(.*)\"\r\n)\t\r\n)", "format": "table", "hide": false, "legendFormat": "__auto", "range": true, "refId": "F" }, { "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, "editorMode": "code", "expr": "sum(\r\n kube_pod_container_resource_requests{cluster=\"$cluster\", namespace=~\"$namespace\",resource=\"cpu\"}\r\n * on(namespace,pod)\r\n group_left(workload, workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\",workload_type!=\"job\",namespace=~\"$namespace\",workload=~\"$workload\"}\r\n) by (workload,cluster,namespace)", "format": "table", "hide": false, "legendFormat": "__auto", "range": true, "refId": "G" }, { "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, "editorMode": "code", "expr": "sum(\r\n kube_pod_container_resource_requests{cluster=\"$cluster\", namespace=~\"$namespace\",resource=\"memory\"}\r\n * on(namespace,pod)\r\n group_left(workload, workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\",workload_type!=\"job\",namespace=~\"$namespace\",workload=~\"$workload\"}\r\n) by (workload,cluster,namespace)", "format": "table", "hide": false, "legendFormat": "__auto", "range": true, "refId": "H" }, { "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, "editorMode": "code", "expr": "sum(\r\n kube_pod_container_resource_limits{cluster=\"$cluster\", namespace=~\"$namespace\",resource=~\".*gpu.*\",resource!~\".*mem.*\"}\r\n * on(namespace,pod)\r\n group_left(workload, workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\",workload_type!=\"job\",namespace=~\"$namespace\",workload=~\"$workload\"}\r\n) by (workload,cluster,namespace,resource)", "format": "table", "hide": false, "legendFormat": "__auto", "range": true, "refId": "I" }, { "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, "editorMode": "code", "expr": "sum(\r\n kube_pod_container_resource_limits{cluster=\"$cluster\", namespace=~\"$namespace\",resource=~\".*gpu.*\",resource=~\".*mem.*\"}\r\n * on(namespace,pod)\r\n group_left(workload, workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\",workload_type!=\"job\",namespace=~\"$namespace\",workload=~\"$workload\"}\r\n) by (workload,cluster,namespace)", "format": "table", "hide": false, "legendFormat": "__auto", "range": true, "refId": "J" }, { "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, "editorMode": "code", "expr": "sum(\r\n kube_pod_container_resource_requests{cluster=\"$cluster\", namespace=~\"$namespace\",resource=~\".*gpu.*\",resource!~\".*mem.*\"}\r\n * on(namespace,pod)\r\n group_left(workload, workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\",workload_type!=\"job\",namespace=~\"$namespace\",workload=~\"$workload\"}\r\n) by (workload,cluster,namespace,resource)", "format": "table", "hide": false, "legendFormat": "__auto", "range": true, "refId": "K" }, { "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, "editorMode": "code", "expr": "sum(\r\n kube_pod_container_resource_requests{cluster=\"$cluster\", namespace=~\"$namespace\",resource=~\".*gpu.*\",resource=~\".*mem.*\"}\r\n * on(namespace,pod)\r\n group_left(workload, workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\",workload_type!=\"job\",namespace=~\"$namespace\",workload=~\"$workload\"}\r\n) by (workload,cluster,namespace)", "format": "table", "hide": false, "legendFormat": "__auto", "range": true, "refId": "L" } ], "title": "workload resource used", "transformations": [ { "id": "joinByField", "options": { "byField": "workload", "mode": "outer" } }, { "id": "organize", "options": { "excludeByName": { "Time": true, "Time 1": true, "Time 10": true, "Time 11": true, "Time 2": true, "Time 3": true, "Time 4": true, "Time 5": true, "Time 6": true, "Time 7": true, "Time 8": true, "Time 9": true, "cluster 10": true, "cluster 11": true, "cluster 2": true, "cluster 3": true, "cluster 4": true, "cluster 5": true, "cluster 6": true, "cluster 7": true, "cluster 8": true, "cluster 9": true, "container": true, "job": false, "namespace 10": true, "namespace 11": true, "namespace 2": true, "namespace 3": true, "namespace 4": true, "namespace 5": true, "namespace 6": true, "namespace 7": true, "namespace 8": true, "namespace 9": true, "resource 2": true }, "includeByName": {}, "indexByName": { "Time 1": 1, "Time 2": 7, "Time 3": 11, "Time 4": 15, "Time 5": 20, "Time 6": 25, "Time 7": 26, "Time 8": 29, "Value #A": 6, "Value #B": 10, "Value #C": 2, "Value #D": 19, "Value #E": 24, "Value #F": 3, "Value #G": 18, "Value #H": 23, "cluster 1": 4, "cluster 2": 8, "cluster 3": 12, "cluster 4": 16, "cluster 5": 21, "cluster 6": 27, "cluster 7": 30, "container": 14, "namespace 1": 5, "namespace 2": 9, "namespace 3": 13, "namespace 4": 17, "namespace 5": 22, "namespace 6": 28, "namespace 7": 31, "workload": 0 }, "renameByName": { "Value #A": "CPU", "Value #B": "MEM(G)", "Value #C": "POD数量", "Value #D": "CPU Limts", "Value #E": "MEM(G) Limits", "Value #F": "qps/m", "Value #G": "CPU Requests", "Value #H": "MEM(G) Requests", "Value #I": "GPU Limits", "Value #J": "GPU MEM Limits", "Value #K": "GPU Requests", "Value #L": "GPU MEM Requests", "cluster 1": "cluster", "namespace 1": "namespace", "pod 1": "", "resource": "GPU Type", "resource 1": "CPU Type", "workload 1": "", "workload 2": "" } } } ], "type": "table" }, { "datasource": { "type": "datasource", "uid": "-- Mixed --" }, "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "auto", "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null }, { "color": "red", "value": 80 } ] } }, "overrides": [] }, "gridPos": { "h": 12, "w": 24, "x": 0, "y": 18 }, "id": 5, "options": { "legend": { "calcs": [ "last", "max" ], "displayMode": "table", "placement": "bottom", "showLegend": true, "sortBy": "Last", "sortDesc": true }, "tooltip": { "mode": "single", "sort": "none" } }, "targets": [ { "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, "editorMode": "code", "expr": "sum(increase(http_server_requests_seconds_count{cluster=\"$cluster\",namespace=~\"$namespace\",job=~\"$workload\",uri!~\"/actuator/.*\"}[1m])) by(job)", "legendFormat": "qps-jvm:{{job}}", "range": true, "refId": "A" }, { "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, "editorMode": "code", "expr": "sum(\r\n max_over_time(node_namespace_pod_container:container_cpu_usage_seconds_total:sum_irate{cluster=\"$cluster\",namespace=~\"$namespace\", container!=\"POD\"}[$__range])\r\n * on(namespace,pod)\r\n group_left(workload, workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\",workload_type!=\"job\",namespace=~\"$namespace\",workload=~\"$workload\"}\r\n) by (workload,cluster,namespace)", "hide": false, "legendFormat": "cpu_used:{{workload}}", "range": true, "refId": "B" }, { "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, "editorMode": "code", "expr": "sum(\r\n max_over_time(container_memory_working_set_bytes{cluster=\"$cluster\", container!=\"\", image!=\"\",namespace=~\"$namespace\"}[$__range])\r\n * on(namespace,pod)\r\n group_left(workload, workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\",workload_type!=\"job\",namespace=~\"$namespace\",workload=~\"$workload\"}\r\n) by (workload,cluster,namespace)/1024/1024/1024", "hide": false, "legendFormat": "mem_used:{{workload}}", "range": true, "refId": "C" }, { "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, "editorMode": "code", "expr": "sum(delta(my_promtail_custom_nginx_request_count{cluster=\"$cluster\",exported_service=~\"$workload\"}[1m])) by (exported_service)", "hide": false, "legendFormat": "qps-ingress:{{exported_service}}", "range": true, "refId": "D" } ], "title": "服务访问量/m与资源使用量", "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "auto", "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "decimals": 0, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null }, { "color": "red", "value": 80 } ] }, "unit": "none" }, "overrides": [] }, "gridPos": { "h": 11, "w": 24, "x": 0, "y": 30 }, "id": 2, "options": { "legend": { "calcs": [ "last" ], "displayMode": "table", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "single", "sort": "none" } }, "pluginVersion": "10.4.1", "targets": [ { "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, "editorMode": "code", "expr": "max(kubelet_volume_stats_capacity_bytes{cluster=\"$cluster\", job=\"kubelet\", metrics_path=\"/metrics\"})by(cluster,namespace,node,persistentvolumeclaim)/1024/1024/1024", "instant": false, "legendFormat": "{{persistentvolumeclaim}} {{namespace}} {{cluster}}", "range": true, "refId": "A" } ], "title": "持久卷(G)", "type": "timeseries" } ], "title": "workload资源使用量", "type": "row" } ], "refresh": "", "schemaVersion": 39, "tags": [], "templating": { "list": [ { "current": { "selected": true, "text": "prod-core-langge", "value": "prod-core-langge" }, "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, "definition": "label_values(cluster)", "hide": 0, "includeAll": false, "label": "集群", "multi": false, "name": "cluster", "options": [], "query": { "qryType": 1, "query": "label_values(cluster)", "refId": "PrometheusVariableQueryEditor-VariableQuery" }, "refresh": 1, "regex": "", "skipUrlSync": false, "sort": 0, "type": "query" }, { "current": { "selected": true, "text": [ "cattle-fleet-system" ], "value": [ "cattle-fleet-system" ] }, "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, "definition": "label_values(kube_namespace_status_phase{cluster=\"$cluster\"},namespace)", "hide": 0, "includeAll": false, "label": "namespace", "multi": true, "name": "namespace", "options": [], "query": { "qryType": 1, "query": "label_values(kube_namespace_status_phase{cluster=\"$cluster\"},namespace)", "refId": "PrometheusVariableQueryEditor-VariableQuery" }, "refresh": 1, "regex": "", "skipUrlSync": false, "sort": 0, "type": "query" }, { "current": { "selected": true, "text": [ "All" ], "value": [ "$__all" ] }, "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, "definition": "label_values(namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=~\"$namespace\"},workload)", "hide": 0, "includeAll": true, "label": "workload_name", "multi": true, "name": "workload", "options": [], "query": { "qryType": 1, "query": "label_values(namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=~\"$namespace\"},workload)", "refId": "PrometheusVariableQueryEditor-VariableQuery" }, "refresh": 1, "regex": "", "skipUrlSync": false, "sort": 0, "type": "query" }, { "current": { "selected": false, "text": "4", "value": "4" }, "description": "单个服务cpu需求", "hide": 0, "label": "cpu(核)", "name": "cpu_core", "options": [ { "selected": true, "text": "4", "value": "4" } ], "query": "4", "skipUrlSync": false, "type": "textbox" }, { "current": { "selected": false, "text": "4", "value": "4" }, "description": "单个服务内存需求", "hide": 0, "label": "内存(G)", "name": "mem", "options": [ { "selected": true, "text": "4", "value": "4" } ], "query": "4", "skipUrlSync": false, "type": "textbox" }, { "current": { "selected": false, "text": "0", "value": "0" }, "description": "单个服务cpu需求(core/unit)", "hide": 0, "label": "gpu数", "name": "gpu_count", "options": [ { "selected": true, "text": "0", "value": "0" } ], "query": "0", "skipUrlSync": false, "type": "textbox" }, { "current": { "selected": false, "text": "0", "value": "0" }, "description": "单个服务cpu_mem需求", "hide": 0, "label": "gpu_mem(M)", "name": "gpu_mem", "options": [ { "selected": true, "text": "0", "value": "0" } ], "query": "0", "skipUrlSync": false, "type": "textbox" }, { "current": { "selected": false, "text": "1", "value": "1" }, "description": "服务数量", "hide": 0, "label": "副本数", "name": "workload_count", "options": [ { "selected": true, "text": "1", "value": "1" } ], "query": "1", "skipUrlSync": false, "type": "textbox" } ] }, "time": { "from": "now-5m", "to": "now" }, "timepicker": {}, "timezone": "browser", "title": "资源一览表", "uid": "beo415f3bwsn4a", "version": 44, "weekStart": "" } (责任编辑：liangzh) |