This how-to guide covers how to manage Grafana dashboards across 2i2c clusters, including how to add new dashboards and managing deployments via CI/CD.
Existing dashboards¶
We currently deploy the following dashboards with CI/CD:
JupyterHub Default Dashboards for all clusters
Cloud Cost Dashboards for all AWS clusters
This is handled by the deployer command deployer deploy-dashboards, which clones these GitHub repositories and uses the ./deploy.py script to communicate with the Grafana REST API to deploy the dashboards.
This workflow file defines the GitHub Action to run the CI/CD job once per day.
Adding custom dashboards¶
Prerequisites¶
Install
go-jsonnet(the Grafonnet library is vendored with thejsonnet-builder).
Dashboards as code¶
You can manually prototype and create Grafana dashboards with the UI.
Once you are happy with the dashboard design, you can export the dashboard as JSON.
Example: Usage Quotas Dashboard as JSON
./example-usage-quotas.json
{
"apiVersion": "dashboard.grafana.app/v1beta1",
"kind": "Dashboard",
"metadata": {
"name": "adw5m5h",
"generation": 36,
"creationTimestamp": "2026-05-21T14:50:12Z",
"labels": {},
"annotations": {}
},
"spec": {
"annotations": {
"list": [
{
"builtIn": 1,
"datasource": {
"type": "grafana",
"uid": "-- Grafana --"
},
"enable": true,
"hide": true,
"iconColor": "rgba(0, 211, 255, 1)",
"name": "Annotations & Alerts",
"type": "dashboard"
}
]
},
"editable": true,
"fiscalYearStartMonth": 0,
"graphTooltip": 0,
"id": 31,
"links": [],
"panels": [
{
"datasource": {
"type": "prometheus",
"uid": "P1809F7CD0C75ACF3"
},
"fieldConfig": {
"defaults": {
"color": {
"mode": "palette-classic"
},
"custom": {
"axisBorderShow": false,
"axisCenteredZero": false,
"axisColorMode": "text",
"axisLabel": "",
"axisPlacement": "auto",
"barAlignment": 0,
"barWidthFactor": 0.6,
"drawStyle": "line",
"fillOpacity": 0,
"gradientMode": "none",
"hideFrom": {
"legend": false,
"tooltip": false,
"viz": false
},
"insertNulls": false,
"lineInterpolation": "linear",
"lineStyle": {
"fill": "solid"
},
"lineWidth": 2,
"pointSize": 5,
"scaleDistribution": {
"type": "linear"
},
"showPoints": "never",
"showValues": false,
"spanNulls": false,
"stacking": {
"group": "A",
"mode": "none"
},
"thresholdsStyle": {
"mode": "off"
}
},
"fieldMinMax": false,
"mappings": [],
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green",
"value": 0
}
]
},
"unit": "GiB-hr"
},
"overrides": []
},
"gridPos": {
"h": 13,
"w": 24,
"x": 0,
"y": 0
},
"id": 1,
"options": {
"legend": {
"calcs": [
"last"
],
"displayMode": "table",
"placement": "bottom",
"showLegend": true,
"sortBy": "Last",
"sortDesc": true
},
"tooltip": {
"hideZeros": false,
"mode": "single",
"sort": "none"
}
},
"pluginVersion": "12.3.1",
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "P1809F7CD0C75ACF3"
},
"editorMode": "code",
"expr": "max(jupyterhub_memory_usage_byte_hours_total{namespace=~\"$hub\", policy_group!=\"\", username=~\"$username\"}) by (namespace, username, policy_group, window) / 2^30",
"hide": false,
"instant": false,
"legendFormat": "user={{ username }}, policy group={{ policy_group }} ({{ namespace }})",
"range": true,
"refId": "A"
},
{
"datasource": {
"type": "prometheus",
"uid": "P1809F7CD0C75ACF3"
},
"editorMode": "code",
"expr": "max(jupyterhub_memory_limit_byte_hours_total{namespace=~\"$hub\"}) by (namespace, username, policy_group, window) / 2^30",
"hide": true,
"instant": false,
"legendFormat": "__auto",
"range": true,
"refId": "B"
}
],
"title": "Cumulative compute usage over the last 30 days",
"type": "timeseries"
},
{
"datasource": {
"type": "prometheus",
"uid": "P1809F7CD0C75ACF3"
},
"description": "This happens when a user server is allowed to launch when the usage quota system is unavailable.",
"fieldConfig": {
"defaults": {
"color": {
"mode": "palette-classic"
},
"custom": {
"axisBorderShow": false,
"axisCenteredZero": false,
"axisColorMode": "text",
"axisLabel": "",
"axisPlacement": "auto",
"barAlignment": 0,
"barWidthFactor": 0.6,
"drawStyle": "line",
"fillOpacity": 0,
"gradientMode": "none",
"hideFrom": {
"legend": false,
"tooltip": false,
"viz": false
},
"insertNulls": false,
"lineInterpolation": "linear",
"lineWidth": 1,
"pointSize": 5,
"scaleDistribution": {
"type": "linear"
},
"showPoints": "auto",
"showValues": false,
"spanNulls": false,
"stacking": {
"group": "A",
"mode": "none"
},
"thresholdsStyle": {
"mode": "off"
}
},
"mappings": [],
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green",
"value": 0
},
{
"color": "red",
"value": 80
}
]
}
},
"overrides": []
},
"gridPos": {
"h": 8,
"w": 12,
"x": 0,
"y": 13
},
"id": 2,
"options": {
"legend": {
"calcs": [],
"displayMode": "list",
"placement": "bottom",
"showLegend": true
},
"tooltip": {
"hideZeros": false,
"mode": "single",
"sort": "none"
}
},
"pluginVersion": "12.3.1",
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "P1809F7CD0C75ACF3"
},
"editorMode": "code",
"expr": "sum by (namespace) (changes(jupyterhub_usage_quotas_fail_open_total[30m]))",
"instant": false,
"legendFormat": "__auto",
"range": true,
"refId": "A"
}
],
"title": "Monitoring - Fail opens",
"type": "timeseries"
},
{
"datasource": {
"type": "prometheus",
"uid": "P1809F7CD0C75ACF3"
},
"description": "User server launch denied due to exhausted quota.",
"fieldConfig": {
"defaults": {
"color": {
"mode": "palette-classic"
},
"custom": {
"axisBorderShow": false,
"axisCenteredZero": false,
"axisColorMode": "text",
"axisLabel": "",
"axisPlacement": "auto",
"barAlignment": 0,
"barWidthFactor": 0.6,
"drawStyle": "line",
"fillOpacity": 0,
"gradientMode": "none",
"hideFrom": {
"legend": false,
"tooltip": false,
"viz": false
},
"insertNulls": false,
"lineInterpolation": "linear",
"lineWidth": 1,
"pointSize": 5,
"scaleDistribution": {
"type": "linear"
},
"showPoints": "auto",
"showValues": false,
"spanNulls": false,
"stacking": {
"group": "A",
"mode": "none"
},
"thresholdsStyle": {
"mode": "off"
}
},
"mappings": [],
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green",
"value": 0
},
{
"color": "red",
"value": 80
}
]
}
},
"overrides": []
},
"gridPos": {
"h": 8,
"w": 12,
"x": 12,
"y": 13
},
"id": 3,
"options": {
"legend": {
"calcs": [],
"displayMode": "list",
"placement": "bottom",
"showLegend": true
},
"tooltip": {
"hideZeros": false,
"mode": "single",
"sort": "none"
}
},
"pluginVersion": "12.3.1",
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "P1809F7CD0C75ACF3"
},
"editorMode": "code",
"expr": "changes((max by (namespace) (jupyterhub_request_duration_seconds_count{code=\"422\",handler=\"jupyterhub.handlers.pages.SpawnPendingHandler\"} or 0 * jupyterhub_request_duration_seconds_count))[30m:1m])",
"instant": false,
"legendFormat": "__auto",
"range": true,
"refId": "A"
}
],
"title": "Monitoring - Denied server launch (last 30m)",
"type": "timeseries"
}
],
"preload": false,
"schemaVersion": 42,
"tags": [],
"templating": {
"list": [
{
"allowCustomValue": false,
"current": {
"text": "All",
"value": [
"$__all"
]
},
"datasource": {
"type": "prometheus",
"uid": "P1809F7CD0C75ACF3"
},
"definition": "label_values(jupyterhub_memory_usage_byte_hours_total,namespace)",
"includeAll": true,
"label": "Hub",
"multi": true,
"name": "hub",
"options": [],
"query": {
"qryType": 1,
"query": "label_values(jupyterhub_memory_usage_byte_hours_total,namespace)",
"refId": "PrometheusVariableQueryEditor-VariableQuery"
},
"refresh": 1,
"regex": "",
"type": "query"
},
{
"allowCustomValue": false,
"current": {
"text": "All",
"value": [
"$__all"
]
},
"datasource": {
"type": "prometheus",
"uid": "P1809F7CD0C75ACF3"
},
"definition": "label_values(jupyterhub_memory_usage_byte_hours_total,username)",
"includeAll": true,
"label": "User",
"multi": true,
"name": "username",
"options": [],
"query": {
"qryType": 1,
"query": "label_values(jupyterhub_memory_usage_byte_hours_total,username)",
"refId": "PrometheusVariableQueryEditor-VariableQuery"
},
"refresh": 1,
"regex": "",
"sort": 1,
"type": "query"
}
]
},
"time": {
"from": "now-7d",
"to": "now"
},
"timepicker": {},
"timezone": "browser",
"title": "Usage Quotas - Experimental",
"uid": "adw5m5h",
"version": 36
},
"status": {}
}Grafonnet¶
We can use the JSON as a template to encapsulate the dashboard as Grafonnet for reproducibility.
In the top-level dashboards folder, you can find
common.libsonnetdefines common configuration such as datasources, variables and styling*.jsonnetdefines dashboard specific code.
The Grafonnet documentation is a good resource for learning how to encapsulate your dashboard design as Grafonnet/Jsonnet code.
You can check your work is valid Jsonnet as you develop by running
jsonnet -J dashboards/vendor dashboards/<dashboard-name>.jsonnetExample: Usage Quotas Dashboard as Grafonnet
./usage-quotas.jsonnet
local grafonnet = import 'github.com/grafana/grafonnet/gen/grafonnet-v11.1.0/main.libsonnet';
local dashboard = grafonnet.dashboard;
local ts = grafonnet.panel.timeSeries;
local tb = grafonnet.panel.table;
local prometheus = grafonnet.query.prometheus;
local var = grafonnet.dashboard.variable;
local link = grafonnet.dashboard.link;
local common = import './common.libsonnet';
local windowVar =
var.query.new('window')
+ var.query.queryTypes.withLabelValues('window', 'jupyterhub_memory_usage_byte_hours_total')
+ var.query.withDatasourceFromVariable(common.variables.prometheus)
+ var.query.generalOptions.withDescription('Rolling window (days) quota policy is applied over.')
;
local computeUsage =
common.tsOptions
+ ts.new('Cumulative compute usage over the last $window days')
+ ts.panelOptions.withDescription(
|||
Time series of the cumulative compute usage over the last $window days.
|||
)
+ ts.panelOptions.withGridPos(h=13, w=24, x=0, y=0)
+ ts.queryOptions.withTargets([
prometheus.new(
'$PROMETHEUS_DS',
|||
max(jupyterhub_memory_usage_byte_hours_total{namespace=~"$hub", policy_group!="", username=~"$user", window=~"$window"}) by (namespace, username, policy_group, window) / 2^30
|||
)
+ prometheus.withLegendFormat(
|||
user={{ username }}, policy group={{ policy_group }} ({{ namespace }})
|||
),
])
+ ts.options.legend.withCalcs(value=['last'])
+ ts.options.legend.withDisplayMode('table')
+ ts.options.legend.withPlacement('bottom')
+ ts.options.legend.withShowLegend()
+ ts.options.legend.withSortBy('Last')
+ ts.options.legend.withSortDesc()
+ ts.options.tooltip.withMode('single')
+ ts.standardOptions.withUnit('GiB-hr')
;
local computeLimits =
tb.new('Compute usage policies')
+ tb.panelOptions.withDescription(
|||
Compute usage quota limits individually applied to policy group members.
|||
)
+ tb.panelOptions.withGridPos(h=8, w=10, x=0, y=13)
+ tb.queryOptions.withTargets([
prometheus.new(
'$PROMETHEUS_DS',
|||
avg(jupyterhub_memory_limit_byte_hours_total{namespace=~"$hub", window=~"$window"}) by (namespace, window, policy_group) / 2^30
|||
)
+ prometheus.withInstant()
+ prometheus.withLegendFormat('{{policy_group}} ({{namespace}}) – {{window}} day window'),
])
+ tb.queryOptions.withTransformations([
tb.queryOptions.transformation.withId('reduce')
+ tb.queryOptions.transformation.withOptions({
labelsToFields: false,
reducers: [
'lastNotNull',
],
}),
tb.queryOptions.transformation.withId('organize')
+ tb.queryOptions.transformation.withOptions({
renameByName: {
Field: 'policy group',
'Last *': 'quota limit (GiB-hr)',
},
}),
])
+ tb.standardOptions.withOverrides(
[
{
matcher: {
id: 'byName',
options: 'policy group',
},
properties: [
{
id: 'custom.width',
value: 293,
},
],
},
]
)
;
local deniedServer =
common.tsOptions
+ ts.new('Monitoring - Denied server launch')
+ ts.panelOptions.withDescription(
|||
Server launch denied due to user exceeding compute quota limit.
|||
)
+ ts.panelOptions.withGridPos(h=8, w=7, x=10, y=13)
+ ts.queryOptions.withTargets([
prometheus.new(
'$PROMETHEUS_DS',
|||
changes((max by (namespace) (jupyterhub_request_duration_seconds_count{code="422",handler="jupyterhub.handlers.pages.SpawnPendingHandler"} or 0 * jupyterhub_request_duration_seconds_count))[30m:1m])
|||
)
+ prometheus.withLegendFormat('__auto'),
])
;
local failOpens =
common.tsOptions
+ ts.new('Monitoring - Fail opens')
+ ts.panelOptions.withDescription(
|||
This happens when a user server is allowed to launch when the usage quota system is unavailable.
|||
)
+ ts.panelOptions.withGridPos(h=8, w=7, x=17, y=13)
+ ts.queryOptions.withTargets([
prometheus.new(
'$PROMETHEUS_DS',
|||
sum by (namespace) (changes(jupyterhub_usage_quotas_fail_open_total[30m]))
|||
),
])
+ ts.options.legend.withShowLegend(false)
;
dashboard.new('Usage Quotas')
+ dashboard.withUid('compute-usage-quotas')
+ dashboard.withTimezone('utc')
+ dashboard.withEditable(true)
+ dashboard.time.withFrom('now-7d')
+ dashboard.withVariables([
common.variables.prometheus,
common.variables.hub,
common.variables.user,
windowVar,
])
+ dashboard.withLinks([
link.link.new('Community Hub Guide', 'https://docs.2i2c.org/admin/user-management/compute-quotas/'),
])
+ dashboard.withPanels(
[
computeUsage,
computeLimits,
deniedServer,
failOpens,
],
)
Manually deploy dashboards¶
To manually deploy your dashboard to a Grafana instance, run the command
deployer deploy-dashboards --dashboard-type custom $CLUSTER_NAME