Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 2 additions & 0 deletions .github/workflows/deploy-production.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -107,13 +107,15 @@ jobs:
cp ~/.env-production dashboard-production/.env &&
cd dashboard-production &&
git checkout ${GITHUB_SHA} &&
export DASHBOARD_VERSION=${DASHBOARD_VERSION} &&
docker compose -f docker-compose-next.yml pull --policy=always &&
docker compose -f docker-compose-next.yml up -d --remove-orphans
"
env:
SSH_USER: ${{ secrets.STAGING_USER }}
SSH_HOST: ${{ secrets.STAGING_HOST }}
SSH_KEY: ${{ secrets.STAGING_KEY }}
DASHBOARD_VERSION: ${{ github.event.inputs.tag }}

- name: Discord notification on success
if: success()
Expand Down
8 changes: 8 additions & 0 deletions backend/kernelCI/wsgi.py
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,15 @@
import os

from django.core.wsgi import get_wsgi_application
from prometheus_client import Gauge

os.environ.setdefault("DJANGO_SETTINGS_MODULE", "kernelCI.settings")

application = get_wsgi_application()

Gauge(
"dashboard_build_info",
"Deployed backend version, always reported as 1",
["version"],
multiprocess_mode="max",
).labels(version=os.environ.get("DASHBOARD_VERSION") or "unknown").set(1)
2 changes: 2 additions & 0 deletions docker-compose-next.yml
Original file line number Diff line number Diff line change
Expand Up @@ -52,6 +52,8 @@ services:
- --bind=0.0.0.0:8000
- --timeout=250
env_file: .env
environment:
- DASHBOARD_VERSION=${DASHBOARD_VERSION:-unknown}
volumes:
- backend-data:${BACKEND_VOLUME_DIR:-/volume_data}
restart: always
Expand Down
1 change: 1 addition & 0 deletions docker-compose.dev.yml
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,7 @@ services:
env_file: [.env]
environment:
- PYTHONOPTIMIZE=0 # required for runserver/reload to work correctly
- DASHBOARD_VERSION=${DASHBOARD_VERSION:-local-dev}
networks: [private, public]
ports: ["8000:8000"]
command:
Expand Down
2 changes: 2 additions & 0 deletions docker-compose.yml
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,8 @@ services:
- backend-data:${BACKEND_VOLUME_DIR:-/volume_data}
env_file:
- .env
environment:
- DASHBOARD_VERSION=${DASHBOARD_VERSION:-local-dev}
networks:
- private
- public
Expand Down
2 changes: 2 additions & 0 deletions docs/monitoring.md
Original file line number Diff line number Diff line change
Expand Up @@ -71,6 +71,7 @@ After importing the dashboard, you'll have:
- Total Calls
- Average Response Time
- Total Time (cumulative time per endpoint)
- **Deploys** - Annotation and Running Version panel from `dashboard_build_info` (`DASHBOARD_VERSION`)

### Aggregation Process Dashboard

Expand All @@ -93,6 +94,7 @@ The monitoring system supports multi-worker Gunicorn deployments using Prometheu
### Configuration

#### Environment Variables
- `DASHBOARD_VERSION`: Label on `dashboard_build_info` (default: `unknown`)
- `PROMETHEUS_METRICS_ENABLED`: Set to `true` to enable Prometheus metrics (default: `false`)
- `PROMETHEUS_METRICS_PORT`: Port for the metrics aggregator (default: `8001`)
- `PROMETHEUS_MULTIPROC_DIR`: Directory for multiprocess metric files (default: `/tmp/prometheus_multiproc_dir`)
Expand Down
103 changes: 95 additions & 8 deletions monitoring/dashboard.json
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,23 @@
"iconColor": "rgba(0, 211, 255, 1)",
"name": "Annotations & Alerts",
"type": "dashboard"
},
{
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"enable": true,
"iconColor": "purple",
"name": "Deploys",
"target": {
"expr": "dashboard_build_info unless dashboard_build_info offset 1m",
"instant": false,
"interval": "60s",
"legendFormat": "",
"refId": "Anno"
},
"titleFormat": "Deploy {{version}}"
}
]
},
Expand All @@ -22,6 +39,60 @@
"links": [],
"liveNow": false,
"panels": [
{
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"fieldConfig": {
"defaults": {
"color": {
"fixedColor": "text",
"mode": "fixed"
},
"mappings": [],
"noValue": "unknown"
},
"overrides": []
},
"gridPos": {
"h": 3,
"w": 24,
"x": 0,
"y": 0
},
"id": 4,
"options": {
"colorMode": "none",
"graphMode": "none",
"justifyMode": "auto",
"orientation": "auto",
"reduceOptions": {
"calcs": [
"lastNotNull"
],
"fields": "",
"values": false
},
"textMode": "name"
},
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"editorMode": "code",
"expr": "dashboard_build_info",
"instant": true,
"legendFormat": "{{version}}",
"range": false,
"refId": "A"
}
],
"title": "Running Version",
"type": "stat"
},
{
"datasource": {
"type": "prometheus",
Expand Down Expand Up @@ -82,7 +153,7 @@
"h": 8,
"w": 12,
"x": 0,
"y": 0
"y": 3
},
"id": 1,
"options": {
Expand Down Expand Up @@ -173,7 +244,7 @@
"h": 8,
"w": 12,
"x": 12,
"y": 0
"y": 3
},
"id": 2,
"options": {
Expand Down Expand Up @@ -355,7 +426,7 @@
"h": 8,
"w": 24,
"x": 0,
"y": 8
"y": 11
},
"id": 3,
"options": {
Expand Down Expand Up @@ -443,11 +514,15 @@
"operation": "groupby"
},
"Total Calls": {
"aggregations": ["sum"],
"aggregations": [
"sum"
],
"operation": "aggregate"
},
"Total Time": {
"aggregations": ["sum"],
"aggregations": [
"sum"
],
"operation": "aggregate"
}
},
Expand Down Expand Up @@ -525,15 +600,27 @@
"templating": {
"list": [
{
"current": { "selected": false, "text": "Time Range", "value": "range" },
"current": {
"selected": false,
"text": "Time Range",
"value": "range"
},
"hide": 0,
"includeAll": false,
"multi": false,
"label": "Summary Mode",
"name": "summary_mode",
"options": [
{ "selected": false, "text": "All Time", "value": "all" },
{ "selected": true, "text": "Time Range", "value": "range" }
{
"selected": false,
"text": "All Time",
"value": "all"
},
{
"selected": true,
"text": "Time Range",
"value": "range"
}
],
"query": "all,range",
"queryValue": "",
Expand Down
Loading