Commit f0903eec authored by 20after4's avatar 20after4
Browse files

Add datasette + dashboards to the train-stats project.

parent 5220a3d2
databases:
train:
queries:
train-overview:
SELECT
version,
rollbacks,
rollbacks_time,
group2_delay_days,
(group0_delay_days +
group1_delay_days +
group2_delay_days) as total_delay,
total_time as train_total_time,
(select count(*) from blocker b where b.train_id = t.id) as blockers,
(select count(*) from blocker b where b.train_id = t.id and resolved = 1) as resolved_blockers,
patches,
(select max(time_in_review) from patch p where p.train_id = t.id)/(60*60) as max_time_in_review,
(select max(comments) from patch where patch.train_id = t.id) as max_comments_per_patch,
(select max(start_time - created) from patch p where p.train_id = t.id)/(60*60) as max_cycle_time
FROM train t
ORDER BY version DESC limit 10
plugins:
datasette-dashboards:
train:
title: Train Dashboard
description: Train metrics demo
layout:
- [train-timing, events-count]
- [train-timing, train-patches]
filters:
date_start:
name: Date Start
type: date
default: "2021-01-01"
date_end:
name: Date End
type: date
charts:
analysis-note:
library: markdown
display: |-
# Analysis notes
> A quick rundown of events statistics and KPIs
events-count:
title: Total number of trains
db: train
query: SELECT count(*) as count FROM train
library: metric
display:
field: count
prefix:
suffix:
train-patches:
title: Number of trains by conductor
db: train
query: SELECT conductor, count(version) as count, sum(patches) as patches, sum(rollbacks) as rollbacks FROM train WHERE TRUE GROUP BY conductor
library: vega
display:
mark: { type: bar, tooltip: true }
encoding:
y: { field: conductor, type: nominal }
color: { field: conductor, type: nominal }
x: { field: count, type: quantitative }
train-timing:
title: Cycle time and Lead time
db: train
query: SELECT
t.id,
t.version,
datetime(start_time, 'unixepoch') as start_time,
(start_time - created)/(60*60) as cycle_time,
(start_time - submitted)/(60*60) as lead_time,
datetime(created, 'unixepoch') as created,
datetime(submitted, 'unixepoch') submitted
FROM
patch p
JOIN train t ON t.id = p.train_id
WHERE
(
lead_time > 0
AND cycle_time > 0
)
order by start_time desc
library: vega
display:
transform: [
{
fold: ["lead_time", "cycle_time"]
}
]
layer: [
{
mark: { type: bar }
},
{
mark: { type: text, color: white, dx: 10 },
encoding: {
text: {
field: value,
aggregate: mean,
format: d,
fontSize: 18
},
color: { value: white }
}
}
]
encoding: {
x: {
aggregate: mean,
field: value,
axis: { title: "Lead, Cycle time (minutes)", labelFontSize: 12 },
type: quantitative,
# "scale": {"zero": false},
},
y: {
field: version,
type: ordinal,
axis: {
labelAngle: 0,
labelFontSize: 17
},
},
color: {
field: key,
type: nominal,
}
}
#!/bin/bash
# This is a script that can be used to reload datasette and refresh firefox when you save changes.
# Auto reload/refresh makes testing changes a lot easier/quicker.
#
# This is essentially an infinite loop that restarts datasette when datasette exits,
# then triggers firefox to refresh the current tab using xdotool (unix/xwindows only, sorry mac people)
#
# This script triggers a restart by sending a signal to the datasette process:
#
# SIGUSR1 causes datasette to exit and the loop below restarts it.
#
# To make this useful, I have a vs.code plugin set up to run this script every time
# I save a file in the data/ subdirectory. This avoids complex/resource limited filesystem watchers.
#
# run this script in a spare terminal to keep datasette running, then have your editor trigger
# this same script to restart datasette. The script doesn't go into an infinite loop if it finds
# and existing datasette process to kill. If no datasette process is founed then it loops.
# ctrl-c to end the infinite loop.
#
source venv/bin/activate
PORT=8002
refresh() {
# refresh the active tab in the first visible firefox window that is found.
# if you have more than one firefox window, minimize all but one.
sleep 2
CURRENT_WID=$(xdotool getwindowfocus)
WID=$(xdotool search --onlyvisible --limit 1 --name "Mozilla Firefox")
xdotool windowactivate $WID
xdotool key F5
xdotool windowactivate $CURRENT_WID
exit 0 # don't run the infinite loop if we successfully killed a background process
}
# restart and refresh
pkill -f "datasette.+$PORT" -USR1 && refresh;
# we only get this far if datasette wasn't already running, so start it and go into the loop:
command="datasette -p $PORT -h 0.0.0.0 --reload ${fileDirname} $@"
echo "running: $command"
cleanup () {
exit 0;
}
trap "cleanup" SIGINT
# trap the SIGINT so that ctrl+c still works in the controlling terminal.
while [ 1 ]; do
echo $command;
$command;
done
echo "Exiting..."
{
"sql_time_limit_ms": 8000
}
\ No newline at end of file
[tool.poetry]
name = "train-stats"
version = "0.1.0"
description = "exploratory data analysis of wikimedia production deployments."
authors = ["Tyler Cipriani <thcipriani@wikimedia.org>",
"Mukunda Modell <mmodell@wikimedia.org>"]
[tool.poetry.dependencies]
python = "^3.7"
datasette = "^0.58.1"
datasette-dashboards = "^0.1.6"
datasette-vega = "^0.6.2"
[tool.poetry.dev-dependencies]
ipython = "^7.27.0"
ddd = "^0.6"
phabricator = "^0.8.1"
[build-system]
requires = ["poetry-core>=1.0.0"]
build-backend = "poetry.core.masonry.api"
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment