From c8aac01e0ce2293757a48b27cb0001aae218da0b Mon Sep 17 00:00:00 2001 From: Mukunda Modell Date: Tue, 19 Oct 2021 23:06:58 -0500 Subject: [PATCH 1/8] A whole bunch more changes. The most important change included here is the view system in ddd_datasette.py there is a request url router that delegates to python modules in www/templates/views/*.py and then passes the variables from the model to a template with a matching filename (except the extension is html) So the url /-/ddd/$page loads the python module from `www/templates/views/$page.py` which returns some variables for the view. Then the variables are passed to a jinja template named `www/templates/views/$page.html` Also, canned queries are now loaded from sql files in www/sql/$db_name/$query_name Finally, the function `sql()` can be called from templates and it can reference canned queries (See extra_template_vars in ddd_datasette.py) --- .gitignore | 1 + README.md | 15 +- etc/systemd/ddd-datasette.sh | 1 + src/datasette-dashboards | 2 +- src/ddd/boardmetrics.py | 54 +- src/ddd/boardmetrics_mapper.py | 67 ++- src/ddd/boardmetrics_schema.py | 96 +++- src/ddd/phab.py | 9 +- src/ddd/phobjects.py | 191 +++++-- typings/datasette/__init__.pyi | 2 +- typings/datasette/app.pyi | 6 +- typings/datasette/database.pyi | 113 ++-- www/metadata-wip.yaml | 392 +++++++++++++ www/metadata.yaml | 521 +++++++++--------- www/plugins/ddd_datasette.py | 196 ++++--- www/plugins/phabricator_datasette_plugin.py | 202 ++++++- .../sql/metrics/all_events.bak.sql | 0 www/{plugins => }/sql/metrics/all_events.sql | 13 +- .../sql/metrics/column_events.sql | 0 .../sql/metrics/columns_rollup.sql | 0 .../sql/metrics/project_events.sql | 0 www/static/styles.css | 9 + www/templates/pages/project/{project}.html | 175 ------ www/templates/pages/projects.html | 22 - www/templates/pages/train/{version}.html | 10 - .../pages/trainsummary-{version}.html | 49 -- www/templates/project_columns.html | 11 +- www/templates/subprojects.html | 7 +- www/templates/task_metrics.html | 7 + www/templates/test.html | 1 + www/templates/views/sankey.html | 84 +++ www/templates/views/sankey.py | 39 ++ 32 files changed, 1524 insertions(+), 771 deletions(-) create mode 100644 etc/systemd/ddd-datasette.sh create mode 100644 www/metadata-wip.yaml rename www/{plugins => }/sql/metrics/all_events.bak.sql (100%) rename www/{plugins => }/sql/metrics/all_events.sql (80%) rename www/{plugins => }/sql/metrics/column_events.sql (100%) rename www/{plugins => }/sql/metrics/columns_rollup.sql (100%) rename www/{plugins => }/sql/metrics/project_events.sql (100%) delete mode 100644 www/templates/pages/project/{project}.html delete mode 100644 www/templates/pages/projects.html delete mode 100644 www/templates/pages/train/{version}.html delete mode 100644 www/templates/pages/trainsummary-{version}.html create mode 100644 www/templates/task_metrics.html create mode 100644 www/templates/test.html create mode 100644 www/templates/views/sankey.html create mode 100644 www/templates/views/sankey.py diff --git a/.gitignore b/.gitignore index 480d749..2b64cfd 100644 --- a/.gitignore +++ b/.gitignore @@ -9,6 +9,7 @@ __pycache__ *.egg-info *.db /.eggs +/build /dist /.venv /test/*.json diff --git a/README.md b/README.md index 2413fb7..2aebd4c 100644 --- a/README.md +++ b/README.md @@ -1,7 +1,16 @@ -# D³ +# Data³ -`ddd` or `d³` is a toolkit for accessing APIs and processing data from disperate - systems. +Data³ is a toolkit and general framework for visualizing just about any data. Wikimedia's engineering productivity team have begun assembling a toolkit to help us organize, analyze and visualize data collected from our development, deployment, testing and project planning processes. There is a need for better tooling and data collection in order to have reliable and accessible data to inform data-driven decision-making. This is important because we need to measure the impact of changes to our deployment processes and team practices so that we can know whether a change to our process is beneficial and quantify the impacts of the changes we make. + +The first applications for the Data³ tools are focused on exploring software development and deployment data, as well as workflow metrics exported from Wikimedia's phabricator instance. + +The core of the toolkit consists of the following: + +* Datasette.io provides a front-end for browsing and querying one or more SQLite databases. +* A customized version of datasette-dashboards is included for visualizing the output of queries in Vega/Vega-Lite charts and it can render jinja templates for custom reports or interactive displays. +* A comprehensive python library and command line interface for querying and processing Phabricator task data exported via conduit api requests. +* Several custom dashboards for datasette which provide visualization of metrics related to Phabricator tasks and workflows. +* A custom dashboard to explore data and statistics about production MediaWiki deployments. ## Status diff --git a/etc/systemd/ddd-datasette.sh b/etc/systemd/ddd-datasette.sh new file mode 100644 index 0000000..f307f83 --- /dev/null +++ b/etc/systemd/ddd-datasette.sh @@ -0,0 +1 @@ +datasette --reload --metadata www/metadata.yaml -h 0.0.0.0 -p 8001 www \ No newline at end of file diff --git a/src/datasette-dashboards b/src/datasette-dashboards index bf224aa..2a3cdd1 160000 --- a/src/datasette-dashboards +++ b/src/datasette-dashboards @@ -1 +1 @@ -Subproject commit bf224aa117be3f9c882ce4de27b09f8fa3dccb8c +Subproject commit 2a3cdd1f03e5d20462da5579076edfb924272ece diff --git a/src/ddd/boardmetrics.py b/src/ddd/boardmetrics.py index f3f0406..84c3a7c 100755 --- a/src/ddd/boardmetrics.py +++ b/src/ddd/boardmetrics.py @@ -6,13 +6,14 @@ import pathlib import sqlite3 import subprocess import sys -from datetime import datetime +from datetime import datetime, timedelta from pprint import pprint from sqlite3 import Connection from typing import Iterable, Optional, Sized import click from rich.console import Console +from ddd import console import typer from rich.status import Status from sqlite_utils.db import Database, chunks @@ -21,7 +22,7 @@ from typer import Option, Typer from ddd.boardmetrics_mapper import maptransactions from ddd.boardmetrics_schema import Config, config from ddd.phab import Conduit -from ddd.phobjects import PHID, PHObject, PHObjectEncoder, init_caches +from ddd.phobjects import PHID, PHObject, PHObjectEncoder, Project, init_caches thisdir = pathlib.Path(__file__).parent mock_default = thisdir / ".." / "test" / "transactions.json" @@ -44,15 +45,29 @@ def cache_tasks(conduit, cache, tasks, sts): # cache.store_all(r.data) -def cache_projects(conduit: Conduit, cache, sts): - r = conduit.project_search(constraints={"maxDepth": 2}) +def cache_projects(conduit: Conduit, cache, sts, project): - r.fetch_all(sts) + def store_projects(r): + r.fetch_all(sts) + + for p in r.data: + # console.log(p.data) + p.save() + + cache.store_all(r.data) + + if project == 'all': + r = conduit.project_search(constraints={"maxDepth": 2}) + store_projects(r) + else: + if not isinstance(project, list): + r = conduit.project_search(constraints={"ancestors": [project]}) + store_projects(r) + project = [project] + r = conduit.project_search(constraints={"phids": project}) + store_projects(r) - for project in r.data: - project.save() - cache.store_all(r.data) @cli.command() @@ -73,6 +88,8 @@ def cache_columns(ctx: typer.Context, project: str = Option("all")): with config.console.status("[bold green]Fetching more pages...") as sts: r.fetch_all(sts) + proxy_phids = [] + with config.console.status( "[bold green]Saving to sqlite..." ) as sts, config.db.conn as conn: @@ -81,6 +98,8 @@ def cache_columns(ctx: typer.Context, project: str = Option("all")): for col in r.data: count += 1 col.save() + if col['proxyPHID']: + proxy_phids.append(col['proxyPHID']) # col.project.save() if round((count / total) * 100) > pct: pct = round((count / total) * 100) @@ -95,7 +114,9 @@ def cache_columns(ctx: typer.Context, project: str = Option("all")): PHObject.resolve_phids(config.phab, cache) with config.console.status("[bold green]Fetching projects") as sts: - cache_projects(config.phab, cache, sts) + cache_projects(config.phab, cache, sts, project) + cache_projects(config.phab, cache, sts, proxy_phids) + @cli.command() @@ -114,6 +135,8 @@ def map( console = config.console project_phid = project + all_projects:set[Project] + try: kvcache, cache = init_caches(db, phab) except sqlite3.OperationalError as err: @@ -149,7 +172,7 @@ def map( "Processing [bold blue]transactions[/bold blue]..." ) as sts: conn # type: Connection - datapoints, metrics, all_metrics, taskids = maptransactions( + datapoints, all_projects, all_metrics, taskids = maptransactions( project_phid, transactions, conn, console, phab, sts, kvcache ) @@ -166,7 +189,12 @@ def map( for metric in all_metrics: if metric.last: metric.end(datetime.now().timestamp()) + last_ended_at = None for span in metric.spans: + if last_ended_at and last_ended_at >= span.start: + span.start = last_ended_at + timedelta(seconds=1) + if span.end <= span.start: + span.end = span.start + timedelta(seconds=1) row = ( metric.task, metric.value, @@ -175,6 +203,7 @@ def map( span.end, span.duration(), ) + last_ended_at = span.end task_metric_rows.append(row) load_data_with_progress( @@ -185,6 +214,11 @@ def map( task_metric_rows, ) + # for proj in all_projects: + # console.log(proj) + # for metric in proj.all_metrics().values(): + # console.log(metric) + with console.status("Updating [bold]phobjects[/bold].") as sts: cache_tasks(config.phab, cache, taskids, sts) PHObject.resolve_phids(config.phab, cache) diff --git a/src/ddd/boardmetrics_mapper.py b/src/ddd/boardmetrics_mapper.py index 5f761a9..9c49d9f 100644 --- a/src/ddd/boardmetrics_mapper.py +++ b/src/ddd/boardmetrics_mapper.py @@ -1,8 +1,9 @@ +import re import sqlite3 from collections import deque from operator import itemgetter from pprint import pprint -from typing import Iterable, Mapping, Tuple +from typing import Iterable, Mapping, Set, Tuple from rich.console import Console from rich.status import Status @@ -14,11 +15,14 @@ from ddd.phobjects import ( KKVCache, PHIDRef, PHObject, + Project, Task, TimeSpanMetric, sqlite_insert_statement, ) +RE_AT_MENTION = re.compile(r'@([\w\-\d]+)[\b:]?', + re.MULTILINE|re.IGNORECASE) def maptransactions( project_phid, @@ -28,7 +32,7 @@ def maptransactions( phab: Conduit, sts: Status, cache: KKVCache, -) -> Tuple[Iterable[Iterable], Iterable[Iterable], Iterable[TimeSpanMetric], Iterable]: +) -> Tuple[Iterable[Iterable], Set[Project], Set[TimeSpanMetric], Iterable]: mapper = PropertyMatcher() # functions decorated with @ttype will take a transaction object and distill @@ -37,6 +41,7 @@ def maptransactions( # ids = [id for id in tasks.keys()] all_metrics = set() + all_projects = set() @mapper("transactionType=core:edge", "meta.edge:type=41") def projects(t, context: Task): @@ -57,14 +62,23 @@ def maptransactions( if default_column: ref = p res.append(("columns", ref, None, default_column)) + if "core.create" in t["meta"]: state = "created" else: state = "tagged" - metric = PHObject.instance(p).metric(task=t["taskID"]).start(ts, state) + proj = PHObject.instance(p) + assert(isinstance(proj, Project)) + context._all_projects.add(proj) + metric = proj.metric(task=t["taskID"]).start(ts, state) all_metrics.add(metric) context.metric(key=p, metric=metric) - + proxyColumn = cache.get_proxy(p) + if proxyColumn and proxyColumn.fromPHID: + parent_project = cache.get_project(proxyColumn.fromPHID) + parent_metric = parent_project.object.metric(task=t['taskID'], key=(context, 'column')) + parent_metric.start(ts, proxyColumn.fromPHID) + context.metric(key=(proxyColumn.fromPHID, 'column'), metric=parent_metric) for p in oldValue: metric = PHObject.instance(p).metric(task=t["taskID"]).end(ts, "untagged") all_metrics.add(metric) @@ -79,7 +93,14 @@ def maptransactions( @mapper("transactionType=core:comment") def comment(t, ctx): - return [("comment", None, t["taskID"], None)] + comments = t['comments'] + if comments: + m = RE_AT_MENTION.findall(comments) + mentions = [['@mention', None, t['taskID'], mention.lower()] for mention in set(m)] + return [("comment", None, None, None), *mentions] + else: + return None + @mapper("transactionType=core:edge", "meta.edge:type=3") def subtask(t, context): @@ -100,15 +121,15 @@ def maptransactions( @mapper("transactionType=status") def status(t, context: Task): ts = int(t["dateCreated"]) - state = t["newValue"] - if state in ("open", "stalled", "progress"): + # console.log('status', task.phid, hash(task), state) + if state in Task.OPEN_STATUS: # for metric in context.metrics(is_started=False): # metric.start(state) context.metric(key="status").start(ts, state) - elif state in ("declined", "resolved", "invalid"): - for metric in context.metrics(is_ended=False): - metric.end(ts, state) + elif state in Task.CLOSED_STATUS: + #for metric in context.metrics(is_ended=False): + # metric.end(ts, state) context.metric(key="status").end(ts, state) return [("status", "global", t["oldValue"], t["newValue"])] @@ -116,12 +137,13 @@ def maptransactions( def assign(t, context): ts = int(t["dateCreated"]) if t["oldValue"]: - context.metric(key="assign").val(t["oldValue"]).end(ts, "reassign") + context.metric(key="assign").val(t["oldValue"]).end(ts-1, "reassign") context.metric(key="assign").val(t["newValue"]).start(ts, "assign") return [("assign", "global", t["oldValue"], t["newValue"])] @mapper("transactionType=core:create") def create(t, context): + context.metric(key="create").val('open').start(ts, 'open') return [("status", "global", None, "open")] @mapper("transactionType=core:columns") @@ -141,16 +163,23 @@ def maptransactions( source = cache.resolve_column(fromcol) ref = obj["boardPHID"] res.append(("columns", ref, fromcol, tocol)) + project = PHObject.instance(ref) + metric = project.metric(task=t["taskID"], key=(context,'column')).start(ts, tocol) + context.metric(key=(ref,tocol), metric=metric) + metric.key = 'column' if source or target: + for i in ("fromPHID", "toPHID"): - PHObject.instance(ref).metric(task=t["taskID"]).start(ts, tocol) srcphid = getattr(source, i, None) tophid = getattr(target, i, None) if srcphid and tophid: - PHObject.instance(ref).metric(task=t["taskID"]).start( + + metric = project.metric(key=(context,'column'), task=t["taskID"]).start( ts, tophid ) + context.metric(key=(ref, tophid), metric=metric) + metric.key = 'column' res.append(("milestone", ref, srcphid, tophid)) return res @@ -206,7 +235,17 @@ def maptransactions( except sqlite3.InterfaceError as e: print(event) continue + + start = task.startofwork() + if start: + all_metrics.add(start) + for metric in task.all_metrics().values(): all_metrics.add(metric) + for proj in task._all_projects: + all_projects.add(proj) + + + con.commit() - return datapoints, metrics, all_metrics, taskids + return datapoints, all_projects, all_metrics, taskids diff --git a/src/ddd/boardmetrics_schema.py b/src/ddd/boardmetrics_schema.py index bae0427..bb0f2c0 100644 --- a/src/ddd/boardmetrics_schema.py +++ b/src/ddd/boardmetrics_schema.py @@ -4,7 +4,8 @@ from ddd.phobjects import sqlite_connect from pathlib import Path from typing import Optional from rich.console import Console -from typer import Option, Context +from typer.params import Option +from typer.models import Context from ddd.phab import Conduit from sqlite_utils.db import Database @@ -38,15 +39,100 @@ class Config: --sql CREATE UNIQUE INDEX IF NOT EXISTS ts_column_task on column_metrics(ts, column, task); --sql - CREATE TABLE IF NOT EXISTS task_metrics(task, metric phid, state, ts datetime, ts2 datetime, duration seconds); + CREATE TABLE IF NOT EXISTS task_metrics(task int, metric phid, state, ts datetime, ts2 datetime, duration seconds); --sql - CREATE UNIQUE INDEX IF NOT EXISTS task_metric ON task_metrics(task, metric, state, ts); + CREATE UNIQUE INDEX IF NOT EXISTS task_metric ON task_metrics(ts, task, metric, state); --sql - CREATE TABLE IF NOT EXISTS events(ts, task, project phid, user phid, event, old, new); + CREATE INDEX IF NOT EXISTS task_metrics_task on task_metrics(task); + --sql + CREATE INDEX IF NOT EXISTS task_metrics_metric on task_metrics(metric); + --sql + CREATE TABLE IF NOT EXISTS events(ts datetime, task int, project phid, user phid, event, old text, new text); --sql CREATE UNIQUE INDEX IF NOT EXISTS events_pk on events(ts, task, project, event, old, new); --sql - CREATE INDEX IF NOT EXISTS events_project on events(event, project, old, new); + CREATE INDEX IF NOT EXISTS events_project on events(project, event); + --sql + CREATE INDEX IF NOT EXISTS events_task on events(task); + --sql + CREATE TABLE IF NOT EXISTS Project ( + name TEXT, + status TEXT, + phid TEXT PRIMARY KEY, + dateCreated DATETIME, + dateModified DATETIME, + id INTEGER, + uri TEXT, + typeName TEXT, + type TEXT, + fullName TEXT, + slug TEXT, + subtype TEXT, + milestone TEXT, + depth INTEGER, + parent phid, + icon TEXT, + color TEXT, + spacePHID phid, + policy TEXT, + description TEXT, + [custom.custom:repository] TEXT, + [custom.sprint:start] TEXT, + [custom.sprint:end] TEXT, + [attachments] TEXT + ); + --sql + CREATE INDEX IF NOT EXISTS Project_parent on Project(parent); + --sql + CREATE INDEX IF NOT EXISTS Project_status on Project(status); + --sql + CREATE TABLE IF NOT EXISTS ProjectColumn ( + name TEXT, + status TEXT, + phid TEXT PRIMARY KEY, + dateCreated DATETIME, + dateModified DATETIME, + proxyPHID phid, + project phid, + isDefaultColumn INTEGER, + policy TEXT, + id INTEGER, + type TEXT, + attachments TEXT + ); + --sql + CREATE INDEX IF NOT EXISTS ProjectColumn_status on ProjectColumn(status); + --sql + CREATE INDEX IF NOT EXISTS ProjectColumn_project on ProjectColumn(project); + --sql + CREATE INDEX IF NOT EXISTS ProjectColumn_proxyPHID on ProjectColumn(proxyPHID); + --sql + CREATE INDEX IF NOT EXISTS ProjectColumn_isDefaultColumn on ProjectColumn(isDefaultColumn); + --sql + DROP VIEW IF EXISTS view_task_cycletime; + --sql + CREATE VIEW IF NOT EXISTS + view_task_cycletime + AS + SELECT + task, + min(start_date) AS start, + max(start_date) AS end, + (max(ts) - min(ts))/3600 AS hours, + (max(ts) - min(ts))/86400 AS days + FROM ( + SELECT + task, + ts, + datetime(ts, 'unixepoch') AS start_date + FROM + task_metrics + WHERE + metric IN ('startofwork', 'endofwork') + ORDER BY + task, metric=='startofwork' DESC + ) + GROUP BY task; --sql DROP VIEW IF EXISTS view_column_metrics; --sql diff --git a/src/ddd/phab.py b/src/ddd/phab.py index 35b57c7..c6aa36b 100644 --- a/src/ddd/phab.py +++ b/src/ddd/phab.py @@ -131,8 +131,15 @@ class Conduit(object): self, queryKey="all", constraints: MutableMapping = {} ) -> Cursor: """Find projects""" + + query = {} + if not constraints or not len(constraints) and queryKey is not None: + query["queryKey"] = queryKey + if len(constraints): + query["constraints"] = constraints + return self.request( - "project.search", {"queryKey": queryKey, "constraints": constraints} + "project.search", query ) def maniphest_search(self, constraints: MutableMapping = {}) -> Cursor: diff --git a/src/ddd/phobjects.py b/src/ddd/phobjects.py index 1734daa..1c0563b 100644 --- a/src/ddd/phobjects.py +++ b/src/ddd/phobjects.py @@ -2,12 +2,12 @@ from __future__ import annotations from abc import abstractproperty from dataclasses import dataclass -from typing import TYPE_CHECKING, Deque +from typing import TYPE_CHECKING, Deque, Hashable, Set if TYPE_CHECKING: from ddd.phab import Conduit -import datetime +from datetime import datetime, timedelta import json import sqlite3 import time @@ -70,8 +70,8 @@ class PhabObjectBase(UserDict, dict): id: int name: str fullName: str - dateCreated: datetime.datetime - dateModified: datetime.datetime + dateCreated: datetime + dateModified: datetime status: Status def __init__(self, phid: PHID, **kwargs): @@ -287,8 +287,8 @@ class PHObject(PhabObjectBase, SubclassCache[PHID, PhabObjectBase]): kwargs.pop("dateModified") if "dateModified" in kwargs else dateCreated ) super().__init__(phid) - self.dateCreated = datetime.datetime.fromtimestamp(dateCreated) - self.dateModified = datetime.datetime.fromtimestamp(dateModified) + self.dateCreated = datetime.fromtimestamp(dateCreated) + self.dateModified = datetime.fromtimestamp(dateModified) def __eq__(self, other: object): if isinstance(other, PhabObjectBase): @@ -300,13 +300,14 @@ class PHObject(PhabObjectBase, SubclassCache[PHID, PhabObjectBase]): return self.dateModified < other.dateModified return NotImplemented - def get_table(self) -> Table: - if hasattr(self.__class__, "table") and self.__class__.table is not None: + def get_table(self, table_name=None) -> Table: + if not table_name and hasattr(self.__class__, "table") and self.__class__.table is not None: return self.__class__.table - - name = self.__class__.__name__ - table = Table(PHObject.db, name, pk="phid", alter=True) - self.__class__.table = table + if not table_name: + table_name = self.__class__.__name__ + table = Table(PHObject.db, table_name, pk="phid", alter=True) + if table_name == self.__class__.__name__: + self.__class__.table = table return table def save(self, data: Optional[MutableMapping] = None): @@ -328,6 +329,9 @@ class PHObject(PhabObjectBase, SubclassCache[PHID, PhabObjectBase]): self.update(data) except NotFoundError as e: console.log(e) + table=self.get_table('phobjects') + data = table.get(self.phid) + self.update(data) return self @@ -383,6 +387,9 @@ class PHObject(PhabObjectBase, SubclassCache[PHID, PhabObjectBase]): return __class__.instances + def __hash__(self): + return hash(self.phid) + class PHObjectWithFields(PHObject): def update(self, *args, **kwds): @@ -397,23 +404,35 @@ class User(PHObject): pass +def dt(ts:Union[datetime, str, int]) -> datetime: + '''Convert to datetime, from a timestamp as int or string''' + if isinstance(ts, datetime): + return ts + elif isinstance(ts, str): + ts = int(ts) + return datetime.fromtimestamp(ts) + +ONE_SECOND = timedelta(seconds=1) + +class NotStartedError(Exception): + """Metric is missing it's start time""" + pass + @dataclass class TimeSpan: - start: datetime.datetime - end: Optional[datetime.datetime] = None - state: Optional[str] = None + start: datetime + end: Optional[datetime] = None + action: Optional[str] = None + state: Optional[str|PHID] = None def duration(self): - - if self.end: - delta = self.end - self.start - else: - delta = datetime.datetime.now() - self.start + end = self.end if self.end else datetime.now() + delta = end - self.start return delta.total_seconds() class TimeSpanMetric: - spans: Deque + spans: Deque[TimeSpan] last: Optional[TimeSpan] key: Optional[str] project: Any @@ -427,10 +446,11 @@ class TimeSpanMetric: self._value = None def start(self, ts, state="open") -> TimeSpanMetric: + ts=dt(ts) if self.last: - self.end(ts - 1) - ts = datetime.datetime.fromtimestamp(ts) + self.end(ts - ONE_SECOND) self.last = TimeSpan(start=ts, state=state) + self.spans.append(self.last) return self def started(self): @@ -442,20 +462,32 @@ class TimeSpanMetric: def empty(self): return self.last is None and len(self.spans) == 0 + def limit(self, maxlen:int): + spans = deque(maxlen = maxlen) + if len(self.spans) > maxlen: + for _ in range(maxlen): + spans.appendleft(self.spans.pop()) + else: + spans.extend(self.spans) + self.spans = spans + return self + def end(self, ts, state=None, implied_start=True) -> TimeSpanMetric: - if not self.last: + ts=dt(ts) + + if (implied_start and state is not None and + (self.last is None or self.last.state != state)): + self.start(ts - ONE_SECOND, state) + elif not self.last: if not implied_start: return self - self.start(ts - 1) + self.start(ts - ONE_SECOND) assert self.last is not None - if isinstance(ts, str): - ts = int(ts) - ts = datetime.datetime.fromtimestamp(ts) + self.last.end = ts if state: self.last.state = state - self.spans.append(self.last) self.last = None return self @@ -469,6 +501,10 @@ class TimeSpanMetric: def value(self): return self.key if self._value is None else self._value + @property + def state(self): + return self.last.state if self.last else self.spans[-1:].state + @value.setter def value(self, v): self._value = v @@ -484,20 +520,23 @@ class TimeSpanMetric: self._task = val def __repr__(self) -> str: - return f"TimeSpanMetric(t={self.task} p={self.project} last={self.last} spans={self.spans})" + return f"TimeSpanMetric(k={self.key} t={self._task} p={self.project} v='{self._value}' last={self.last})" class MetricsMixin: _metrics: defaultdict[Any, TimeSpanMetric] phid: PHID - def metrics(self, is_started=True, is_ended=True): + def metrics(self, is_started=None, is_ended=None) -> Set[TimeSpanMetric]: """Retrieve metrics, optionally filtered by their state""" - return { - k: v - for k, v in self._metrics.items() - if ((v.last is None == is_ended) or (v.last is not None == is_started)) - } + res = set() + for v in self._metrics.values(): + if is_ended is not None and is_ended != v.ended(): + continue + if is_started is not None and is_started != v.started(): + continue + res.add(v) + return res def close(self, ts, state=None): """Record the end of a metric's time period""" @@ -540,8 +579,10 @@ class Project(PHObjectWithFields, MetricsMixin): self._metrics = defaultdict(TimeSpanMetric) self._metrics["project"] = self["phid"] - def metric(self, task) -> TimeSpanMetric: - return super().metric(key=task, task=task, value=self.phid) + def metric(self, task, key=None) -> TimeSpanMetric: + if not key: + key = task + return super().metric(key=key, task=task, value=self.phid, project=self) def default_column(self) -> ProjectColumn: if self._default: @@ -583,15 +624,46 @@ class ProjectColumn(PHObjectWithFields): class Task(PHObjectWithFields, MetricsMixin): + OPEN_STATUS = ("open", "stalled", "progress") + CLOSED_STATUS = ("declined", "resolved", "invalid", "duplicate") + _all_projects:Set[Project] = set() + def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) self._metrics = defaultdict(TimeSpanMetric) def metric( - self, key: str, metric: Optional[TimeSpanMetric] = None + self, key: Hashable, metric: Optional[TimeSpanMetric] = None ) -> TimeSpanMetric: - return super().metric(key=key, metric=metric, task=self.phid) + metric = super().metric(key=key, metric=metric, task=self.phid) + return metric + def startofwork(self) -> Optional[TimeSpanMetric]: + now = datetime.now() + start = now + state = None + startofwork = self.metric('startofwork') + endofwork = self.metric('endofwork').limit(1) + for key, metric in self._metrics.items(): + start_metric = None + if key == 'status': + for span in metric.spans: + if span.state in Task.CLOSED_STATUS: + endofwork.start(span.start, span.state) + elif span.state == 'progress': + start = span.start + state = span.state + break + elif metric.key == 'column' and metric.spans[0].start < start: + metric.key = key + start = metric.spans[0].start + state = metric.spans[0].state + + if start is not now: + metric = startofwork.start(start, state) + return metric + else: + return None class Commit(PHObject): pass @@ -675,16 +747,21 @@ class PHIDTypes(Enum): class PHIDRef(object): - __slots__ = ("fromPHID", "toPHID", "object", "relation") + __slots__ = ("fromPHID", "toPHID", "toPHIDRef", "object", "relation") fromPHID: Optional[PHID] toPHID: PHID + toPHIDRef: PHIDRef object: PhabObjectBase relation: Optional[PhabObjectBase] - def __init__(self, toPHID: PHID, fromPHID: Optional[PHID] = None): + def __init__(self, toPHID: Union[PHID, PHIDRef], fromPHID: Optional[PHID] = None): if isinstance(toPHID, PHIDRef): - self.toPHID = toPHID.toPHID + if toPHID.fromPHID: + self.toPHID = toPHID.fromPHID + else: + self.toPHID = toPHID.toPHID + self.toPHIDRef = toPHID else: self.toPHID = toPHID self.fromPHID = fromPHID @@ -694,6 +771,9 @@ class PHIDRef(object): else: self.relation = None + def target(self): + return self.toPHIDRef if self.toPHIDRef else PHIDRef(self.toPHID) + def __repr__(self) -> str: if self.fromPHID: return f"PHIDRef( from {self.fromPHID} => to {self.toPHID})" @@ -761,7 +841,7 @@ def sqlite_insert_statement( return f"{statement} into {table}({','.join(keys)}) values ({placeholders});" -def adapt_datetime(ts: datetime.datetime) -> float: +def adapt_datetime(ts: datetime) -> float: return ts.timestamp() @@ -775,7 +855,7 @@ def register_sqlite_adaptors(): sqlite3.register_adapter(list, sql_encode_json) sqlite3.register_adapter(tuple, sql_encode_json) sqlite3.register_converter("phid", sql_convert_phid) - sqlite3.register_adapter(datetime.datetime, adapt_datetime) + sqlite3.register_adapter(datetime, adapt_datetime) _registered_adapters = True from sqlite_utils.db import COLUMN_TYPE_MAPPING @@ -829,7 +909,7 @@ class PHObjectEncoder(json.JSONEncoder): if isinstance(o, PHIDRef): return o.toPHID - if isinstance(o, datetime.datetime): + if isinstance(o, datetime): return o.isoformat() if isinstance(o, sqlite3.Row): return [i for i in o] @@ -884,8 +964,12 @@ class KKVCache(object): self.cache[key] = PHIDRef(row["project_phid"], row["column_phid"]) if isPHID(row["proxyPHID"]): column_proxy = PHIDRef(row["proxyPHID"], row["column_phid"]) + # forward mapping from parent column to milestone subproject key = (row["column_phid"], "proxyPHID") self.cache[key] = column_proxy + # reverse mapping from milestone project to parent proxy column + key = (row["proxyPHID"], "proxyPHID") + self.cache[key] = column_proxy def get(self, *args, default=None): return self.cache.get(args, default) @@ -893,9 +977,9 @@ class KKVCache(object): def get_project(self, obj): return self.get(obj, "project_phid") - def get_proxy(self, parent_column): + def get_proxy(self, column_or_milestone_phid): """Get the project that a proxy column points to""" - return self.get(parent_column, "proxyPHID") + return self.get(column_or_milestone_phid, "proxyPHID") def get_default_column(self, project_phid): key = (project_phid, "default_column") @@ -915,9 +999,12 @@ class KKVCache(object): if ptype is ProjectColumn: milestone_project = self.get_proxy(phid) if milestone_project: - res = self.get_default_column(milestone_project) - if not res: - res = PHIDRef(milestone_project) + default_col = self.get_default_column(milestone_project) + if isinstance(default_col, PHIDRef): + res = PHIDRef(default_col, phid) + else: + res = milestone_project + elif ptype is Project: res = self.get_default_column(phid) @@ -931,7 +1018,7 @@ class types: PRIMARYKEY = SQLType("PRIMARYKEY", str, "PRIMARY KEY") REAL = SQLType("REAL", float) TEXT = SQLType("TEXT", str) - timestamp = SQLType("timestamp", datetime.datetime) + timestamp = SQLType("timestamp", datetime) class DataCache(object): diff --git a/typings/datasette/__init__.pyi b/typings/datasette/__init__.pyi index 8baca0d..1af73f5 100644 --- a/typings/datasette/__init__.pyi +++ b/typings/datasette/__init__.pyi @@ -3,7 +3,7 @@ This type stub file was generated by pyright. """ from datasette.version import __version__ as __version__ -from datasette.version __version_info__ as __version_info__ +from datasette.version import __version_info__ as __version_info__ from .hookspecs import hookimpl as hookimpl from .hookspecs import hookspec as hookspec diff --git a/typings/datasette/app.pyi b/typings/datasette/app.pyi index 29d5010..ff744b0 100644 --- a/typings/datasette/app.pyi +++ b/typings/datasette/app.pyi @@ -2,7 +2,7 @@ This type stub file was generated by pyright. """ -from datasette.database import Database +from datasette.database import Database, Results from .utils.asgi import NotFound app_root = ... @@ -84,10 +84,10 @@ class Datasette: """Check permissions using the permissions_allowed plugin hook""" ... - async def execute(self, db_name, sql, params=..., truncate=..., custom_time_limit=..., page_size=..., log_sql_errors=...): + async def execute(self, db_name:str, sql:str, params=..., truncate=..., custom_time_limit=..., page_size=..., log_sql_errors=...) -> Results: ... - async def expand_foreign_keys(self, database, table, column, values): # -> dict[tuple[Unknown, Unknown], str]: + async def expand_foreign_keys(self, database:str, table:str, column:str, values): # -> dict[tuple[str, Any], str]: """Returns dict mapping (column, value) -> label""" ... diff --git a/typings/datasette/database.pyi b/typings/datasette/database.pyi index 5da8a50..e42ca27 100644 --- a/typings/datasette/database.pyi +++ b/typings/datasette/database.pyi @@ -2,98 +2,104 @@ This type stub file was generated by pyright. """ +from sqlite3 import Connection, Row +from typing import Iterator, List, Optional, Sequence +from uuid import UUID + connections = ... AttachedDatabase = ... class Database: + conn:Connection + def __init__(self, ds, path=..., is_mutable=..., is_memory=..., memory_name=...) -> None: ... - + @property def cached_table_counts(self): # -> dict[Unknown, Unknown] | None: ... - + def suggest_name(self): # -> str | Unknown: ... - - def connect(self, write=...): # -> Connection: + + def connect(self, write=...) -> Connection: ... - - async def execute_write(self, sql, params=..., block=...): # -> UUID: + + async def execute_write(self, sql, params=..., block=...) -> UUID: ... - - async def execute_write_fn(self, fn, block=...): # -> UUID: + + async def execute_write_fn(self, fn, block=...) -> UUID: ... - + async def execute_fn(self, fn): ... - + async def execute(self, sql, params=..., truncate=..., custom_time_limit=..., page_size=..., log_sql_errors=...): """Executes sql against db_name in a thread""" ... - + @property - def size(self): # -> int: + def size(self) -> int: ... - + async def table_counts(self, limit=...): # -> dict[Unknown, Unknown]: ... - + @property - def mtime_ns(self): # -> int | None: + def mtime_ns(self) -> int | None: ... - - async def attached_databases(self): # -> list[AttachedDatabase]: + + async def attached_databases(self) -> list[AttachedDatabase]: ... - - async def table_exists(self, table): # -> bool: + + async def table_exists(self, table) -> bool: ... - - async def table_names(self): # -> list[Unknown]: + + async def table_names(self) -> Sequence[str]: ... - - async def table_columns(self, table): + + async def table_columns(self, table) -> Sequence[str]: ... - + async def table_column_details(self, table): ... - + async def primary_keys(self, table): ... - + async def fts_table(self, table): ... - - async def label_column_for_table(self, table): # -> None: + + async def label_column_for_table(self, table) -> None: ... - + async def foreign_keys_for_table(self, table): ... - - async def hidden_table_names(self): # -> list[Unknown]: + + async def hidden_table_names(self) -> Sequence[str]: ... - - async def view_names(self): # -> list[Unknown]: + + async def view_names(self) -> Sequence[str]: ... - + async def get_all_foreign_keys(self): ... - + async def get_table_definition(self, table, type_=...): # -> str | None: ... - + async def get_view_definition(self, view): # -> str | None: ... - - def __repr__(self): # -> str: + + def __repr__(self) -> str: ... - + class WriteTask: __slots__ = ... def __init__(self, fn, task_id, reply_queue) -> None: ... - + class QueryInterrupted(Exception): @@ -105,24 +111,29 @@ class MultipleValues(Exception): class Results: - def __init__(self, rows, truncated, description) -> None: + + rows:Sequence[Row] = ... + truncated:bool = ... + description:Sequence[Sequence[str]] = ... + + def __init__(self, rows:Sequence[Row], truncated:bool, description:Sequence[Sequence[str]]) -> None: ... - + @property - def columns(self): # -> list[Unknown]: + def columns(self) -> list[str]: ... - - def first(self): # -> None: + + def first(self) -> Optional[Row]: ... - + def single_value(self): ... - - def __iter__(self): # -> Iterator[Unknown]: + + def __iter__(self) -> Iterator[Row]: ... - - def __len__(self): # -> int: + + def __len__(self) -> int: ... - + diff --git a/www/metadata-wip.yaml b/www/metadata-wip.yaml new file mode 100644 index 0000000..f3b85a9 --- /dev/null +++ b/www/metadata-wip.yaml @@ -0,0 +1,392 @@ +# some work in progress charts + + task-states: + title: Task States + db: metrics + query: + SELECT + task, + '/-/dashboards/task-metrics?task_id='||task as url, + t.name, + state, + metric, + duration, + datetime(ts, 'unixepoch') as start, + datetime(ts2, 'unixepoch') as end, + cume_dist() over win as row_num + FROM + task_metrics m + left join task t on m.task = t.id + where TRUE + [[ AND metric=:project ]] + AND ( + [[ end > :date_start ]] + [[ AND end < :date_end ]] + ) OR ( + [[ start > :date_start ]] + [[ AND start < :date_end ]] + ) + WINDOW win as (ORDER BY ts,ts2) + ORDER BY + task, end + library: vega + display: + { + "$schema": "https://vega.github.io/schema/vega/v5.json", + "description": "Task States", + "autosize": { "type": "fit-x", "contains": "padding" }, + "background": "#00000000", + "padding": 5, + "style": "cell", + "encode": { "update": { "stroke": { "value": null } } }, + "data": + [ + { + "name": "data", + "url": "/metrics.json?_shape=objects&sql=SELECT%20task%2C%20'%2F-%2Fdashboards%2Ftask-metrics%3Ftask_id%3D'%7C%7Ctask%20as%20url%2C%20t.name%2C%20state%2C%20metric%2C%20duration%2C%20datetime(ts%2C%20'unixepoch')%20as%20start%2C%20datetime(ts2%2C%20'unixepoch')%20as%20end%2C%20cume_dist()%20over%20win%20as%20row_num%20FROM%20task_metrics%20m%20left%20join%20task%20t%20on%20m.task%20%3D%20t.id%20where%20TRUE%20%20AND%20metric%3D%3Aproject%20%20AND%20(%20%20end%20%3E%20%3Adate_start%20%20%20AND%20end%20%3C%20%3Adate_end%20%20)%20OR%20(%20%20start%20%3E%20%3Adate_start%20%20%20AND%20start%20%3C%20%3Adate_end%20%20)%20WINDOW%20win%20as%20(ORDER%20BY%20ts%2Cts2)%20ORDER%20BY%20task%2C%20end&project=PHID-PROJ-2b7oz62ylk3jk4aus262&date_end=2021-10-15&date_start=2019-01-01", + "format": + { + "type": "json", + "property": "rows", + "parse": { "start": "date", "end": "date" }, + }, + }, + { + "name": "data_0", + "source": "data", + "transform": + [ + { + "type": "filter", + "expr": '(isDate(datum["start"]) || (isValid(datum["start"]) && isFinite(+datum["start"])))', + }, + ], + }, + { + "name": "data_1", + "source": "data", + "transform": + [ + { + "type": "filter", + "expr": '(isDate(datum["start"]) || (isValid(datum["start"]) && isFinite(+datum["start"]))) && isValid(datum["duration"]) && isFinite(+datum["duration"])', + }, + ], + }, + ], + "signals": + [ + { + "name": "width", + "init": "isFinite(containerSize()[0]) ? containerSize()[0] : 200", + "on": + [ + { + "update": "isFinite(containerSize()[0]) ? containerSize()[0] : 200", + "events": "window:resize", + }, + ], + }, + { "name": "y_step", "value": 20 }, + { + "name": "height", + "update": "bandspace(domain('y').length, 0, 0) * y_step", + }, + ], + "marks": + [ + { + "name": "layer_0_marks", + "type": "rect", + "clip": true, + "style": ["rect"], + "from": { "data": "data_0" }, + "encode": + { + "update": + { + "opacity": { "value": 1 }, + "stroke": { "value": { "value": "black" } }, + "tooltip": + { + "signal": '{"start": timeFormat(datum["start"], ''%b %d, %Y''), "end": timeFormat(datum["end"], ''%b %d, %Y''), "task": isValid(datum["task"]) ? datum["task"] : ""+datum["task"], "state": isValid(datum["state"]) ? datum["state"] : ""+datum["state"], "name": isValid(datum["name"]) ? datum["name"] : ""+datum["name"]}', + }, + "fill": { "scale": "color", "field": "state" }, + "description": + { + "signal": 'isValid(datum["name"]) ? datum["name"] : ""+datum["name"]', + }, + "x": { "scale": "x", "field": "start" }, + "x2": { "scale": "x", "field": "end" }, + "y": { "scale": "y", "field": "task" }, + "height": { "scale": "y", "band": 1 }, + }, + }, + }, + { + "name": "layer_1_marks", + "type": "text", + "style": ["text"], + "from": { "data": "data_0" }, + "encode": + { + "update": + { + "align": { "value": "left" }, + "dx": { "value": 10 }, + "limit": { "value": 200 }, + "cursor": { "value": "pointer" }, + "fill": { "value": "white" }, + "href": + { + "signal": 'isValid(datum["url"]) ? datum["url"] : ""+datum["url"]', + }, + "description": + { + "signal": '"start: " + (timeFormat(datum["start"], ''%b %d, %Y'')) + "; task: " + (isValid(datum["task"]) ? datum["task"] : ""+datum["task"]) + "; url: " + (isValid(datum["url"]) ? datum["url"] : ""+datum["url"]) + "; state: " + (isValid(datum["state"]) ? datum["state"] : ""+datum["state"])', + }, + "x": { "scale": "x", "field": "start" }, + "y": { "scale": "y", "field": "task", "band": 0.5 }, + "text": + { + "signal": 'isValid(datum["state"]) ? datum["state"] : ""+datum["state"]', + }, + "baseline": { "value": "middle" }, + }, + }, + }, + { + "name": "layer_2_marks", + "type": "symbol", + "style": ["point"], + "from": { "data": "data_1" }, + "encode": + { + "update": + { + "opacity": { "value": 0.7 }, + "fill": { "value": "transparent" }, + "stroke": { "value": "#4c78a8" }, + "ariaRoleDescription": { "value": "point" }, + "description": + { + "signal": '"start: " + (timeFormat(datum["start"], ''%b %d, %Y'')) + "; task: " + (isValid(datum["task"]) ? datum["task"] : ""+datum["task"]) + "; duration: " + (format(datum["duration"], ""))', + }, + "x": { "scale": "x", "field": "start" }, + "y": { "scale": "y", "field": "task", "band": 0.5 }, + "size": { "scale": "size", "field": "duration" }, + }, + }, + }, + ], + "scales": + [ + { + "name": "x", + "type": "time", + "domain": + { + "fields": + [ + { "data": "data_0", "field": "start" }, + { "data": "data_0", "field": "end" }, + { "data": "data_1", "field": "start" }, + ], + }, + "range": [0, { "signal": "width" }], + }, + { + "name": "y", + "type": "band", + "domain": + { + "fields": + [ + { "data": "data_0", "field": "task" }, + { "data": "data_1", "field": "task" }, + ], + "sort": true, + }, + "range": { "step": { "signal": "y_step" } }, + "paddingInner": 0, + "paddingOuter": 0, + }, + { + "name": "color", + "type": "ordinal", + "domain": { "data": "data_0", "field": "state", "sort": true }, + "range": "category", + }, + { + "name": "size", + "type": "linear", + "domain": { "data": "data_1", "field": "duration" }, + "range": [0, 361], + "zero": true, + }, + ], + "axes": + [ + { + "scale": "x", + "orient": "bottom", + "gridScale": "y", + "grid": true, + "tickCount": { "signal": "ceil(width/40)" }, + "domain": false, + "labels": false, + "aria": false, + "maxExtent": 0, + "minExtent": 0, + "ticks": false, + "zindex": 0, + }, + { + "scale": "x", + "orient": "bottom", + "grid": false, + "title": "start, end", + "labelFlush": true, + "labelOverlap": true, + "tickCount": { "signal": "ceil(width/40)" }, + "zindex": 0, + }, + { + "scale": "y", + "orient": "left", + "grid": false, + "title": "task", + "zindex": 1, + }, + ], + "legends": + [ + { + "fill": "color", + "symbolType": "square", + "title": "state", + "encode": + { + "symbols": + { + "update": + { + "stroke": { "value": { "value": "black" } }, + "opacity": { "value": 1 }, + }, + }, + }, + }, + { + "size": "size", + "symbolType": "circle", + "title": "duration", + "encode": + { + "symbols": + { + "update": + { + "fill": { "value": "transparent" }, + "stroke": { "value": "#4c78a8" }, + "opacity": { "value": 0.7 }, + }, + }, + }, + }, + ], + "config": { "style": { "arc": { "innerRadius": 50 } } }, + } + + + project-tasks-state: + title: Number of tasks in the project weekly + db: metrics + query: + select * from ( + SELECT + task, + state, + (SELECT name FROM phobjects WHERE phid=m.state) AS name, + metric, + p.name AS project, + sum(duration/(60*60*24*7))/count(distinct task) AS days, + count(*) AS transactions, + count( task) as task_count, + min(datetime(ts,'unixepoch')) AS start, + max(datetime(ts2, 'unixepoch')) AS end, + date(w.date) AS week + FROM + task_metrics m + JOIN Project p ON ( + m.metric = p.phid + [[ AND m.metric = :project ]] + ) + JOIN weeks w + ON date(w.date) >= date(m.ts, 'unixepoch') + AND date(w.date) <= date(m.ts2, 'unixepoch') + GROUP BY week + ) + WHERE TRUE + [[ AND week > :date_start ]] + [[ AND week <= :date_end ]] + ORDER BY + week + library: vega + display: + layer: + - mark: + type: area + tooltip: true + encoding: + x: + field: week + type: temporal + #timeUnit: week + y: + field: task_count + type: quantitative + aggregate: sum + stack: zero + color: + field: state + type: nominal + tooltip: [ + {field: task_count, aggregate: sum, type: quantitative, title: "Task count"}, + {field: name, type: nominal, title: "Name"}, + {field: days, type: quantitative, title: "Average task duration (days)"}, + ] + - mark: + type: line + stroke: black + strokeWidth: 1 + opacity: 0.8 + tooltip: true + encoding: + x: + field: week + type: temporal + #timeUnit: week + y: + field: days + type: quantitative + title: Average task age + scale: + rangeMax: 100 + size: + field: days + type: quantitative + resolve: + scale: + y: independent + # - + # mark: { + # type: point, + # tooltip: true + # } + # encoding: { + # color: { field: state, type: ordinal, scale: { scheme: 'category20' }}, + # y: { field: task_count, type: quantitative, aggregate: sum}, + # x: { field: week, type: temporal }, + # #x2: { field: end, type: temporal } + # } \ No newline at end of file diff --git a/www/metadata.yaml b/www/metadata.yaml index 902a3ae..33edbe9 100644 --- a/www/metadata.yaml +++ b/www/metadata.yaml @@ -9,44 +9,15 @@ plugins: title: Data³ - workflow metrics description: Metrics about projects, tasks and workflows layout: - - [ project-navigation] - - [ project-columns ] - - [ all_events ] - - [ column-metrics ] - - [ waterfall ] - - [ project-tasks-state ] - - [ task-states ] - + - [ project-navigation project-columns project-columns project-columns] + - [ column-metrics column-metrics column-metrics column-metrics] + - [ states states burnup burnup ] + - [ all-events all-events all-events all-events] filters: project: name: Project type: select - query: - WITH project_events as ( - SELECT * FROM ( - SELECT - DISTINCT project, - count(*) AS cnt - FROM - events e - WHERE - e.project LIKE "PHID%" - GROUP BY - project - ORDER BY cnt desc - LIMIT 150 - ) - ) - SELECT - p.phid AS key, - p.name AS label, - p.slug AS slug, - p.uri as uri - FROM Project p - JOIN project_events e - ON e.project=p.phid - WHERE p.depth<2 and not p.name like 'acl*%' - ORDER BY label + query: project_tree task: name: Task ID type: text @@ -57,9 +28,9 @@ plugins: date_end: name: Date End type: date + default: now charts: project-navigation: - title: Projects db: metrics query: project_tree library: jinja @@ -68,21 +39,26 @@ plugins: project-columns: db: metrics query: |- - select + SELECT c.*, p.status as projectStatus, p.uri as projectURI FROM ProjectColumn c - LEFT JOIN Project p on c.proxyPHID=p.phid where c.project=:project and c.status=0 and p.status != 'closed' + LEFT JOIN + Project p on c.proxyPHID=p.phid + WHERE c.project=:project + AND cast(c.status as int) = 0 + AND p.status != 'closed' library: jinja template: project_columns.html visible-when: :project column-metrics: - title: Tasks per column + title: Tasks added & removed from columns, weekly. db: metrics query: select - * + *, + sum(value) as total_tasks from ( SELECT @@ -100,25 +76,24 @@ plugins: p.project_name as project_name, p.project_name||':'||p.column_name AS qualified_name, p.status as column_hidden, - c.value as value, - sum(c.value) over (partition by column_phid order by ts RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW) as total_tasks - + c.value as value FROM column_metrics c join columns p on c.column = p.column_phid - WHERE project_phid=:project + [[ WHERE project_phid=:project ]] order by ts ) where column_hidden = 0 - AND total_tasks > 0 [[ AND project_phid=:project ]] [[ AND ts >= date(:date_start) ]] [[ AND ts <= date(:date_end) ]] + group by week, column_phid library: vega display: + autosize: { "type": "fit-x", "contains": "padding" } mark: - type: area + type: bar tooltip: true baseline: bottom line: true @@ -126,235 +101,176 @@ plugins: y: field: total_tasks type: quantitative - aggregate: median + aggregate: sum + scale: + type: symlog + constant: 1 x: - field: week + field: ts type: temporal - timeunit: month + #timeUnit: week color: - field: column_name + field: qualified_name type: nominal scale: scheme: "category20b" - task-states: - title: Task States + + burnup: + title: Burnup db: metrics query: - SELECT - task, - '/metrics/Task?_sort=phid&id__exact='||task as url, - t.name, - state, - metric, - duration, - datetime(ts, 'unixepoch') as start, - datetime(ts2, 'unixepoch') as end + select sum(value) as val, * from (SELECT + CASE + WHEN w.date >= date(m.ts2, 'unixepoch') AND state='untagged' THEN 0 + ELSE count(task) END AS value, + coalesce(ph.name, state) as state, + state as realstate, + w.date as week FROM - task_metrics m - left join task t on m.task = t.id - where TRUE - [[ AND metric=:project ]] - AND ( - [[ end > :date_start ]] - [[ AND end < :date_end ]] - ) OR ( - [[ start > :date_start ]] - [[ AND start < :date_end ]] - ) - ORDER BY - task, start, end + task_metrics m + JOIN + weeks w on date(w.date) >= date(m.ts, 'unixepoch') /*and w.date <= date(m.ts2, 'unixepoch')*/ + + LEFT JOIN + phobjects ph + ON (m.state=ph.phid) + [[ WHERE m.metric=:project ]] + GROUP BY week, state + ORDER BY week, state, value + ) + WHERE TRUE + [[ AND week >= date(:date_start) ]] + [[ AND week <= date(:date_end)]] + group by week, state + order by week desc + library: vega + display: + height: 450 + mark: { type: area, tooltip: true, clip: true} + encoding: + x: { field: week, type: temporal,scale: { clamp: true, nice: week, zero: false, padding: 0} } + y: { field: val, type: quantitative, scale: {type: pow, exponent: 3} } + #row: { field: task, type: ordinal,} + color: { field: state, scale: { scheme: 'category20' }} + states: + title: Status changes + db: metrics + query: + select + datetime(ts, 'unixepoch') as time, + date(ts, 'unixepoch', 'start of month') as month, + task, + project, + user, + event, + old, + new, + old||'->'||new as old_new, + count(*) as count + from + events e + where true + [[AND cast(e.task as INTEGER) in (select distinct cast(task as INTEGER) as task from task_metrics where metric=:project)]] + and old_new is not null + and event in ('status') + [[AND month >= :date_start ]] + [[AND month <= :date_end ]] + group by month, old_new + order by month + library: vega display: - height: { step: 30 } - encoding: { - x: { - field: start, - type: temporal, - timeUnit: day - }, - x2: { field: end }, - } - layer: [ - { - mark: { - type: bar, - tooltip: true, - clip: true, - stroke: { value: 'black' }, - opacity: 1 - }, + height: 350 + + encoding: + x: { field: month, type: temporal} + y: { field: count, type: quantitative, scale: { type: symlog, constant: 1 }} + color: { field: old_new, type: nominal, scale: { scheme: 'category20' }} + layer: + - mark: { type: point, tooltip: true } encoding: - { - y: { field: task, type: nominal }, - description: { field: name, type: nominal }, - color: { field: state, type: ordinal }, - } - }, - { - mark: { - type: text, - align: left, - dx: 10, - limit: 200 - }, - encoding: { - y: {field: task, type: nominal }, - text: { field: state}, - color: { value: white}, - href: {field: url, type: nominal} - } + y: { field: count, type: quantitative } + - mark: { type: bar, "size": 20, tooltip: true } + encoding: + y: { field: count, type: quantitative, aggregate: mean} + - mark: { type: errorband, extent: ci, opacity: 0.1 } + encoding: + y: { field: count, type: quantitative, aggregate: mean} + + - } - ] - project-tasks-state: - title: Number of tasks in the project weekly + all-events: + title: 'All events' db: metrics - query: - SELECT - task, - state, - (select name from phobjects where phid=m.state) as name, - metric, - p.name as project, - sum(duration)/(60*60*24) as days, - count(*) as transactions, - count(distinct task) as task_count, - min(datetime(ts,'unixepoch')) as start, - max(datetime(ts2, 'unixepoch')) as end, - date(w.date) as week - FROM - task_metrics m - JOIN Project p ON ( - m.metric = p.phid - AND m.metric = [[ :project ]] - ) - join weeks w on date(w.date) >= date(m.ts, 'unixepoch') and date(w.date) <= date(m.ts2, 'unixepoch') - group by task,week - ORDER BY - start desc library: vega + query: all_events display: + height: 450 + transform: + - window: + - op: sum + field: value + as: Cumulative Sum + frame: [null, 0] + layer: - mark: - type: area - tooltip: true + type: circle encoding: x: - field: week + field: start type: temporal - timeUnit: week - #x2: - #field: w - #timeUnit: week y: - field: task_count + field: value type: quantitative - aggregate: sum + scale: { type: symlog, constant: 1 } color: field: name type: nominal - # - - # mark: { - # type: point, - # tooltip: true - # } - # encoding: { - # color: { field: state, type: ordinal, scale: { scheme: 'category20' }}, - # y: { field: task_count, type: quantitative, aggregate: sum}, - # x: { field: week, type: temporal }, - # #x2: { field: end, type: temporal } - # } - waterfall: - title: Burnup - db: metrics - query: - select * from (SELECT - task, - case state when 'untagged' then -count(distinct task) else count(distinct task) end AS value, - state, - metric, p.name as project, - w.date as week, - duration - FROM - task_metrics m - JOIN - weeks w on w.date >= date(m.ts, 'unixepoch') and w.date <= date(m.ts2, 'unixepoch') - [[ AND w.date >= date(:date_start) ]] - [[ AND w.date <= date(:date_end) ]] - JOIN - Project p - ON (m.metric=p.phid ) - where [[ m.metric=:project ]] - group by week, state, metric - ORDER BY week) - library: vega - display: - #width: 100 - mark: { type: area,tooltip: true} - encoding: - x: { field: week, type: temporal, timeUnit: week} - y: { field: value, type: quantitative } - #column: { field: state, type: ordinal,} - color: { field: state, scale: { scheme: 'category20' }} - all_events: - title: 'All events' - db: metrics - library: vega - query: all_events - display: - # transform: - # - pivot: "event" - # groupby: ['phid'] - # value: value - encoding: - color: - field: name - type: nominal - tooltip: [ - {field: start, type: temporal, title: 'Week' }, - {field: value, type: "quantitative", title: "Tasks"}, - #{field: tasks, type: "quantitative", title: 'Task Cumulative'}, - {field: name, type: "nominal", title: "Column / Milestone"}, - {field: task, type: "nominal", title: "Task"}, - {field: event, type: "nominal", title: "Event"}, - ] - layer: - - mark: - type: area - tooltip: true - interpolate: linear - point: true - encoding: - x: - field: start - type: temporal - timeUnit: week - #bin: true - y: - field: value - type: quantitative - stack: normalize - aggregate: sum - - mark: - type: rect - tooltip: true - encoding: - x: - field: start - type: temporal - timeUnit: week - # bin: true - x2: - field: end - type: temporal - timeUnit: week - y: - field: value - type: quantitative - # y2: - # field: value - # type: quantitative - #impute: {method: 'median'} + size: + field: value + type: quantitative + + #row: + # field: name + # type: nominal + tooltip: [ + {field: start, type: temporal, title: 'Week' }, + {field: value, type: "quantitative", title: "Tasks"}, + #{field: tasks, type: "quantitative", title: 'Task Cumulative'}, + {field: name, type: "nominal", title: "Column / Milestone"}, + #{field: task, type: "nominal", title: "Task"}, + ] + - mark: + type: line + encoding: + y: + field: Cumulative Sum + type: quantitative + scale: { type: symlog, constant: 1 } + x: + field: start + type: temporal + color: + field: name + type: nominal + - mark: + type: errorband + extent: ci + borders: true + encoding: + y: + field: Cumulative Sum + type: quantitative + title: Cumulative Sum + scale: { type: symlog, constant: 1, zero: false } + x: + field: start + type: temporal + color: + field: name + type: nominal + task-metrics: title: Data³ - task metrics description: Metrics about a task @@ -434,10 +350,14 @@ plugins: library: jinja template: task_users.html visible-when: :task_id + test-jinja: + title: Jinja test + db: metrics + query: SELECT * from task_metrics where task=:task_id + library: jinja + template: test.html - - - + # visible-when: :task_id databases: metrics: @@ -450,19 +370,61 @@ databases: sql: |- "select c.project_name, c.column_name as new_column, d.column_name as old_column, count(distinct task) as count, date(ts, 'unixepoch', 'start of month') as month, datetime(ts, 'unixepoch') as timestamp, task, project, user, event, old, new from events e join columns c on e.new = c.column_phid join columns d on e.old=d.column_phid where event = 'columns' group by old, new, month order by count desc limit 50" project_tree: - SELECT - phid, - parent, - name, - slug, - uri - FROM - Project - WHERE - status='open' - AND depth=0 - AND subtype='default' - [[ AND :project in (phid, parent) ]] + title: Projects + sql: + select distinct phid AS key, + phid AS phid, + coalesce(parent,phid) AS parentphid, + depth, + fullName AS label, + name AS name, + slug, + uri FROM Project p + JOIN ( + SELECT + t.metric, + count(*) AS count + FROM + task_metrics t + WHERE + t.metric LIKE 'PHID-PROJ%' + GROUP BY + t.metric + ORDER BY count desc + ) AS t + ON + p.phid=t.metric + WHERE + depth < 5 + AND status='open' + AND subtype='default' + AND name NOT LIKE 'acl%' + [[ AND :project=parentphid ]] + ORDER BY parentphid, depth + # SELECT + # phid AS key, + # phid AS phid, + # coalesce(parent,phid) AS parentphid, + # depth, + # fullName AS label, + # name AS name, + # slug, + # uri + # FROM + # Project + # WHERE + # status='open' + # AND subtype='default' + # AND name NOT LIKE 'acl%' + # AND ( + # (icon LIKE '%group%' + # OR name LIKE '%workboard%' + # OR name LIKE '%kanban%' + # OR name LIKE '%planning%' + # OR name LIKE '%backlog%') + # [[ AND :project=parentphid ]] + # ) + # ORDER BY parentphid, depth proxy_phids: SELECT proxy_phid from ProjectColumn where project=:project project_events: @@ -495,6 +457,7 @@ databases: WHERE event in ('columns', 'projects') and :project in (project, old, new) GROUP BY month order by ts + task_details: title: basic task details sql: |- @@ -502,8 +465,8 @@ databases: UPPER(SUBSTR(task.name, 1, 1)) || SUBSTR(task.name, 2) as name, json_extract(data, '$.uri') AS uri, json_extract(task.status, '$.name') AS status - FROM task - JOIN phobjects ON phobjects.phid = task.phid + FROM task + JOIN phobjects ON phobjects.phid = task.phid WHERE id = :task_id task_projects: title: projects task has been tagged on @@ -529,7 +492,7 @@ databases: LEFT OUTER JOIN phobjects authorPhobjects ON authorPHID = authorPhobjects.phid LEFT OUTER JOIN phobjects ownerPhobjects ON ownerPHID = ownerPhobjects.phid LEFT OUTER JOIN phobjects closerPhobjects ON closerPHID = closerPhobjects.phid - WHERE id = :task_id + WHERE id = :task_id task_days_in_project: title: days in project sql: |- @@ -540,10 +503,32 @@ databases: json_extract(data, '$.uri') AS task_uri, printf('%.0f', (sum(duration) / 86400.0)) as days, Project.name as project_name - FROM task_metrics + FROM task_metrics JOIN Project ON metric = project.phid JOIN phobjects ON phobjects.name = 'T' || task_metrics.task WHERE state = 'tagged' AND project.phid = :project AND task = CAST(:task_id AS decimal) + task-assignments: + title: Task Assignments + sql: SELECT DISTINCT + user, + count(task) as task_count, + old, new, + user=new as claim, + (old is not null and new is not null) as reassign, + new is null as unassign + FROM events + WHERE + event='assign' + GROUP BY + task, old, new + ORDER BY + task_count desc + test_query: + title: test title 123 + sql: SELECT * from task_metrics where task=:task_id + #sql: SELECT * from Project where name=:project_name + + extra_css_urls: - /static/styles.css diff --git a/www/plugins/ddd_datasette.py b/www/plugins/ddd_datasette.py index 8620bcf..706ec79 100644 --- a/www/plugins/ddd_datasette.py +++ b/www/plugins/ddd_datasette.py @@ -1,18 +1,119 @@ -from pathlib import Path - -from ddd import console -from ddd.phab import Conduit +import html import inspect +import importlib.util +from pathlib import Path from pprint import pprint -from typing import Dict, NewType, Type, TypeVar -from ddd.phobjects import PHID, PHIDRef, PHObject -from datasette.hookspecs import hookimpl -from importlib import import_module +from typing import Dict, Mapping, NewType, Type + from datasette.app import Datasette -from datasette.database import Database +from datasette.hookspecs import hookimpl +from datasette.utils.asgi import Request, Response +from ddd import console + +from ddd.phobjects import PHID +from jinja2 import Template + + + +async def ddd_view(datasette:Datasette, request, scope, send, receive): + # TODO: proper error handling: missing model or view should be a 404. + + page = request.url_vars["page"] + + views_dir = Path(__file__).parent.parent / "templates" / "views" + model_file = views_dir / page + '.py' + + # import the model module + spec = importlib.util.spec_from_file_location("module.name", model_file.absolute()) + assert(spec) + model_module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(model_module) # type: ignore + model_module.__name__ = f'ddd.model.{page}' + + db = datasette.get_database('metrics') + context = { + "console": console, + "request": request, + **request.url_vars + } + # augment the context with data from the model + context = await model_module.init_context( #type: ignore + datasette, + db, + context=context) + + output = await datasette.render_template(page+'.html', context, request, page) + return Response.html(output) + + + +@hookimpl +def register_routes(): + """return a list of routes, e.g. + [ (path_regex, callback), ] + """ + + def r(*parts, prefix='^/-/'): + """Compose a route entry from several path fragments""" + return prefix + "/".join(parts) + "/?$" + def v(name): + """Compose regex fragment that captures one path level as a named variable""" + return f"(?P<{name}>[a-zA-Z0-9\\-]+)" + def optional(part): + """Mark a path segment as optional by wrapping it with ()?""" + return f"?(part)?" + + + return [ + (r('ddd', v('page'), optional(v('subpage'))), ddd_view) + ] + +@hookimpl +def extra_template_vars(template:str, database:str, table:str, columns:str, view_name:str, request:Request, datasette:Datasette): + queries = canned_queries(datasette, database) + + async def execute_sql(sql, args=None, database=None): + db = datasette.get_database(database) + if sql in queries: + sql = queries[sql] + return (await db.execute(sql, args)).rows + + return { + "sql": execute_sql + } + + + +@hookimpl +def canned_queries(datasette: Datasette, database: str) -> Mapping[str, str]: + # load "canned queries" from the filesystem under + # www/sql/db/query_name.sql + queries = {} + + sqldir = Path(__file__).parent.parent / "sql" + if database: + sqldir = sqldir / database + + if not sqldir.is_dir(): + return queries + + for f in sqldir.glob('*.sql'): + try: + sql = f.read_text('utf8').strip() + if not len(sql): + log(f"Skipping empty canned query file: {f}") + continue + queries[f.stem] = { "sql": sql } + except OSError as err: + log(err) + + return queries + Query = NewType('Query', str) +def log(err): + console.log(err) class QueryListBase: _queries: dict @@ -170,80 +271,3 @@ class MetricsQueryList(QueryList, database='metrics'): order by end_ts; """) - -@hookimpl -def extra_template_vars(datasette: Datasette): - - async def ddd(datasource, args=None): - module = import_module(datasource) - db = datasette.get_database('metrics') - return await (module.run(db, args)) - - async def train_stats(version): - db = datasette.get_database('train') # type: Database - - cur = await ( - db.execute(TrainQueryList.train_properties_for_version, {"version": version}) - ) - - stats = {"participants": {}} - - for idx,key,actor,value in cur.rows: - stats['task'] = f'T{idx}' # type: ignore - if actor == '*': - stats[key] = value - else: - if key not in stats: - stats[key] = {} - stats[key][actor]=PHObject.instance(actor) - if key in ('resolvers', 'unblockers', 'commenters'): - if actor in stats['participants']: - stats['participants'][actor] += int(value) - else: - stats['participants'][actor] = int(value) - - cur = await ( - db.execute(TrainQueryList.train_blockers_joined, {"version": version}) - ) - - for row in cur.rows: - stats[row['metric']] = (row['added'], row['removed'], row['count']) # type: ignore - - PHObject.resolve_phids(Conduit()) - pprint(stats) - return stats - - async def instance(phid:PHID): - if phid[0:2] == '["' and phid[-2:] == '"]': - phid = phid[2:-2] # type: ignore - return PHObject.instance(phid) - - return {"ddd": ddd, "train": train_stats, "PHObject": instance} - - -def log(err): - console.log(err) - -@hookimpl -def canned_queries(datasette: Datasette, database: str): - sqldir = Path(__file__).parent / "sql" / database - if not sqldir.is_dir(): - return None - - queries = {} - - for f in sqldir.glob('*.sql'): - try: - sql = f.read_text('utf8').strip() - if not len(sql): - log(f"Skipping empty canned query file: {f}") - continue - queries[f.stem] = { "sql": sql } - except OSError as err: - log(err) - return queries - - -@hookimpl -async def render_custom_dashboard_chart(chart_display): - return "

test 1 2 3

" diff --git a/www/plugins/phabricator_datasette_plugin.py b/www/plugins/phabricator_datasette_plugin.py index f1c0677..4a9d05b 100644 --- a/www/plugins/phabricator_datasette_plugin.py +++ b/www/plugins/phabricator_datasette_plugin.py @@ -1,13 +1,20 @@ import sqlite3 -from typing import Union +from typing import Collection, Sequence, Union +from urllib.request import Request + +from datasette.facets import ColumnFacet, Facet +from datasette.utils import escape_sqlite, path_with_added_args, path_with_removed_args +import sqlite_utils +from sqlite_utils.db import NotFoundError +import ddd from ddd.phab import Conduit from ddd.phobjects import DataCache, PHIDType, isPHID, PHID, register_sqlite_adaptors, PHObject from datasette.hookspecs import hookimpl from datasette.app import Datasette -from datasette.database import Database -import jinja2 +from datasette.database import Database, QueryInterrupted +from jinja2.utils import Markup, escape import json phab = Conduit() @@ -37,9 +44,9 @@ def prepare_connection(conn: sqlite3.Connection): def A(href, label, target=None): if target: target = f' target="{target}"' - return jinja2.utils.Markup( + return Markup( '{label}'.format( - href=jinja2.utils.escape(href), label=jinja2.utils.escape(label or "") or " ", + href=escape(href), label=escape(label or "") or " ", target=target ) ) @@ -47,12 +54,56 @@ def A(href, label, target=None): @hookimpl def extra_template_vars(datasette): + async def all_subprojects(project:PHID): + sql="""--sql + WITH RECURSIVE parent_of(x, project_name, slug, uri, lvl) AS + ( + SELECT + phid AS x, + name, + '', + '', + 0 + FROM + Project p0 + WHERE + phid = :project + UNION ALL + SELECT + p1.phid AS phid, + p1.name AS project_name, + p1.slug as slug, + p1.uri as uri, + lvl + 1 AS lvl + FROM + Project p1 + JOIN + parent_of p2 ON p1.parent = x + ) + SELECT + po.*, + c.name AS column_name, + c.phid AS column_phid + FROM + parent_of po + JOIN + ProjectColumn c ON c.project = po.x + ORDER BY + lvl, + project_name, + column_name; + """ + + db = datasette.get_database('metrics') - async def execute_query(sql, args=None, database=None): - db = datasette.get_database(database) - return (await db.execute(sql, args)).rows + args={'project':project} + rows = (await db.execute(sql, args)).rows + phids = [row.phid for row in rows] + print(phids) + return phids - return {"sql": execute_query} + return { + "all_subprojects": all_subprojects} @hookimpl @@ -86,12 +137,12 @@ def render_cell( try: data = json.loads(value) data = [A(href=f'/metrics/phobjects/{phid}', label=phid) for phid in data] - return jinja2.Markup(",
".join(data)) + return Markup(",
".join(data)) except ValueError: return None if isPHID(value): - db = datasette.get_database('metrics') # type: Database + #db = datasette.get_database('metrics') # type: Database try: _ = value.index(',') @@ -99,8 +150,135 @@ def render_cell( A(href=f'/metrics/phobjects/{phid}', label=phid) for phid in value.split(',') ] - return jinja2.Markup("
".join(data)) + return Markup("
\n".join(data)) except ValueError: return A(href=f'/metrics/phobjects/{value}', label=value) return None + +class PHIDFacet(ColumnFacet): + # This key must be unique across all facet classes: + type:str = "phid" + sql:str + request:Request + database:str + ds:Datasette + params:Collection + table:str + + async def suggest(self): + + columns_res = await self.ds.execute( + self.database, f"select * from ({self.sql}) limit 1", self.params or [] + ) + columns = columns_res.columns + row = columns_res.first() + if not row: + return [] + + suggested_facets = [] + already_enabled = [c["config"]["simple"] for c in self.get_configs()] + + # Use self.sql and self.params to suggest some facets + suggested_facets = [] + + for column in columns: + ddd.console.log('isPHID: ', row[column], isPHID(row[column])) + if isPHID(row[column]): + ddd.console.log('isPHID: ', row[column], isPHID(row[column])) + suggested_facets.append({ + "name": f"PHID:{column}", + + "toggle_url": self.ds.absolute_url( + self.request, path_with_added_args( + self.request, {"_facet_phid": column} + ) + ), + }) + + return suggested_facets + + async def facet_results(self): + facet_results = {} + facets_timed_out = [] + + qs_pairs = self.get_querystring_pairs() + db = self.ds.get_database(self.database) + PHObject.db = sqlite_utils.db.Database(db.connect()) + facet_size = self.get_facet_size() + for source_and_config in self.get_configs(): + config = source_and_config["config"] + source = source_and_config["source"] + column = config.get("column") or config["simple"] + facet_sql = """ + select {col} as value, count(*) as count from ( + {sql} + ) + where {col} is not null + group by {col} order by count desc, value limit {limit} + """.format( + col=escape_sqlite(column), sql=self.sql, limit=facet_size + 1 + ) + try: + facet_rows_results = await self.ds.execute( + self.database, + facet_sql, + self.params, + truncate=False, + custom_time_limit=self.ds.setting("facet_time_limit_ms"), + ) + facet_results_values = [] + facet_results[column] = { + "name": column, + "type": self.type, + "hideable": source != "metadata", + "toggle_url": path_with_removed_args( + self.request, {"_facet_phid": column} + ), + "results": facet_results_values, + "truncated": len(facet_rows_results) > facet_size, + } + facet_rows = facet_rows_results.rows[:facet_size] + if self.table: + # Attempt to expand foreign keys into labels + values = [row["value"] for row in facet_rows] + expanded = await self.ds.expand_foreign_keys( + self.database, self.table, column, values + ) + else: + expanded = {} + for row in facet_rows: + selected = (column, str(row["value"])) in qs_pairs + obj = PHObject.instance(row["value"]) + try: + obj.load() + ddd.console.log(obj) + except NotFoundError: + continue + if selected: + toggle_path = path_with_removed_args( + self.request, {column: obj.phid} + ) + else: + toggle_path = path_with_added_args( + self.request, {column: obj.phid} + ) + facet_results_values.append( + { + "value": obj.phid, + "label": obj.name, + "count": row["count"], + "toggle_url": self.ds.absolute_url( + self.request, toggle_path + ), + "selected": selected, + } + ) + except QueryInterrupted: + facets_timed_out.append(column) + + return facet_results, facets_timed_out + +@hookimpl +def register_facet_classes(): + return [PHIDFacet] diff --git a/www/plugins/sql/metrics/all_events.bak.sql b/www/sql/metrics/all_events.bak.sql similarity index 100% rename from www/plugins/sql/metrics/all_events.bak.sql rename to www/sql/metrics/all_events.bak.sql diff --git a/www/plugins/sql/metrics/all_events.sql b/www/sql/metrics/all_events.sql similarity index 80% rename from www/plugins/sql/metrics/all_events.sql rename to www/sql/metrics/all_events.sql index 79211c7..7eac5b6 100644 --- a/www/plugins/sql/metrics/all_events.sql +++ b/www/sql/metrics/all_events.sql @@ -1,9 +1,10 @@ -select task, event, start, end, phid, name, sum(value) as value from +select task, event, start, mid, end, phid, name, sum(value) as value from ( SELECT task, event, w.date as start, +date(w.date, '+3 days') as mid, date(w.date, '+7 days') as end, new as phid, (select name from phobjects where phid=e.new) as name, @@ -14,7 +15,8 @@ JOIN events e ON w.date=date(e.ts, 'unixepoch', 'weekday 1', '-7 days') [[ AND date(start) >= date(:date_start) ]] [[ AND date(end) <= date(:date_end) ]] -WHERE event in ( 'columns') +WHERE TRUE +-- event in ( 'columns') [[ AND :project in (project, old, new) ]] union @@ -23,6 +25,7 @@ SELECT task, event, w.date as start, +date(w.date, '+3 days') as mid, date(w.date, '+7 days') as end, old as phid, (select name from phobjects where phid=e.old) as name, @@ -33,10 +36,10 @@ JOIN events e ON w.date=date(e.ts, 'unixepoch', 'weekday 1', '-7 days') [[ AND date(start) >= date(:date_start) ]] [[ AND date(end) <= date(:date_end) ]] - -WHERE event in ( 'columns') +WHERE TRUE +--event in ('' 'columns') [[ AND :project in (project, old, new) ]] ) WHERE name not null and name!="" group by start, phid -order by phid, start, value \ No newline at end of file +order by start --phid, start, value \ No newline at end of file diff --git a/www/plugins/sql/metrics/column_events.sql b/www/sql/metrics/column_events.sql similarity index 100% rename from www/plugins/sql/metrics/column_events.sql rename to www/sql/metrics/column_events.sql diff --git a/www/plugins/sql/metrics/columns_rollup.sql b/www/sql/metrics/columns_rollup.sql similarity index 100% rename from www/plugins/sql/metrics/columns_rollup.sql rename to www/sql/metrics/columns_rollup.sql diff --git a/www/plugins/sql/metrics/project_events.sql b/www/sql/metrics/project_events.sql similarity index 100% rename from www/plugins/sql/metrics/project_events.sql rename to www/sql/metrics/project_events.sql diff --git a/www/static/styles.css b/www/static/styles.css index ed518b4..bee73bc 100644 --- a/www/static/styles.css +++ b/www/static/styles.css @@ -25,4 +25,13 @@ a.phid { .col h3 { font-size: 100%; +} + +.subprojects { + overflow: auto; + height: 400px +} + +.subprojects li { + border-bottom: 1px solid rgb(14, 100, 139) } \ No newline at end of file diff --git a/www/templates/pages/project/{project}.html b/www/templates/pages/project/{project}.html deleted file mode 100644 index d627c26..0000000 --- a/www/templates/pages/project/{project}.html +++ /dev/null @@ -1,175 +0,0 @@ -{% extends "base.html" %} - -{% block extra_head %} - -{% endblock %} -{% block nav %} -
-{% endblock %} -{% block content %} -{% set labels, oldval, newval, valuev, y, phidmap, col = ddd('ddd.project_sankey', project) %} - - -
- -
- -{% endblock %} diff --git a/www/templates/pages/projects.html b/www/templates/pages/projects.html deleted file mode 100644 index f267fbf..0000000 --- a/www/templates/pages/projects.html +++ /dev/null @@ -1,22 +0,0 @@ -{% extends "base.html" %} - -{% set sql_query = "---sql -select - project_name, - printf('%s',project_phid) as phid, - count(*) as count -from - view_column_metrics -group by - project_name -order by count desc" %} - -{% block content %} -

Projects by activity

- -{% endblock %} diff --git a/www/templates/pages/train/{version}.html b/www/templates/pages/train/{version}.html deleted file mode 100644 index c5ffffe..0000000 --- a/www/templates/pages/train/{version}.html +++ /dev/null @@ -1,10 +0,0 @@ -{% extends "base.html" %} - -{% block content %} -

{{ version }}

-{% for row in query("select distinct project_name from columns", -database="metrics") %} -

{{ row.project_name }}

- -{% endfor %} -{% endblock %} diff --git a/www/templates/pages/trainsummary-{version}.html b/www/templates/pages/trainsummary-{version}.html deleted file mode 100644 index e8fec51..0000000 --- a/www/templates/pages/trainsummary-{version}.html +++ /dev/null @@ -1,49 +0,0 @@ -{% extends "base.html" %} -{% set train = train(version) %} - -{% block extra_head %} - -{% endblock %} - -{% block content %} - - -
- This email is a summary of the Wikimedia production deployment of {{version}} - - -

Stats

- - -

🚂🌈

-Thanks to everyone¹ who helped out with this train. Here are some names chosen at random: - -
-

-¹ yes, even the bots -

-

-- Be sure to tune in next time for another exciting trainwreck. -

-
-{% endblock %} \ No newline at end of file diff --git a/www/templates/project_columns.html b/www/templates/project_columns.html index 301bf98..90e57aa 100644 --- a/www/templates/project_columns.html +++ b/www/templates/project_columns.html @@ -1,10 +1,19 @@ +
Columns