Commit 26c7d88a authored by 20after4's avatar 20after4
Browse files

Aggregate train blocker statistics

Sample output:

```
1.36.0-wmf.1   added         1
               comment       4
               resolved      1
1.36.0-wmf.10  added         6
               comment      11
               removed       2
               resolved      3
1.36.0-wmf.11  added         8
               comment      29
               invalid       1
               removed       4
               resolved      3
               stalled       1
1.36.0-wmf.12  added         2
               comment       4
               removed       1
               resolved      1
1.36.0-wmf.13  added         5
               comment      11
               removed       3
               resolved      4
1.36.0-wmf.14  added         5
               comment      11
               removed       4
               resolved      1
1.36.0-wmf.15  comment       3
1.36.0-wmf.16  added         9
               comment      15
               open          2
               removed       5
               resolved      5
1.36.0-wmf.17  comment       2
1.36.0-wmf.18  added         1
               comment      24
               removed       1
1.36.0-wmf.19  comment       3
1.36.0-wmf.2   added         7
               comment      27
               removed       7
               resolved      1
1.36.0-wmf.20  added         6
               comment      21
               declined      1
               removed       3
               resolved      3
1.36.0-wmf.21  added         2
               comment      12
               resolved      3
1.36.0-wmf.22  added         1
               comment       4
               removed       1
1.36.0-wmf.23  comment       3
1.36.0-wmf.24  comment       3
1.36.0-wmf.25  added         4
               comment      27
               removed       2
               resolved      4
1.36.0-wmf.26  added         4
               comment       4
               open          1
               resolved      6
1.36.0-wmf.27  added         5
               comment      12
               removed       2
               resolved      3
1.36.0-wmf.28  added         6
               comment      16
               removed       7
               resolved      2
1.36.0-wmf.29  added         5
               comment       8
               removed       4
               resolved      4
1.36.0-wmf.3   added         5
               comment      12
               removed       4
               resolved      1
1.36.0-wmf.30  added         5
               comment      13
               removed       1
               resolved      4
1.36.0-wmf.31  added         4
               comment       8
               removed       2
               resolved      2
1.36.0-wmf.32  added         2
               comment      17
               removed       2
1.36.0-wmf.33  added         2
               comment      15
               removed       2
1.36.0-wmf.34  added         6
               comment      19
               open          2
               removed       4
               resolved      4
1.36.0-wmf.35  added         7
               comment      10
               open          1
               removed       4
               resolved      5
1.36.0-wmf.36  added         9
               comment      33
               declined      1
               open          1
               removed       2
               resolved      7
1.36.0-wmf.37  added         9
               comment      39
               removed       8
               resolved      1
1.36.0-wmf.38  added         3
               comment       8
               removed       3
1.36.0-wmf.39  added         1
               resolved      1
1.36.0-wmf.4   added         3
               comment      11
               removed       2
               resolved      1
1.36.0-wmf.40  comment       1
1.36.0-wmf.5   comment      19
1.36.0-wmf.6   comment       5
1.36.0-wmf.7   added         1
               comment       1
               removed       1
1.36.0-wmf.8   added         2
               comment       7
               removed       1
               resolved      1
1.36.0-wmf.9   added         4
               comment      25
               open          2
               removed       2
               resolved      3
               stalled       1
1.37.0-wmf.1   added         2
```

Change-Id: Iadf025ef26e017376dcdc8aefff4a0e50651c34d
parent e60d6a60
......@@ -9,32 +9,10 @@ from collections.abc import Iterator
import json
import pprint
import sqlite3
from sqlite3.dbapi2 import Connection
from typing import Optional, Union
class DataCache(object):
con: sqlite3.Connection
def __init__(self, db):
self.con = sqlite3.connect(db)
self.con.execute(
"create table phobjects (phid TEXT PRIMARY KEY, name TEXT, data TEXT)"
)
def row(self, item):
data = json.dumps(item)
return (item.phid, item.name, data)
def store_all(self, items):
for item in items:
self.row(item)
def store_one(self, item, data):
values = (item.phid, item.name, data)
self.con.execute("replace into phobjects values (?, ?, ?)", values)
class DataIterator(Iterator):
......@@ -44,7 +22,7 @@ class DataIterator(Iterator):
data: Iterator
def __init__(self, data, parent=None):
def __init__(self, data:Iterable, parent=None):
self.data = data.__iter__()
self.parent = parent
......
......@@ -2,19 +2,23 @@ from __future__ import annotations
import json
import os
from builtins import str
from collections import UserDict, UserList, deque
from collections.abc import Iterable
from collections.abc import (
Iterable,
Collection,
Mapping,
MutableMapping,
MutableSequence)
from pprint import pprint
from tokenize import Number
from typing import Collection, MutableMapping, MutableSequence, Union
from typing import Union
# todo: remove dependency on requests
import requests
from numpy import real
from ddd.data import Data, DataIterator, wrapitem
from ddd.phobjects import PHObject, isPHID
from ddd.phobjects import PHID, PHObject, PhabObjectBase, isPHID, json_object_hook
class Conduit(object):
......@@ -48,7 +52,7 @@ class Conduit(object):
return os.environ.get("CONDUIT_TOKEN", token)
def raw_request(self, method: str, args: MutableMapping):
def raw_request(self, method: str, args: Mapping) -> requests.Response:
"""
Helper method to call a phabricator api and return a ConduitCursor
which can be used to iterate over all of the resulting records.
......@@ -58,7 +62,7 @@ class Conduit(object):
r = requests.post(f"{self.phab_url}{method}", data=req)
return r
def request(self, method: str, args: MutableMapping):
def request(self, method: str, args: MutableMapping) -> ConduitCursor:
r = self.raw_request(method=method, args=args)
return ConduitCursor(conduit=self, res=r, method=method, args=args)
......@@ -80,7 +84,7 @@ class Conduit(object):
req = flatten_for_post(req)
req["api.token"] = self.token
r = requests.post(f"{self.phab_url}{method}", data=req)
json = r.json()
json = r.json(object_hook=json_object_hook)
if json["error_info"] is not None:
raise ConduitException(conduit=self, message=json["error_info"])
return json
......@@ -115,7 +119,8 @@ class ConduitCursor(object):
self.handle_result(res)
def retry(self):
pass
res = self.conduit.raw_request(method=self.method, args=self.args)
self.handle_result(res)
def next_page(self):
"""
......@@ -136,14 +141,18 @@ class ConduitCursor(object):
def fetch_all(self):
while self.has_more():
self.next_page()
return self.data
def asdict(self, key="id"):
return { obj[key]:obj for obj in self.data }
def handle_result(self, res):
def handle_result(self, res:requests.Response):
"""
Process the result from a conduit call and store the records, along
with a cursor for fetching further pages when the result exceeds the
limit for a single request. The default and maximum limit is 100.
"""
json = res.json()
json = res.json(object_hook=json_object_hook)
if json["error_info"] is not None:
raise ConduitException(conduit=self.conduit, message=json["error_info"])
......@@ -166,33 +175,11 @@ class ConduitCursor(object):
def has_more(self):
return self.cursor.get("after", None)
def resolve_phids(self, data=None):
if data is None:
data = self.data
if isinstance(data, MutableMapping):
iter = data.items()
elif isinstance(data, (Iterable, MutableSequence)):
iter = enumerate(data)
else:
return
for key, val in iter:
if key != "phid" and isPHID(val):
data[key] = PHObject.instance(val)
elif isinstance(val, (MutableSequence, MutableMapping)):
self.resolve_phids(data=val)
if data is self.data and len(PHObject.instances):
PHObject.resolve_phids(self.conduit)
def __iter__(self):
return DataIterator(self.data)
def __getitem__(self, key):
if key not in self.data:
raise KeyError(key)
return wrapitem(self.data[key])
return self.data[key]
def __len__(self):
return len(self.data)
......@@ -215,7 +202,8 @@ class ConduitException(Exception):
def __str__(self):
return "ConduitException: " + self.message
def flatten_for_post(h, result=None, kk=None):
def flatten_for_post(h, result:dict=None, kk=None) -> dict[str, str]:
"""
Since phab expects x-url-encoded form post data (meaning each
individual list element is named). AND because, evidently, requests
......
from __future__ import annotations
from collections import UserDict
from itertools import repeat
import json
import sqlite3
import time
from collections.abc import Sequence, Mapping, MutableMapping, Iterator
from datetime import datetime
from enum import Enum
from functools import total_ordering
from pprint import pprint
import time
from typing import ClassVar, Dict, Generic, Mapping, NewType, Type, TypeVar, Union
from datetime import datetime
from sqlite3 import Connection
from typing import (Any, ClassVar, Generic, NewType, Optional, Type, TypeVar,
Union)
"""
Phabricator Objects
......@@ -25,7 +32,16 @@ from datetime import datetime
"""
PHID = NewType("PHID", str)
Status = NewType('Status', str)
class Status(Enum):
Open = "open"
Closed = "closed"
Resolved = "closed"
Duplicate = "closed"
Declined = "closed"
Unknown = "unknown"
def isPHID(value: str):
......@@ -63,36 +79,47 @@ class SubclassCache(Generic[TID, T]):
obj = cls.instances.get(id)
return obj
@classmethod
def resolve_phids(cls, conduit):
phids = [phid for phid in cls.instances.keys()]
res = conduit.raw_request(method="phid.query", args={"phids": phids})
objs = res.json()
for key, vals in objs["result"].items():
cls.instances[key].update(vals)
class PhabObjectBase(object):
class PhabObjectBase(UserDict, dict):
phid: PHID
id: int = 0
data: dict = {}
name: str
fullName: str
dateCreated: datetime
dateModified: datetime
status: Status
def __init__(self, phid: PHID):
def __init__(self, phid: PHID, **kwargs):
super().__init__(**kwargs)
self.phid = PHID(phid)
self.name = "(Unknown object)"
self.status = Status("unknown")
self.fullName = ""
def __getattr__(self, attr:str) -> Any:
if attr in self.data:
return self.data[attr]
raise AttributeError(f'{__name__!r} has no attribute {attr!r}')
def __setattr__(self, name: str, value: Any) -> None:
self.data[name] = value
object.__setattr__(self, name, value)
def __setitem__(self, name, value):
self.data[name] = value
object.__setattr__(self, name, value)
def update(self, data: Mapping):
self.__dict__.update(data)
def __delattr__(self, attr):
if attr in self.data:
del(self.data[attr])
object.__delattr__(self, attr)
def __str__(self) -> str:
return self.name if len(self.name) else self.phid
def __repr__(self) -> str:
return f'{self.__class__.__name__}(name="{self.name}", phid="{self.phid}")'
PHO = TypeVar("PHO", bound=PhabObjectBase)
......@@ -119,15 +146,10 @@ class PHObject(PhabObjectBase, SubclassCache[PHID, PhabObjectBase]):
del kwargs["dateModified"]
else:
self.dateModified = self.dateCreated
if len(kwargs):
self.update(kwargs)
self.__dict__.update(kwargs)
def __str__(self):
return self.name if len(self.name) else self.phid
def __repr__(self):
# {self.__dict__}
return f'<{self.__module__}.{self.__class__.__name__}(phid="{self.phid}", name="{self.name}")>'
def __eq__(self, other: PhabObjectBase):
if isinstance(other, PhabObjectBase):
......@@ -144,17 +166,34 @@ class PHObject(PhabObjectBase, SubclassCache[PHID, PhabObjectBase]):
return len(self.__dict__.keys()) > 1
@classmethod
def instance(cls, phid: PHID) -> PhabObjectBase:
def instance(cls, phid: PHID, data:Optional[Mapping] = None) -> PhabObjectBase:
obj = __class__.byid(phid)
if obj:
return obj
if not obj:
phidtype = PHIDType(phid)
if isinstance(phidtype, str):
phidtype = __class__.subclass(phidtype, cls)
newinstance = phidtype(phid)
__class__.instances[phid] = newinstance
return newinstance
obj = phidtype(phid)
__class__.instances[phid] = obj
if data:
obj.update(data)
return obj
@classmethod
def resolve_phids(cls, conduit, cache:Optional[DataCache]=None):
phids = [phid for phid in cls.instances.keys()]
if cache:
objs = cache.load(phids)
for data in objs:
phid = data['phid']
phids -= phid
cls.instance(phid).update(data)
res = conduit.raw_request(method="phid.query", args={"phids": phids})
objs = res.json()
for key, vals in objs["result"].items():
cls.instances[key].update(vals)
class User(PHObject):
......@@ -170,8 +209,11 @@ class ProjectColumn(PHObject):
class Task(PHObject):
pass
def update(self, data:MutableMapping):
if 'fields' in data.keys():
super().update(data['fields'])
del(data['fields'])
super().update(data)
class Commit(PHObject):
......@@ -188,12 +230,14 @@ class Application(PHObject):
class Transaction(PHObject):
relationships = {
rel_by_name = {
"has-subtask": 3,
"has-parent": 4,
"merge-in": 62,
"close-as-duplicate": 63,
"has-project": 41
}
rel_by_id = { val:key for key, val in rel_by_name.items() }
class Event(PHObject):
......@@ -201,6 +245,19 @@ class Event(PHObject):
pass
class EdgeType(Enum):
TASK_COMMIT = 1
COMMIT_TASK = 2
SUB_TASK = 3
PARENT_TASK = 4
PROJECT_MEMBER = 13
SUBSCRIBER = 21
OBJECT_FILE = 25
FILE_OBJECT = 26
PROJECT_TAG = 41
MENTION = 51
DUPLICATE_TASK = 63
class PHIDTypes(Enum):
PHOB = PHObject
PROJ = Project
......@@ -213,3 +270,85 @@ class PHIDTypes(Enum):
PCOL = ProjectColumn
USER = User
class PHIDRef(object):
__slots__ = ('fromPHID', 'toPHID', 'object', 'relation')
fromPHID:Optional[PHID]
toPHID:PHID
object:PhabObjectBase
relation:Optional[PhabObjectBase]
def __init__(self, toPHID:PHID, fromPHID:Optional[PHID] = None):
self.toPHID = toPHID
self.fromPHID = fromPHID
self.object = PHObject.instance(self.toPHID)
if fromPHID:
self.relation = PHObject.instance(fromPHID)
else:
self.relation = None
def __repr__(self) -> str:
return "PHIDRef('%s')" % self.toPHID
def __str__(self) -> str:
return self.toPHID
def json_object_hook(obj:dict):
""" Instantiate PHID objects during json parsing """
phid = obj.get('phid', None)
for k, v in obj.items():
if not k == 'phid' and not isinstance(v, PHIDRef) and isPHID(v):
obj[k] = PHIDRef(v, phid)
if phid and isPHID(phid):
return PHObject.instance(phid=phid, data=obj)
else:
return obj
class DataCache(object):
con:sqlite3.Connection
replace_phobject = """
REPLACE INTO phobjects (phid, name, dateCreated, dateModified, data)
VALUES ( ?, ?, ?, ?, ?)
"""
def __init__(self, db):
self.con = sqlite3.connect(db)
self.con.row_factory = sqlite3.Row()
self.con.execute(
"""CREATE TABLE if not exists phobjects (
id real,
phid TEXT PRIMARY KEY,
authorPHID text,
name TEXT,
fullname TEXT,
dateCreated real,
dateModified real,
status TEXT,
data TEXT
); """)
def load(self, phids):
placeholders = ",".join(repeat("?", len(phids)))
select = f"SELECT * FROM phobjects where phid in ({placeholders})"
if isinstance(phids, list):
return self.con.executemany(select, phids)
else:
return self.con.execute(select, phids)
def row(self, item):
data = json.dumps(item)
return (item.phid, item.name, item.dateCreated, item.dateModified, data)
def store_all(self, items:Sequence[PHObject]):
rows = [self.row(item) for item in items]
self.con.executemany(self.replace_phobject, rows)
def store_one(self, item:PHObject):
values = self.row(item)
self.con.execute(self.replace_phobject, values)
......@@ -4,7 +4,7 @@ import sys
from collections import UserDict
from operator import itemgetter
from pprint import pprint
from typing import Mapping
from typing import Callable, Mapping
import pandas as pd
import requests
......@@ -13,34 +13,18 @@ from IPython.display import display
import ddd
from ddd.mw import version
from ddd.phab import Conduit, ConduitException
from ddd.phobjects import PHIDType, Task, PHObject
from ddd.phobjects import PHIDRef, PHIDType, Task, PHObject, Transaction, EdgeType
phab = Conduit()
pd.options.display.max_columns = None
pd.options.display.max_rows = None
# find all train blocker tasks
r = phab.request(
"maniphest.search",
{
"queryKey": "ZKaMIUs_NEXo",
"constraints": {
'ids': ['249964'],
},
"limit": "50",
"attachments": {"projects": False, "columns": False},
},
)
r.fetch_all()
tasks = {task.id:task for task in r}
pd.options.display.max_colwidth = None
pd.options.display.width = 2000
def gettransactions(taskids):
"""a generator function that will yield formatted transaction details for a given list of task ids"""
formatters = {}
def ttype(ttype):
......@@ -68,25 +52,26 @@ def gettransactions(taskids):
def edge(t):
ov = t["oldValue"]
nv = t["newValue"]
edgetype = t['meta']['edge:type']
if edgetype != EdgeType.SUB_TASK.value:
# ignore relationships other than subtasks
# print('edgetype: %s' % edgetype)
return None
# some older phabricator transactions had a different format, fixup:
if isinstance(nv, Mapping):
nv = [key for key in nv.keys()]
if isinstance(ov, Mapping):
ov = [key for key in ov.keys()]
# ... we only care about tasks:
if len(ov) == 0 and len(nv) > 0 and PHIDType(nv[0]) is Task:
return ("added", [obj for obj in map(PHObject.instance, nv)])
elif len(ov) > 0 and len(nv) == 0 and PHIDType(ov[0]) is Task:
return ("removed", [obj for obj in map(PHObject.instance, ov)])
if len(ov) > len(nv):
diff = set(ov).difference(nv)
action = 'removed'
elif len(nv) > len(ov):
diff = set(nv).difference(ov)
action='added'
else:
print("--- new %s --- old %s ---" % (len(nv), len(ov)))
pprint(nv)
pprint(ov)
# ignore other edge types
pprint(t)
return None
tasklist = [PHIDRef(obj) for obj in diff]
tasklist = tasklist[0] if len(tasklist) == 1 else tasklist
return (action, tasklist)
# a subtask was closed or otherwise changed status
@ttype("unblock")
def unblock(t):
......@@ -94,10 +79,9 @@ def gettransactions(taskids):
ov = t["oldValue"]
for item in nv.items():
phid, action = item
return (action, PHObject.instance(phid))
return (action, PHIDRef(phid))
# a comment was added
#@ttype("core:comment")
@ttype("core:comment")
def comment(t):
# todo: we could check for the risky revision template here, if we care
# to count that.
......@@ -111,46 +95,65 @@ def gettransactions(taskids):
if nv:
return ('version', nv)
#@ttype("core:columns")
def columns(t):
pprint(t)
#@ttype("status")
def status(t):
pprint(t)
# ids = [id for id in tasks.keys()]
ids = [id for id in taskids.keys()]
transactions = phab.request(
"maniphest.gettasktransactions",
{
"ids": tasks.keys(),
"ids": ids,
},
)
for taskid, t in transactions.result.items():
st = sorted(t, key=itemgetter("dateCreated"))
task = tasks[int(taskid)]
train_version = ''
for tr in st:
trnstype = tr["transactionType"]
if trnstype not in formatters.keys():
# Ignore all transactions which do not have a matching formatter
# you can uncomment the print statement below to see what other
# transaction types are available:
print(trnstype)
# print(trnstype)
continue
# format the transaction data using the matching formatter function
formatted = formatters[trnstype](tr)
# yield the result when there is one:
if formatted and formatted[0] == 'version':
train_version=formatted[1]
formatted = None
if formatted:
yield ('T'+tr['taskID'], tr["dateCreated"], tr['authorPHID'], *formatted)
yield ('T'+tr['taskID'], train_version, tr["dateCreated"], tr['authorPHID'], *formatted)
# find all train blocker tasks
r = phab.request(
"maniphest.search",
{