Commit 26c7d88a authored by 20after4's avatar 20after4
Browse files

Aggregate train blocker statistics

Sample output:

```
1.36.0-wmf.1   added         1
               comment       4
               resolved      1
1.36.0-wmf.10  added         6
               comment      11
               removed       2
               resolved      3
1.36.0-wmf.11  added         8
               comment      29
               invalid       1
               removed       4
               resolved      3
               stalled       1
1.36.0-wmf.12  added         2
               comment       4
               removed       1
               resolved      1
1.36.0-wmf.13  added         5
               comment      11
               removed       3
               resolved      4
1.36.0-wmf.14  added         5
               comment      11
               removed       4
               resolved      1
1.36.0-wmf.15  comment       3
1.36.0-wmf.16  added         9
               comment      15
               open          2
               removed       5
               resolved      5
1.36.0-wmf.17  comment       2
1.36.0-wmf.18  added         1
               comment      24
               removed       1
1.36.0-wmf.19  comment       3
1.36.0-wmf.2   added         7
               comment      27
               removed       7
               resolved      1
1.36.0-wmf.20  added         6
               comment      21
               declined      1
               removed       3
               resolved      3
1.36.0-wmf.21  added         2
               comment      12
               resolved      3
1.36.0-wmf.22  added         1
               comment       4
               removed       1
1.36.0-wmf.23  comment       3
1.36.0-wmf.24  comment       3
1.36.0-wmf.25  added         4
               comment      27
               removed       2
               resolved      4
1.36.0-wmf.26  added         4
               comment       4
               open          1
               resolved      6
1.36.0-wmf.27  added         5
               comment      12
               removed       2
               resolved      3
1.36.0-wmf.28  added         6
               comment      16
               removed       7
               resolved      2
1.36.0-wmf.29  added         5
               comment       8
               removed       4
               resolved      4
1.36.0-wmf.3   added         5
               comment      12
               removed       4
               resolved      1
1.36.0-wmf.30  added         5
               comment      13
               removed       1
               resolved      4
1.36.0-wmf.31  added         4
               comment       8
               removed       2
               resolved      2
1.36.0-wmf.32  added         2
               comment      17
               removed       2
1.36.0-wmf.33  added         2
               comment      15
               removed       2
1.36.0-wmf.34  added         6
               comment      19
               open          2
               removed       4
               resolved      4
1.36.0-wmf.35  added         7
               comment      10
               open          1
               removed       4
               resolved      5
1.36.0-wmf.36  added         9
               comment      33
               declined      1
               open          1
               removed       2
               resolved      7
1.36.0-wmf.37  added         9
               comment      39
               removed       8
               resolved      1
1.36.0-wmf.38  added         3
               comment       8
               removed       3
1.36.0-wmf.39  added         1
               resolved      1
1.36.0-wmf.4   added         3
               comment      11
               removed       2
               resolved      1
1.36.0-wmf.40  comment       1
1.36.0-wmf.5   comment      19
1.36.0-wmf.6   comment       5
1.36.0-wmf.7   added         1
               comment       1
               removed       1
1.36.0-wmf.8   added         2
               comment       7
               removed       1
               resolved      1
1.36.0-wmf.9   added         4
               comment      25
               open          2
               removed       2
               resolved      3
               stalled       1
1.37.0-wmf.1   added         2
```

Change-Id: Iadf025ef26e017376dcdc8aefff4a0e50651c34d
parent e60d6a60
...@@ -9,32 +9,10 @@ from collections.abc import Iterator ...@@ -9,32 +9,10 @@ from collections.abc import Iterator
import json import json
import pprint import pprint
import sqlite3
from sqlite3.dbapi2 import Connection
from typing import Optional, Union from typing import Optional, Union
class DataCache(object):
con: sqlite3.Connection
def __init__(self, db):
self.con = sqlite3.connect(db)
self.con.execute(
"create table phobjects (phid TEXT PRIMARY KEY, name TEXT, data TEXT)"
)
def row(self, item):
data = json.dumps(item)
return (item.phid, item.name, data)
def store_all(self, items):
for item in items:
self.row(item)
def store_one(self, item, data):
values = (item.phid, item.name, data)
self.con.execute("replace into phobjects values (?, ?, ?)", values)
class DataIterator(Iterator): class DataIterator(Iterator):
...@@ -44,7 +22,7 @@ class DataIterator(Iterator): ...@@ -44,7 +22,7 @@ class DataIterator(Iterator):
data: Iterator data: Iterator
def __init__(self, data, parent=None): def __init__(self, data:Iterable, parent=None):
self.data = data.__iter__() self.data = data.__iter__()
self.parent = parent self.parent = parent
......
...@@ -2,19 +2,23 @@ from __future__ import annotations ...@@ -2,19 +2,23 @@ from __future__ import annotations
import json import json
import os import os
from builtins import str
from collections import UserDict, UserList, deque from collections import UserDict, UserList, deque
from collections.abc import Iterable from collections.abc import (
Iterable,
Collection,
Mapping,
MutableMapping,
MutableSequence)
from pprint import pprint from pprint import pprint
from tokenize import Number from tokenize import Number
from typing import Collection, MutableMapping, MutableSequence, Union from typing import Union
# todo: remove dependency on requests # todo: remove dependency on requests
import requests import requests
from numpy import real from numpy import real
from ddd.data import Data, DataIterator, wrapitem from ddd.data import Data, DataIterator, wrapitem
from ddd.phobjects import PHObject, isPHID from ddd.phobjects import PHID, PHObject, PhabObjectBase, isPHID, json_object_hook
class Conduit(object): class Conduit(object):
...@@ -48,7 +52,7 @@ class Conduit(object): ...@@ -48,7 +52,7 @@ class Conduit(object):
return os.environ.get("CONDUIT_TOKEN", token) return os.environ.get("CONDUIT_TOKEN", token)
def raw_request(self, method: str, args: MutableMapping): def raw_request(self, method: str, args: Mapping) -> requests.Response:
""" """
Helper method to call a phabricator api and return a ConduitCursor Helper method to call a phabricator api and return a ConduitCursor
which can be used to iterate over all of the resulting records. which can be used to iterate over all of the resulting records.
...@@ -58,7 +62,7 @@ class Conduit(object): ...@@ -58,7 +62,7 @@ class Conduit(object):
r = requests.post(f"{self.phab_url}{method}", data=req) r = requests.post(f"{self.phab_url}{method}", data=req)
return r return r
def request(self, method: str, args: MutableMapping): def request(self, method: str, args: MutableMapping) -> ConduitCursor:
r = self.raw_request(method=method, args=args) r = self.raw_request(method=method, args=args)
return ConduitCursor(conduit=self, res=r, method=method, args=args) return ConduitCursor(conduit=self, res=r, method=method, args=args)
...@@ -80,7 +84,7 @@ class Conduit(object): ...@@ -80,7 +84,7 @@ class Conduit(object):
req = flatten_for_post(req) req = flatten_for_post(req)
req["api.token"] = self.token req["api.token"] = self.token
r = requests.post(f"{self.phab_url}{method}", data=req) r = requests.post(f"{self.phab_url}{method}", data=req)
json = r.json() json = r.json(object_hook=json_object_hook)
if json["error_info"] is not None: if json["error_info"] is not None:
raise ConduitException(conduit=self, message=json["error_info"]) raise ConduitException(conduit=self, message=json["error_info"])
return json return json
...@@ -115,7 +119,8 @@ class ConduitCursor(object): ...@@ -115,7 +119,8 @@ class ConduitCursor(object):
self.handle_result(res) self.handle_result(res)
def retry(self): def retry(self):
pass res = self.conduit.raw_request(method=self.method, args=self.args)
self.handle_result(res)
def next_page(self): def next_page(self):
""" """
...@@ -136,14 +141,18 @@ class ConduitCursor(object): ...@@ -136,14 +141,18 @@ class ConduitCursor(object):
def fetch_all(self): def fetch_all(self):
while self.has_more(): while self.has_more():
self.next_page() self.next_page()
return self.data
def asdict(self, key="id"):
return { obj[key]:obj for obj in self.data }
def handle_result(self, res): def handle_result(self, res:requests.Response):
""" """
Process the result from a conduit call and store the records, along Process the result from a conduit call and store the records, along
with a cursor for fetching further pages when the result exceeds the with a cursor for fetching further pages when the result exceeds the
limit for a single request. The default and maximum limit is 100. limit for a single request. The default and maximum limit is 100.
""" """
json = res.json() json = res.json(object_hook=json_object_hook)
if json["error_info"] is not None: if json["error_info"] is not None:
raise ConduitException(conduit=self.conduit, message=json["error_info"]) raise ConduitException(conduit=self.conduit, message=json["error_info"])
...@@ -166,33 +175,11 @@ class ConduitCursor(object): ...@@ -166,33 +175,11 @@ class ConduitCursor(object):
def has_more(self): def has_more(self):
return self.cursor.get("after", None) return self.cursor.get("after", None)
def resolve_phids(self, data=None):
if data is None:
data = self.data
if isinstance(data, MutableMapping):
iter = data.items()
elif isinstance(data, (Iterable, MutableSequence)):
iter = enumerate(data)
else:
return
for key, val in iter:
if key != "phid" and isPHID(val):
data[key] = PHObject.instance(val)
elif isinstance(val, (MutableSequence, MutableMapping)):
self.resolve_phids(data=val)
if data is self.data and len(PHObject.instances):
PHObject.resolve_phids(self.conduit)
def __iter__(self): def __iter__(self):
return DataIterator(self.data) return DataIterator(self.data)
def __getitem__(self, key): def __getitem__(self, key):
if key not in self.data: return self.data[key]
raise KeyError(key)
return wrapitem(self.data[key])
def __len__(self): def __len__(self):
return len(self.data) return len(self.data)
...@@ -215,7 +202,8 @@ class ConduitException(Exception): ...@@ -215,7 +202,8 @@ class ConduitException(Exception):
def __str__(self): def __str__(self):
return "ConduitException: " + self.message return "ConduitException: " + self.message
def flatten_for_post(h, result=None, kk=None):
def flatten_for_post(h, result:dict=None, kk=None) -> dict[str, str]:
""" """
Since phab expects x-url-encoded form post data (meaning each Since phab expects x-url-encoded form post data (meaning each
individual list element is named). AND because, evidently, requests individual list element is named). AND because, evidently, requests
......
from __future__ import annotations from __future__ import annotations
from collections import UserDict
from itertools import repeat
import json
import sqlite3
import time
from collections.abc import Sequence, Mapping, MutableMapping, Iterator
from datetime import datetime
from enum import Enum from enum import Enum
from functools import total_ordering from functools import total_ordering
from pprint import pprint from pprint import pprint
import time from sqlite3 import Connection
from typing import ClassVar, Dict, Generic, Mapping, NewType, Type, TypeVar, Union from typing import (Any, ClassVar, Generic, NewType, Optional, Type, TypeVar,
from datetime import datetime Union)
""" """
Phabricator Objects Phabricator Objects
...@@ -25,7 +32,16 @@ from datetime import datetime ...@@ -25,7 +32,16 @@ from datetime import datetime
""" """
PHID = NewType("PHID", str) PHID = NewType("PHID", str)
Status = NewType('Status', str)
class Status(Enum):
Open = "open"
Closed = "closed"
Resolved = "closed"
Duplicate = "closed"
Declined = "closed"
Unknown = "unknown"
def isPHID(value: str): def isPHID(value: str):
...@@ -63,36 +79,47 @@ class SubclassCache(Generic[TID, T]): ...@@ -63,36 +79,47 @@ class SubclassCache(Generic[TID, T]):
obj = cls.instances.get(id) obj = cls.instances.get(id)
return obj return obj
@classmethod
def resolve_phids(cls, conduit):
phids = [phid for phid in cls.instances.keys()]
res = conduit.raw_request(method="phid.query", args={"phids": phids})
objs = res.json()
for key, vals in objs["result"].items():
cls.instances[key].update(vals)
class PhabObjectBase(UserDict, dict):
class PhabObjectBase(object):
phid: PHID phid: PHID
id: int = 0 id: int = 0
data: dict = {}
name: str name: str
fullName: str fullName: str
dateCreated: datetime dateCreated: datetime
dateModified: datetime dateModified: datetime
status: Status status: Status
def __init__(self, phid: PHID): def __init__(self, phid: PHID, **kwargs):
super().__init__(**kwargs)
self.phid = PHID(phid) self.phid = PHID(phid)
self.name = "(Unknown object)" self.name = "(Unknown object)"
self.status = Status("unknown") self.status = Status("unknown")
self.fullName = "" self.fullName = ""
def __getattr__(self, attr:str) -> Any:
if attr in self.data:
return self.data[attr]
raise AttributeError(f'{__name__!r} has no attribute {attr!r}')
def __setattr__(self, name: str, value: Any) -> None:
self.data[name] = value
object.__setattr__(self, name, value)
def __setitem__(self, name, value):
self.data[name] = value
object.__setattr__(self, name, value)
def __delattr__(self, attr):
if attr in self.data:
del(self.data[attr])
object.__delattr__(self, attr)
def __str__(self) -> str:
return self.name if len(self.name) else self.phid
def update(self, data: Mapping): def __repr__(self) -> str:
self.__dict__.update(data) return f'{self.__class__.__name__}(name="{self.name}", phid="{self.phid}")'
PHO = TypeVar("PHO", bound=PhabObjectBase) PHO = TypeVar("PHO", bound=PhabObjectBase)
...@@ -119,15 +146,10 @@ class PHObject(PhabObjectBase, SubclassCache[PHID, PhabObjectBase]): ...@@ -119,15 +146,10 @@ class PHObject(PhabObjectBase, SubclassCache[PHID, PhabObjectBase]):
del kwargs["dateModified"] del kwargs["dateModified"]
else: else:
self.dateModified = self.dateCreated self.dateModified = self.dateCreated
if len(kwargs):
self.update(kwargs)
self.__dict__.update(kwargs)
def __str__(self):
return self.name if len(self.name) else self.phid
def __repr__(self):
# {self.__dict__}
return f'<{self.__module__}.{self.__class__.__name__}(phid="{self.phid}", name="{self.name}")>'
def __eq__(self, other: PhabObjectBase): def __eq__(self, other: PhabObjectBase):
if isinstance(other, PhabObjectBase): if isinstance(other, PhabObjectBase):
...@@ -144,17 +166,34 @@ class PHObject(PhabObjectBase, SubclassCache[PHID, PhabObjectBase]): ...@@ -144,17 +166,34 @@ class PHObject(PhabObjectBase, SubclassCache[PHID, PhabObjectBase]):
return len(self.__dict__.keys()) > 1 return len(self.__dict__.keys()) > 1
@classmethod @classmethod
def instance(cls, phid: PHID) -> PhabObjectBase: def instance(cls, phid: PHID, data:Optional[Mapping] = None) -> PhabObjectBase:
obj = __class__.byid(phid) obj = __class__.byid(phid)
if obj: if not obj:
return obj phidtype = PHIDType(phid)
if isinstance(phidtype, str):
phidtype = __class__.subclass(phidtype, cls)
obj = phidtype(phid)
__class__.instances[phid] = obj
if data:
obj.update(data)
return obj
@classmethod
def resolve_phids(cls, conduit, cache:Optional[DataCache]=None):
phids = [phid for phid in cls.instances.keys()]
phidtype = PHIDType(phid) if cache:
if isinstance(phidtype, str): objs = cache.load(phids)
phidtype = __class__.subclass(phidtype, cls) for data in objs:
newinstance = phidtype(phid) phid = data['phid']
__class__.instances[phid] = newinstance phids -= phid
return newinstance cls.instance(phid).update(data)
res = conduit.raw_request(method="phid.query", args={"phids": phids})
objs = res.json()
for key, vals in objs["result"].items():
cls.instances[key].update(vals)
class User(PHObject): class User(PHObject):
...@@ -170,8 +209,11 @@ class ProjectColumn(PHObject): ...@@ -170,8 +209,11 @@ class ProjectColumn(PHObject):
class Task(PHObject): class Task(PHObject):
pass def update(self, data:MutableMapping):
if 'fields' in data.keys():
super().update(data['fields'])
del(data['fields'])
super().update(data)
class Commit(PHObject): class Commit(PHObject):
...@@ -188,12 +230,14 @@ class Application(PHObject): ...@@ -188,12 +230,14 @@ class Application(PHObject):
class Transaction(PHObject): class Transaction(PHObject):
relationships = { rel_by_name = {
"has-subtask": 3, "has-subtask": 3,
"has-parent": 4, "has-parent": 4,
"merge-in": 62, "merge-in": 62,
"close-as-duplicate": 63, "close-as-duplicate": 63,
"has-project": 41
} }
rel_by_id = { val:key for key, val in rel_by_name.items() }
class Event(PHObject): class Event(PHObject):
...@@ -201,6 +245,19 @@ class Event(PHObject): ...@@ -201,6 +245,19 @@ class Event(PHObject):
pass pass
class EdgeType(Enum):
TASK_COMMIT = 1
COMMIT_TASK = 2
SUB_TASK = 3
PARENT_TASK = 4
PROJECT_MEMBER = 13
SUBSCRIBER = 21
OBJECT_FILE = 25
FILE_OBJECT = 26
PROJECT_TAG = 41
MENTION = 51
DUPLICATE_TASK = 63
class PHIDTypes(Enum): class PHIDTypes(Enum):
PHOB = PHObject PHOB = PHObject
PROJ = Project PROJ = Project
...@@ -213,3 +270,85 @@ class PHIDTypes(Enum): ...@@ -213,3 +270,85 @@ class PHIDTypes(Enum):
PCOL = ProjectColumn PCOL = ProjectColumn
USER = User USER = User
class PHIDRef(object):
__slots__ = ('fromPHID', 'toPHID', 'object', 'relation')
fromPHID:Optional[PHID]
toPHID:PHID
object:PhabObjectBase
relation:Optional[PhabObjectBase]
def __init__(self, toPHID:PHID, fromPHID:Optional[PHID] = None):
self.toPHID = toPHID
self.fromPHID = fromPHID
self.object = PHObject.instance(self.toPHID)
if fromPHID:
self.relation = PHObject.instance(fromPHID)
else:
self.relation = None
def __repr__(self) -> str:
return "PHIDRef('%s')" % self.toPHID
def __str__(self) -> str:
return self.toPHID
def json_object_hook(obj:dict):
""" Instantiate PHID objects during json parsing """
phid = obj.get('phid', None)
for k, v in obj.items():
if not k == 'phid' and not isinstance(v, PHIDRef) and isPHID(v):
obj[k] = PHIDRef(v, phid)
if phid and isPHID(phid):
return PHObject.instance(phid=phid, data=obj)
else:
return obj
class DataCache(object):
con:sqlite3.Connection
replace_phobject = """
REPLACE INTO phobjects (phid, name, dateCreated, dateModified, data)
VALUES ( ?, ?, ?, ?, ?)
"""
def __init__(self, db):
self.con = sqlite3.connect(db)
self.con.row_factory = sqlite3.Row()
self.con.execute(
"""CREATE TABLE if not exists phobjects (
id real,
phid TEXT PRIMARY KEY,
authorPHID text,
name TEXT,
fullname TEXT,
dateCreated real,
dateModified real,
status TEXT,
data TEXT
); """)
def load(self, phids):
placeholders = ",".join(repeat("?", len(phids)))
select = f"SELECT * FROM phobjects where phid in ({placeholders})"
if isinstance(phids, list):
return self.con.executemany(select, phids)
else:
return self.con.execute(select, phids)
def row(self, item):
data = json.dumps(item)
return (item.phid, item.name, item.dateCreated, item.dateModified, data)
def store_all(self, items:Sequence[PHObject]):
rows = [self.row(item) for item in items]
self.con.executemany(self.replace_phobject, rows)
def store_one(self, item:PHObject):
values = self.row(item)
self.con.execute(self.replace_phobject, values)
...@@ -4,7 +4,7 @@ import sys ...@@ -4,7 +4,7 @@ import sys
from collections import UserDict from collections import UserDict
from operator import itemgetter from operator import itemgetter
from pprint import pprint from pprint import pprint
from typing import Mapping from typing import Callable, Mapping
import pandas as pd import pandas as pd
import requests import requests
...@@ -13,34 +13,18 @@ from IPython.display import display ...@@ -13,34 +13,18 @@ from IPython.display import display
import ddd import ddd
from ddd.mw import version from ddd.mw import version
from ddd.phab import Conduit, ConduitException from ddd.phab import Conduit, ConduitException
from ddd.phobjects import PHIDType, Task, PHObject from ddd.phobjects import PHIDRef, PHIDType, Task, PHObject, Transaction, EdgeType
phab = Conduit() phab = Conduit()
pd.options.display.max_columns = None pd.options.display.max_columns = None
pd.options.display.max_rows = None pd.options.display.max_rows = None