Commit 8d7c1607 authored by 20after4's avatar 20after4
Browse files

Added packaging (setup.cfg and pyproject.toml) and rearranged files

Moved trainblockers.py and boardmetrics.py into the ddd package
proper and made them importable for reuse outside the package.

Set up packaging so that we can publish the package to pypi.

Change-Id: I4001da1206c466d1575232601b57090fa77b5a1e
parent 95a4349a
...@@ -2,4 +2,9 @@ __pycache__ ...@@ -2,4 +2,9 @@ __pycache__
.mypy_cache .mypy_cache
*.csv *.csv
*.pyc *.pyc
.ipynb_checkpoints
.spyproject
.vscode .vscode
cache.db
*.egg-info
/dist
\ No newline at end of file
#!/usr/bin/python3 #!/usr/bin/python3
from collections.abc import Callable
from operator import itemgetter from operator import itemgetter
import pandas as pd
from IPython.display import display
from pprint import pprint
import ddd
from ddd.mw import MWVersion, version from ddd.mw import MWVersion, version
from ddd.phab import Conduit from ddd.phab import Conduit
from ddd.phobjects import PHIDRef, PHObject, EdgeType from ddd.phobjects import PHIDRef, PHObject, EdgeType
from ddd.data import PropertyMatcher from ddd.data import PropertyMatcher
pd.options.display.max_columns = None
pd.options.display.max_rows = None
pd.options.display.max_colwidth = 50
pd.options.display.width = 400
#%% #%%
phab = Conduit() def gettransactions(phab:Conduit, taskids):
# find all train blocker tasks
r = phab.request(
"maniphest.search",
{
"constraints": {"projects": ["release-engineering-team"]},
"limit": "30",
"attachments": {"projects": True, "columns": True},
},
)
#%%
def gettransactions(taskids):
mapper = PropertyMatcher() mapper = PropertyMatcher()
ids = [id for id in taskids.keys()] ids = [id for id in taskids.keys()]
transactions = phab.request( transactions = phab.request(
...@@ -60,31 +38,6 @@ def gettransactions(taskids): ...@@ -60,31 +38,6 @@ def gettransactions(taskids):
def status(t): def status(t):
return [("status",'', t["oldValue"], t["newValue"])] return [("status",'', t["oldValue"], t["newValue"])]
# @trnstype("unblock")
def unblock(t):
"""a subtask was closed or otherwise changed status"""
nv = t["newValue"]
for item in nv.items():
phid, action = item
return [[action, PHIDRef(phid)]]
# @trnstype("core:comment")
def comment(t):
# todo: we could check for the risky revision template here, if we care
# to count that.
nl = "\\n"
txt = str(t["comments"]).replace("\n", nl)
return [["comment", txt]]
@mapper('ransactionType=core:customfield')
def customfield(t):
"""Collect the version number from the release.version custom field"""
nv = version(str(t["newValue"]))
if nv:
train["version"] = nv
return None
@mapper('transactionType=core:columns') @mapper('transactionType=core:columns')
def columns(t): def columns(t):
newv = t["newValue"] newv = t["newValue"]
...@@ -106,18 +59,40 @@ def gettransactions(taskids): ...@@ -106,18 +59,40 @@ def gettransactions(taskids):
yield newrow yield newrow
#%% #%%
def main():
import pandas as pd
from IPython.display import display
# r.fetch_all() phab = Conduit()
tasks = r.asdict()
# now collect all of the formatted transaction details
rows = [row for row in gettransactions(tasks)]
PHObject.resolve_phids(phab) # find all train blocker tasks
r = phab.request(
"maniphest.search",
{
"constraints": {"projects": ["release-engineering-team"]},
"limit": "30",
"attachments": {"projects": True, "columns": True},
},
)
# r.fetch_all()
tasks = r.asdict()
# now collect all of the formatted transaction details
rows = [row for row in gettransactions(phab, tasks)]
#%% PHObject.resolve_phids(phab)
data = pd.DataFrame.from_records(
#%%
pd.options.display.max_columns = None
pd.options.display.max_rows = None
pd.options.display.max_colwidth = 50
pd.options.display.width = 400
data = pd.DataFrame.from_records(
rows, rows,
columns=["ts", "task", "description", "what", "from", "to"], columns=["ts", "task", "description", "what", "from", "to"],
index=["ts", "task"], index=["ts", "task"],
) )
display(data) display(data)
if __name__ == '__main__':
main()
...@@ -17,7 +17,6 @@ from operator import itemgetter ...@@ -17,7 +17,6 @@ from operator import itemgetter
# todo: remove dependency on requests # todo: remove dependency on requests
import requests import requests
from numpy import real
from ddd.data import Data, DataIterator, wrapitem from ddd.data import Data, DataIterator, wrapitem
from ddd.phobjects import PHID, PHObject, PhabObjectBase, isPHID, json_object_hook from ddd.phobjects import PHID, PHObject, PhabObjectBase, isPHID, json_object_hook
...@@ -28,6 +27,7 @@ class Cursor(object): ...@@ -28,6 +27,7 @@ class Cursor(object):
conduit: Conduit conduit: Conduit
data: deque[Data] data: deque[Data]
result: MutableMapping result: MutableMapping
method: str
def asdict(self, key="id"): def asdict(self, key="id"):
return {obj[key]: obj for obj in self.data} return {obj[key]: obj for obj in self.data}
...@@ -142,9 +142,6 @@ class ConduitCursor(Cursor): ...@@ -142,9 +142,6 @@ class ConduitCursor(Cursor):
api so that one api call can be treated as a single collection of results even api so that one api call can be treated as a single collection of results even
though it's split across multiple requests to the server. though it's split across multiple requests to the server.
""" """
method: str
def __init__( def __init__(
self, self,
conduit: Conduit, conduit: Conduit,
...@@ -179,7 +176,7 @@ class ConduitCursor(Cursor): ...@@ -179,7 +176,7 @@ class ConduitCursor(Cursor):
self.cursor = self.result["cursor"] self.cursor = self.result["cursor"]
else: else:
self.cursor = {} self.cursor = {}
# pprint(self.result)
if "data" in self.result: if "data" in self.result:
# Modern conduit methods return a result map with the key "data" # Modern conduit methods return a result map with the key "data"
# mapped to a list of records and the key "cursor" maps to a record # mapped to a list of records and the key "cursor" maps to a record
...@@ -189,8 +186,6 @@ class ConduitCursor(Cursor): ...@@ -189,8 +186,6 @@ class ConduitCursor(Cursor):
# Older methods just return a result: # Older methods just return a result:
self.data.extend(self.result.values()) self.data.extend(self.result.values())
def __iter__(self): def __iter__(self):
return DataIterator(self.data) return DataIterator(self.data)
......
#!/usr/bin/python3 #!/usr/bin/python3
import json
import sys
from collections import UserDict
from operator import itemgetter from operator import itemgetter
from pprint import pprint
from typing import Callable, Mapping
import pandas as pd from ddd.mw import MWVersion, version
import requests from ddd.phab import Conduit
from IPython.display import display from ddd.phobjects import PHIDRef, PHObject, EdgeType
import ddd #%%
from ddd.mw import version def gettransactions(phab:Conduit, taskids):
from ddd.phab import Conduit, ConduitException """a generator function that will yield formatted transaction details for a given list of task ids"""
from ddd.phobjects import PHIDRef, PHIDType, Task, PHObject, Transaction, EdgeType
phab = Conduit()
pd.options.display.max_columns = None
pd.options.display.max_rows = None
pd.options.display.max_colwidth = None
pd.options.display.width = 2000
train: dict[str, MWVersion] = {"version": MWVersion("")}
def gettransactions(taskids):
"""a generator function that will yield formatted transaction details for a given list of task ids"""
formatters = {} formatters = {}
def ttype(ttype): def ttype(ttype):
...@@ -42,17 +28,15 @@ def gettransactions(taskids): ...@@ -42,17 +28,15 @@ def gettransactions(taskids):
# it down to just the information we care about. # it down to just the information we care about.
# add new functions to match other transaction types. # add new functions to match other transaction types.
# edges point to one of several related objects such as:
# * subtask
# * project tag,
# * git commit
# * differential revision
@ttype("core:edge") @ttype("core:edge")
def edge(t): def edge(t):
"""edge transactions point to related objects such as subtasks, mentioned tasks and project tags.
The type of relationship is specified by an integer in meta['edge:type']. Edge type constants
are defined in the enum `ddd.phobjects.EdgeType`
"""
ov = t["oldValue"] ov = t["oldValue"]
nv = t["newValue"] nv = t["newValue"]
edgetype = t['meta']['edge:type'] edgetype = t["meta"]["edge:type"]
if edgetype != EdgeType.SUB_TASK.value: if edgetype != EdgeType.SUB_TASK.value:
# ignore relationships other than subtasks # ignore relationships other than subtasks
# print('edgetype: %s' % edgetype) # print('edgetype: %s' % edgetype)
...@@ -60,55 +44,52 @@ def gettransactions(taskids): ...@@ -60,55 +44,52 @@ def gettransactions(taskids):
if len(ov) > len(nv): if len(ov) > len(nv):
diff = set(ov).difference(nv) diff = set(ov).difference(nv)
action = 'removed' action = "removed"
elif len(nv) > len(ov): elif len(nv) > len(ov):
diff = set(nv).difference(ov) diff = set(nv).difference(ov)
action='added' action = "added"
else: else:
pprint(t) added = [("added", PHIDRef(phid)) for phid in nv]
return None removed = [("removed", PHIDRef(phid)) for phid in ov]
return added + removed
tasklist = [PHIDRef(obj) for obj in diff] tasklist = [PHIDRef(obj) for obj in diff]
tasklist = tasklist[0] if len(tasklist) == 1 else tasklist tasklist = tasklist[0] if len(tasklist) == 1 else tasklist
return (action, tasklist) return [(action, tasklist)]
# a subtask was closed or otherwise changed status
@ttype("unblock") @ttype("unblock")
def unblock(t): def unblock(t):
""" a subtask was closed or otherwise changed status"""
nv = t["newValue"] nv = t["newValue"]
ov = t["oldValue"]
for item in nv.items(): for item in nv.items():
phid, action = item phid, action = item
return (action, PHIDRef(phid)) return [(action, PHIDRef(phid))]
@ttype("core:comment") @ttype("core:comment")
def comment(t): def comment(t):
# todo: we could check for the risky revision template here, if we care # todo: we could check for the risky revision template here, if we care
# to count that. # to count that.
nl = '\\n' nl = "\\n"
txt = str(t["comments"]).replace("\n", nl) txt = str(t["comments"]).replace("\n", nl)
return ("comment", txt) return [("comment", txt)]
@ttype("core:customfield") @ttype("core:customfield")
def customfield(t): def customfield(t):
nv = version(str(t['newValue'])) """Collect the version number from the release.version custom field"""
nv = version(str(t["newValue"]))
if nv: if nv:
return ('version', nv) train["version"] = nv
return None
# ids = [id for id in tasks.keys()] # ids = [id for id in tasks.keys()]
ids = [id for id in taskids.keys()] ids = [id for id in taskids.keys()]
transactions = phab.request( transactions = phab.request("maniphest.gettasktransactions", {"ids": ids,},)
"maniphest.gettasktransactions",
{
"ids": ids,
},
)
for taskid, t in transactions.result.items(): for taskid, t in transactions.result.items():
st = sorted(t, key=itemgetter("dateCreated")) st = sorted(t, key=itemgetter("dateCreated"))
task = tasks[int(taskid)]
train_version = ''
for tr in st: for tr in st:
trnstype = tr["transactionType"] trnstype = tr["transactionType"]
...@@ -119,41 +100,55 @@ def gettransactions(taskids): ...@@ -119,41 +100,55 @@ def gettransactions(taskids):
# print(trnstype) # print(trnstype)
continue continue
# format the transaction data using the matching formatter function # format the transaction data using the matching formatter function
formatted = formatters[trnstype](tr) actions = formatters[trnstype](tr)
if not actions:
continue
# yield the result when there is one: # yield the result when there is one:
if formatted and formatted[0] == 'version': for action in actions:
train_version=formatted[1] yield (
formatted = None "T" + tr["taskID"],
train["version"],
tr["dateCreated"],
tr["authorPHID"],
*action,
)
if formatted:
yield ('T'+tr['taskID'], train_version, tr["dateCreated"], tr['authorPHID'], *formatted)
#%%
def main():
import pandas as pd
from IPython.display import display
# find all train blocker tasks pd.options.display.max_columns = None
r = phab.request( pd.options.display.max_rows = None
pd.options.display.max_colwidth = None
pd.options.display.width = 2000
phab = Conduit()
# find all train blocker tasks
r = phab.request(
"maniphest.search", "maniphest.search",
{ {
"constraints": { "constraints": {
'subtypes': ['release'], "subtypes": ["release"],
# 'ids': ['281146'], # 'ids': ['281146'],
}, },
"limit": "50", "limit": "50",
"attachments": {"projects": True, "columns": True}, "attachments": {"projects": True, "columns": True},
}, },
) )
r.fetch_all()
#tasks = {task[id]:task for task in r} r.fetch_all()
tasks = r.asdict() tasks = r.asdict()
# now collect all of the formatted transaction details
rows = [row for row in gettransactions(phab, tasks)]
PHObject.resolve_phids(phab)
# now collect all of the formatted transaction details data = pd.DataFrame(
rows = [row for row in gettransactions(tasks)] rows, columns=["task", "version", "timestamp", "author", "action", "values"]
PHObject.resolve_phids(phab) )
data = pd.DataFrame(rows, columns=[ grouped = data.groupby(["version", "action"])
'task', 'version', 'timestamp', 'author', 'action', 'values']) display(grouped["values"].count())
grouped = data.groupby(['version', 'action'])
display(grouped['values'].count())
if __name__ == "__main__": if __name__ == '__main__':
pass main()
\ No newline at end of file
================
PropertyMatcher
================
Use an instance of PropertyMatcher as a decorator to register callback functions
that match specified patterns within a tree of json data. The patterns are
specified as string arguments to the decorator declaration.
To match a specific key=value combination, simply specify the key name followed
by the value, separated by an equal sign and no whitespace or other characters.
To match a property nested within a sub-object, specify the path to the object
separated by dots, then a colon, then a key=value combination for the property
you want to match.
So in the example below, we match when both the top-level transactionType attribute
has the specific value "core:edge" and the object contains a child object named meta which contains a child called edge which has an attribute named type with a
value of 41
Usage example:
```python
def process_transactions(transactions):
mapper = PropertyMatcher()
@mapper("transactionType=core:edge", "meta.edge:type=41")
def edge(t):
''' match project edge transactions '''
oldValue = [PHIDRef(p) for p in t["oldValue"]]
newValue = [PHIDRef(p) for p in t["newValue"]]
return [["projects", '', oldValue, newValue]]
for taskid, t in transactions.result.items():
st = sorted(t, key=itemgetter("dateCreated"))
for record in st:
for row in mapper.run(record):
if row:
yield row
# fetch a bunch of data from somewhere:
transactions = get_some_transactions()
for match in process_transactions(transactions):
# do something with the match:
process(match)
```
\ No newline at end of file
[build-system]
requires = [
"setuptools >= 40.9.0",
"wheel",
]
build-backend = "setuptools.build_meta"
[metadata]
name = ddd
version = 0.0.1
[options]
packages = ddd
install_requires =
requests
[options.entry_points]
console_scripts =
workboardmetrics = ddd.workboardmetrics:main
trainblockers = ddd.trainblockers:main
\ No newline at end of file
import setuptools
setuptools.setup()
\ No newline at end of file
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment