Added packaging (setup.cfg and pyproject.toml) and rearranged files

Moved trainblockers.py and boardmetrics.py into the ddd package
proper and made them importable for reuse outside the package.

Set up packaging so that we can publish the package to pypi.

Change-Id: I4001da1206c466d1575232601b57090fa77b5a1e

.gitignore

@@ -2,4 +2,9 @@ __pycache__
 .mypy_cache
 *.csv
 *.pyc
+.ipynb_checkpoints
+.spyproject
 .vscode
+cache.db
+*.egg-info
+/dist
\ No newline at end of file

boardmetrics.py → ddd/boardmetrics.py

 #!/usr/bin/python3
-from collections.abc import Callable
 from operator import itemgetter
-import pandas as pd
-from IPython.display import display
-from pprint import pprint
 
-import ddd
 from ddd.mw import MWVersion, version
 from ddd.phab import Conduit
 from ddd.phobjects import PHIDRef, PHObject, EdgeType
 from ddd.data import PropertyMatcher
 
-pd.options.display.max_columns = None
-pd.options.display.max_rows = None
-pd.options.display.max_colwidth = 50
-pd.options.display.width = 400
-
 #%%
-phab = Conduit()
-# find all train blocker tasks
-r = phab.request(
-    "maniphest.search",
-    {
-        "constraints": {"projects": ["release-engineering-team"]},
-        "limit": "30",
-        "attachments": {"projects": True, "columns": True},
-    },
-)
-#%%
-
-def gettransactions(taskids):
+def gettransactions(phab:Conduit, taskids):
     mapper = PropertyMatcher()
     ids = [id for id in taskids.keys()]
     transactions = phab.request(
@@ -60,31 +38,6 @@ def gettransactions(taskids):
     def status(t):
         return [("status",'', t["oldValue"], t["newValue"])]
 
-    # @trnstype("unblock")
-    def unblock(t):
-        """a subtask was closed or otherwise changed status"""
-        nv = t["newValue"]
-
-        for item in nv.items():
-            phid, action = item
-            return [[action, PHIDRef(phid)]]
-
-    # @trnstype("core:comment")
-    def comment(t):
-        # todo: we could check for the risky revision template here, if we care
-        # to count that.
-        nl = "\\n"
-        txt = str(t["comments"]).replace("\n", nl)
-        return [["comment", txt]]
-
-    @mapper('ransactionType=core:customfield')
-    def customfield(t):
-        """Collect the version number from the release.version custom field"""
-        nv = version(str(t["newValue"]))
-        if nv:
-            train["version"] = nv
-        return None
-
     @mapper('transactionType=core:columns')
     def columns(t):
         newv = t["newValue"]
@@ -106,18 +59,40 @@ def gettransactions(taskids):
                     yield newrow
 
 #%%
+def main():
+    import pandas as pd
+    from IPython.display import display
 
-# r.fetch_all()
-tasks = r.asdict()
-# now collect all of the formatted transaction details
-rows = [row for row in gettransactions(tasks)]
+    phab = Conduit()
 
-PHObject.resolve_phids(phab)
+    # find all train blocker tasks
+    r = phab.request(
+        "maniphest.search",
+        {
+            "constraints": {"projects": ["release-engineering-team"]},
+            "limit": "30",
+            "attachments": {"projects": True, "columns": True},
+        },
+    )
+    # r.fetch_all()
+    tasks = r.asdict()
+    # now collect all of the formatted transaction details
+    rows = [row for row in gettransactions(phab, tasks)]
 
-#%%
-data = pd.DataFrame.from_records(
+    PHObject.resolve_phids(phab)
+
+    #%%
+    pd.options.display.max_columns = None
+    pd.options.display.max_rows = None
+    pd.options.display.max_colwidth = 50
+    pd.options.display.width = 400
+
+    data = pd.DataFrame.from_records(
         rows,
         columns=["ts", "task", "description", "what", "from", "to"],
         index=["ts", "task"],
-)
-display(data)
+    )
+    display(data)
+
+if __name__ == '__main__':
+    main()

ddd/phab.py

@@ -17,7 +17,6 @@ from operator import itemgetter
 
 # todo: remove dependency on requests
 import requests
-from numpy import real
 
 from ddd.data import Data, DataIterator, wrapitem
 from ddd.phobjects import PHID, PHObject, PhabObjectBase, isPHID, json_object_hook
@@ -28,6 +27,7 @@ class Cursor(object):
     conduit: Conduit
     data: deque[Data]
     result: MutableMapping
+    method: str
 
     def asdict(self, key="id"):
         return {obj[key]: obj for obj in self.data}
@@ -142,9 +142,6 @@ class ConduitCursor(Cursor):
     api so that one api call can be treated as a single collection of results even
     though it's split across multiple requests to the server.
     """
-
-    method: str
-
     def __init__(
         self,
         conduit: Conduit,
@@ -179,7 +176,7 @@ class ConduitCursor(Cursor):
             self.cursor = self.result["cursor"]
         else:
             self.cursor = {}
-        # pprint(self.result)
+
         if "data" in self.result:
             # Modern conduit methods return a result map with the key "data"
             # mapped to a list of records and the key "cursor" maps to a record
@@ -189,8 +186,6 @@ class ConduitCursor(Cursor):
             # Older methods just return a result:
             self.data.extend(self.result.values())
 
-
-
     def __iter__(self):
         return DataIterator(self.data)
 

trainblockers.py → ddd/trainblockers.py

 #!/usr/bin/python3
-import json
-import sys
-from collections import UserDict
 from operator import itemgetter
-from pprint import pprint
-from typing import Callable, Mapping
 
-import pandas as pd
-import requests
-from IPython.display import display
+from ddd.mw import MWVersion, version
+from ddd.phab import Conduit
+from ddd.phobjects import PHIDRef, PHObject, EdgeType
 
-import ddd
-from ddd.mw import version
-from ddd.phab import Conduit, ConduitException
-from ddd.phobjects import PHIDRef, PHIDType, Task, PHObject, Transaction, EdgeType
-
-phab = Conduit()
-
-pd.options.display.max_columns = None
-pd.options.display.max_rows = None
-pd.options.display.max_colwidth = None
-pd.options.display.width = 2000
+#%%
+def gettransactions(phab:Conduit, taskids):
+    """a generator function that will yield formatted transaction details for a given list of task ids"""
 
+    train: dict[str, MWVersion] = {"version": MWVersion("")}
 
-def gettransactions(taskids):
-    """a generator function that will yield formatted transaction details for a given list of task ids"""
     formatters = {}
 
     def ttype(ttype):
@@ -42,17 +28,15 @@ def gettransactions(taskids):
     # it down to just the information we care about.
     # add new functions to match other transaction types.
 
-    # edges point to one of several related objects such as:
-    # * subtask
-    # * project tag,
-    # * git commit
-    # * differential revision
-
     @ttype("core:edge")
     def edge(t):
+        """edge transactions point to related objects such as subtasks, mentioned tasks and project tags.
+            The type of relationship is specified by an integer in meta['edge:type']. Edge type constants
+            are defined in the enum `ddd.phobjects.EdgeType`
+        """
         ov = t["oldValue"]
         nv = t["newValue"]
-        edgetype = t['meta']['edge:type']
+        edgetype = t["meta"]["edge:type"]
         if edgetype != EdgeType.SUB_TASK.value:
             # ignore relationships other than subtasks
             # print('edgetype: %s' % edgetype)
@@ -60,55 +44,52 @@ def gettransactions(taskids):
 
         if len(ov) > len(nv):
             diff = set(ov).difference(nv)
-            action = 'removed'
+            action = "removed"
         elif len(nv) > len(ov):
             diff = set(nv).difference(ov)
-            action='added'
+            action = "added"
         else:
-            pprint(t)
-            return None
+            added = [("added", PHIDRef(phid)) for phid in nv]
+            removed = [("removed", PHIDRef(phid)) for phid in ov]
+            return added + removed
 
         tasklist = [PHIDRef(obj) for obj in diff]
         tasklist = tasklist[0] if len(tasklist) == 1 else tasklist
-        return (action, tasklist)
+        return [(action, tasklist)]
 
-    # a subtask was closed or otherwise changed status
     @ttype("unblock")
     def unblock(t):
+        """ a subtask was closed or otherwise changed status"""
         nv = t["newValue"]
-        ov = t["oldValue"]
+
         for item in nv.items():
             phid, action = item
-            return (action, PHIDRef(phid))
+            return [(action, PHIDRef(phid))]
 
     @ttype("core:comment")
     def comment(t):
         # todo: we could check for the risky revision template here, if we care
         # to count that.
-        nl = '\\n'
+        nl = "\\n"
         txt = str(t["comments"]).replace("\n", nl)
-        return ("comment", txt)
+        return [("comment", txt)]
 
     @ttype("core:customfield")
     def customfield(t):
-        nv = version(str(t['newValue']))
+        """Collect the version number from the release.version custom field"""
+        nv = version(str(t["newValue"]))
         if nv:
-            return ('version', nv)
+            train["version"] = nv
+        return None
 
     # ids = [id for id in tasks.keys()]
     ids = [id for id in taskids.keys()]
 
-    transactions = phab.request(
-        "maniphest.gettasktransactions",
-        {
-            "ids": ids,
-        },
-    )
+    transactions = phab.request("maniphest.gettasktransactions", {"ids": ids,},)
 
     for taskid, t in transactions.result.items():
         st = sorted(t, key=itemgetter("dateCreated"))
-        task = tasks[int(taskid)]
-        train_version = ''
+
         for tr in st:
 
             trnstype = tr["transactionType"]
@@ -119,41 +100,55 @@ def gettransactions(taskids):
                 # print(trnstype)
                 continue
             # format the transaction data using the matching formatter function
-            formatted = formatters[trnstype](tr)
+            actions = formatters[trnstype](tr)
+            if not actions:
+                continue
             # yield the result when there is one:
-            if formatted and formatted[0] == 'version':
-                train_version=formatted[1]
-                formatted = None
+            for action in actions:
+                yield (
+                    "T" + tr["taskID"],
+                    train["version"],
+                    tr["dateCreated"],
+                    tr["authorPHID"],
+                    *action,
+                )
 
-            if formatted:
-                yield ('T'+tr['taskID'], train_version, tr["dateCreated"], tr['authorPHID'], *formatted)
 
+#%%
+def main():
+    import pandas as pd
+    from IPython.display import display
 
-# find all train blocker tasks
-r = phab.request(
+    pd.options.display.max_columns = None
+    pd.options.display.max_rows = None
+    pd.options.display.max_colwidth = None
+    pd.options.display.width = 2000
+
+    phab = Conduit()
+    # find all train blocker tasks
+    r = phab.request(
         "maniphest.search",
         {
             "constraints": {
-            'subtypes': ['release'],
+                "subtypes": ["release"],
                 # 'ids': ['281146'],
             },
             "limit": "50",
             "attachments": {"projects": True, "columns": True},
         },
-)
-
-r.fetch_all()
+    )
 
-#tasks = {task[id]:task for task in r}
-tasks = r.asdict()
+    r.fetch_all()
+    tasks = r.asdict()
+    # now collect all of the formatted transaction details
+    rows = [row for row in gettransactions(phab, tasks)]
+    PHObject.resolve_phids(phab)
 
-# now collect all of the formatted transaction details
-rows = [row for row in gettransactions(tasks)]
-PHObject.resolve_phids(phab)
-data = pd.DataFrame(rows, columns=[
-    'task', 'version', 'timestamp', 'author', 'action', 'values'])
-grouped = data.groupby(['version', 'action'])
-display(grouped['values'].count())
+    data = pd.DataFrame(
+        rows, columns=["task", "version", "timestamp", "author", "action", "values"]
+    )
+    grouped = data.groupby(["version", "action"])
+    display(grouped["values"].count())
 
-if __name__ == "__main__":
-    pass
+if __name__ == '__main__':
+    main()
\ No newline at end of file

docs/PropertyMatcher.md

+================
+PropertyMatcher
+================
+
+Use an instance of PropertyMatcher as a decorator to register callback functions
+that match specified patterns within a tree of json data.  The patterns are
+specified as string arguments to the decorator declaration.
+
+To match a specific key=value combination, simply specify the key name followed
+by the value, separated by an equal sign and no whitespace or other characters.
+To match a property nested within a sub-object, specify the path to the object
+separated by dots, then a colon, then a key=value combination for the property
+you want to match.
+
+So in the example below, we match when both the top-level transactionType attribute
+has the specific value "core:edge" and the object contains a child object named meta which contains a child called edge which has an attribute named type with a
+value of 41
+
+Usage example:
+
+```python
+def process_transactions(transactions):
+    mapper = PropertyMatcher()
+
+    @mapper("transactionType=core:edge", "meta.edge:type=41")
+    def edge(t):
+        ''' match project edge transactions '''
+        oldValue = [PHIDRef(p) for p in t["oldValue"]]
+        newValue = [PHIDRef(p) for p in t["newValue"]]
+        return [["projects", '', oldValue, newValue]]
+
+    for taskid, t in transactions.result.items():
+        st = sorted(t, key=itemgetter("dateCreated"))
+        for record in st:
+            for row in mapper.run(record):
+                if row:
+                    yield row
+
+# fetch a bunch of data from somewhere:
+transactions = get_some_transactions()
+
+for match in process_transactions(transactions):
+    # do something with the match:
+    process(match)
+
+```
\ No newline at end of file

pyproject.toml

+[build-system]
+requires = [
+  "setuptools >= 40.9.0",
+  "wheel",
+]
+build-backend = "setuptools.build_meta"

setup.cfg

+[metadata]
+name = ddd
+version = 0.0.1
+
+[options]
+packages = ddd
+install_requires =
+    requests
+
+[options.entry_points]
+console_scripts =
+    workboardmetrics = ddd.workboardmetrics:main
+    trainblockers = ddd.trainblockers:main
\ No newline at end of file

setup.py

+import setuptools
+setuptools.setup()
\ No newline at end of file