community-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From humbed...@apache.org
Subject svn commit: r1864822 - in /comdev/reporter.apache.org/trunk/scripts: pdata.py rapp/kibble.py
Date Fri, 09 Aug 2019 16:42:38 GMT
Author: humbedooh
Date: Fri Aug  9 16:42:38 2019
New Revision: 1864822

URL: http://svn.apache.org/viewvc?rev=1864822&view=rev
Log:
move kibble into wsgi so we can also tweak mailing list stats etc atl

Added:
    comdev/reporter.apache.org/trunk/scripts/rapp/kibble.py   (with props)
Modified:
    comdev/reporter.apache.org/trunk/scripts/pdata.py

Modified: comdev/reporter.apache.org/trunk/scripts/pdata.py
URL: http://svn.apache.org/viewvc/comdev/reporter.apache.org/trunk/scripts/pdata.py?rev=1864822&r1=1864821&r2=1864822&view=diff
==============================================================================
--- comdev/reporter.apache.org/trunk/scripts/pdata.py (original)
+++ comdev/reporter.apache.org/trunk/scripts/pdata.py Fri Aug  9 16:42:38 2019
@@ -23,6 +23,7 @@
 
 import os, sys, re, json, subprocess, time
 import base64, requests
+import rapp.kibble
 
 CACHE_TIMEOUT = 14400
 
@@ -278,6 +279,10 @@ def generate(user, project, runkibble):
             lastRead = time.time()
         
         emails = {}
+        mlid = project # mailing list ID, might differ from ldap id
+        for k, v in pmap.items():
+            if v == project:
+                mlid = k
         
         for entry in mld: # e.g. hc-dev, ant-users, ws-dev
             tlp = entry.split("-")[0]
@@ -285,9 +290,7 @@ def generate(user, project, runkibble):
             if tlp == "empire":
                 tlp = "empire-db"
                 nentry = entry.replace("empire-", "empire-db-")
-            if tlp in pmap: # convert ml prefix to PMC internal name
-                tlp = pmap[tlp]
-            if tlp  == project:
+            if tlp  == mlid:
                 emails[nentry] = mld[entry]
                 
         dates = {}
@@ -373,19 +376,9 @@ def generate(user, project, runkibble):
         kibble = None
         if runkibble:
             try:
-                xenv = os.environ.copy()
-                if 'SCRIPT_NAME' in xenv:
-                    del xenv['SCRIPT_NAME']
-                cmd = ('%s/site/wizard/kibble.py' % RAOHOME_FULL, project)
-                if jdata and jdata[2]:
-                    cmd += tuple(jdata[2])
-                txt = subprocess.check_output(cmd, env = xenv)
-                if type(txt) is bytes:
-                    txt = txt.decode('ascii')
-                kibble = json.loads(txt)
-            except subprocess.CalledProcessError as e:
-                return None
-                
+                kibble = rapp.kibble.stats(project, jira = jdata[2], mlid = mlid)
+            except:
+                pass
             
         output = {
             'count': count,

Added: comdev/reporter.apache.org/trunk/scripts/rapp/kibble.py
URL: http://svn.apache.org/viewvc/comdev/reporter.apache.org/trunk/scripts/rapp/kibble.py?rev=1864822&view=auto
==============================================================================
--- comdev/reporter.apache.org/trunk/scripts/rapp/kibble.py (added)
+++ comdev/reporter.apache.org/trunk/scripts/rapp/kibble.py Fri Aug  9 16:42:38 2019
@@ -0,0 +1,331 @@
+#!/usr/bin/env python3
+""" Script for fetching somekibble data for reports """
+import requests
+import json
+import time
+import re
+import os
+
+RAO_HOME = '/var/www/reporter.apache.org'
+TOKEN = open('%s/data/kibble-token.txt' % RAO_HOME).read().strip()
+
+def stats(project, jira = [], mlid = None):
+    BEFORE = int(time.time() - (91*86400))
+    if not mlid:
+        mlid = project
+    
+    # Check for cache?
+    if not re.match(r"^[-_.a-z0-9]+$", project):
+        project = "bogus"
+    cache_file = '/tmp/kibble-cache-%s.json' % project
+    runit = True
+    js = {}
+    if (os.path.exists(cache_file) and os.path.getmtime(cache_file) > (time.time() - 86400)):
+        output = open(cache_file, "r").read()
+        try:
+            js = json.loads(output)
+            assert('prs' in js)
+            runit = False
+        except:
+            pass
+    if runit:
+        # Issues/PRs
+        issues = requests.post('https://demo.kibble.apache.org/api/issue/issues',
+                  headers = {
+                    'Content-Type': 'application/json',
+                    'Kibble-Token': TOKEN,
+                  },
+                  json = {
+                    "page":"issues",
+                    "quick":True,
+                    "interval": "week",
+                    "subfilter":"/(?:incubator-)?" + project + ".*\\.git",
+                    "distinguish":True
+                    }
+                 ).json();
+        after = [x for x in issues['timeseries'] if x['date'] > BEFORE]
+        before = [x for x in issues['timeseries'] if x['date'] <= BEFORE]
+        pro_before = 0
+        prc_before = 0
+        for month in before:
+            pro_before += month.get('pull requests opened', 0)
+            prc_before += month.get('pull requests closed', 0)
+        pro_after = 0
+        prc_after = 0
+        for month in after:
+            pro_after += month.get('pull requests opened', 0)
+            prc_after += month.get('pull requests closed', 0)
+        pro_change = '%u%%' % int((pro_after - pro_before) / (pro_before or 1) * 100)
+        prc_change = '%u%%' % int((prc_after - prc_before) / (prc_before or 1) * 100)
+        
+        
+        iso_before = 0
+        isc_before = 0
+        for month in before:
+            iso_before += month.get('issues opened', 0)
+            isc_before += month.get('issues closed', 0)
+        iso_after = 0
+        isc_after = 0
+        for month in after:
+            iso_after += month.get('issues opened', 0)
+            isc_after += month.get('issues closed', 0)
+        iso_change = '%u%%' % int((iso_after - iso_before) / (iso_before or 1) * 100)
+        isc_change = '%u%%' % int((isc_after - isc_before) / (isc_before or 1) * 100)
+        
+        github_ts = issues['timeseries']
+        
+        # Busiest GH issues/PRs
+        bissues = requests.post('https://demo.kibble.apache.org/api/issue/top',
+                  headers = {
+                    'Content-Type': 'application/json',
+                    'Kibble-Token': TOKEN,
+                  },
+                  json = {
+                    "quick":True,
+                    "interval": "month",
+                    "from": BEFORE, 
+                    "to": int(time.time()),
+                    "subfilter":"/(?:incubator-)?" + project + ".*\\.git",
+                    }
+                 ).json();
+        
+        # JIRA ??
+        jio_before = 0
+        jic_before = 0
+        jio_after = 0
+        jic_after = 0
+        jira_ts = []
+        if jira:
+            issues = requests.post('https://demo.kibble.apache.org/api/issue/issues',
+                      headers = {
+                        'Content-Type': 'application/json',
+                        'Kibble-Token': TOKEN,
+                      },
+                      json = {
+                        "quick":True,
+                        "interval": "week",
+                        "subfilter":"/browse/(" + "|".join(jira) + ")$",
+                        }
+                     ).json();
+            after = [x for x in issues['timeseries'] if x['date'] > BEFORE]
+            before = [x for x in issues['timeseries'] if x['date'] <= BEFORE]
+        
+            for month in before:
+                jio_before += month.get('issues opened', 0)
+                jic_before += month.get('issues closed', 0)
+            for month in after:
+                jio_after += month.get('issues opened', 0)
+                jic_after += month.get('issues closed', 0)
+            jira_ts = issues['timeseries']
+            
+        jio_change = '%u%%' % int((jio_after - jio_before) / (jio_before or 1) * 100)
+        jic_change = '%u%%' % int((jic_after - jic_before) / (jic_before or 1) * 100)
+        
+        
+        # Busiest JIRAs
+        bjiras = []
+        if jira:
+            bjiras = requests.post('https://demo.kibble.apache.org/api/issue/top',
+                      headers = {
+                        'Content-Type': 'application/json',
+                        'Kibble-Token': TOKEN,
+                      },
+                      json = {
+                        "quick":True,
+                        "interval": "month",
+                        "from": BEFORE, 
+                        "to": int(time.time()),
+                        "subfilter":"/browse/(" + "|".join(jira) + ")$",
+                        }
+                     ).json();
+            
+        
+        # Commits
+        commits = requests.post('https://demo.kibble.apache.org/api/code/commits',
+                  headers = {
+                    'Content-Type': 'application/json',
+                    'Kibble-Token': TOKEN,
+                  },
+                  json = {
+                    "page":"issues",
+                    "quick":True,
+                    "interval": "week",
+                    "subfilter":"/(?:incubator-)?" + project + ".*\\.git",
+                    }
+                 ).json();
+        after = [x for x in commits['timeseries'] if x['date'] > BEFORE]
+        before = [x for x in commits['timeseries'] if x['date'] <= BEFORE]
+        cmt_before = 0
+        for month in before:
+            cmt_before += month.get('commits', 0)
+        cmt_after = 0
+        for month in after:
+            cmt_after += month.get('commits', 0)
+        cmt_change = '%u%%' % int((cmt_after - cmt_before) / (cmt_before or 1) * 100)
+        commit_ts = commits['timeseries']
+        
+        # Committers
+        authors_b = requests.post('https://demo.kibble.apache.org/api/code/committers',
+                  headers = {
+                    'Content-Type': 'application/json',
+                    'Kibble-Token': TOKEN,
+                  },
+                  json = {
+                    "page":"issues",
+                    "quick":True,
+                    "from": int(BEFORE - (90*86400)),
+                    "to": BEFORE,
+                    "interval": "99999d",
+                    "subfilter":"/(?:incubator-)?" + project + ".*\\.git",
+                    }
+                 ).json()
+        authors_a = requests.post('https://demo.kibble.apache.org/api/code/committers',
+                  headers = {
+                    'Content-Type': 'application/json',
+                    'Kibble-Token': TOKEN,
+                  },
+                  json = {
+                    "page":"issues",
+                    "quick":True,
+                    "from": BEFORE,
+                    "to": int(time.time()),
+                    "interval": "99999d",
+                    "subfilter":"/(?:incubator-)?" + project + ".*\\.git",
+                    }
+                 ).json();
+        after = authors_a['timeseries']
+        before = authors_b['timeseries']
+        cmtp_before = 0
+        for month in before:
+            cmtp_before += month.get('authors', 0)
+        cmtp_after = 0
+        for month in after:
+            cmtp_after += month.get('authors', 0)
+        cmtp_change = '%u%%' % int((cmtp_after - cmtp_before) / (cmtp_before or 1) * 100)
+        
+        
+        # Dev+user list traffic?
+        dev_ts = []
+        user_ts = []
+        if True:
+            ml = r'dev@%s\.apache\.org' % mlid
+            emails = requests.post('https://demo.kibble.apache.org/api/mail/timeseries-single',
+                      headers = {
+                        'Content-Type': 'application/json',
+                        'Kibble-Token': TOKEN,
+                      },
+                      json = {
+                        "quick":True,
+                        "interval": "week",
+                        "subfilter":"\?%s$" % ml,
+                        }
+                     ).json()
+            dev_ts = emails['timeseries']
+            
+            ml = r'users?@%s\.apache\.org' % mlid
+            emails = requests.post('https://demo.kibble.apache.org/api/mail/timeseries-single',
+                      headers = {
+                        'Content-Type': 'application/json',
+                        'Kibble-Token': TOKEN,
+                      },
+                      json = {
+                        "quick":True,
+                        "interval": "week",
+                        "subfilter":"\?%s$" % ml,
+                        }
+                     ).json()
+            user_ts = emails['timeseries']
+            
+            
+        # Most discussed email topics
+        # This requires 4 months of data because data is compiled into
+        # monthly segments, so mid-month requests return skewed data.
+        topics = requests.post('https://demo.kibble.apache.org/api/mail/top-topics',
+                  headers = {
+                    'Content-Type': 'application/json',
+                    'Kibble-Token': TOKEN,
+                  },
+                  json = {
+                    "quick":True,
+                    "interval": "month",
+                    "from": BEFORE - (86400*30), 
+                    "to": int(time.time()),
+                    "subfilter":"(dev|users?)@" + project + "\.apache\.org",
+                    }
+                 ).json();
+        
+        js = {
+            'prs': {
+                'before': {
+                    'opened': pro_before,
+                    'closed': prc_before,
+                },
+                'after': {
+                    'opened': pro_after,
+                    'closed': prc_after,
+                },
+                'change': {
+                    'opened': pro_change,
+                    'closed': prc_change,
+                }
+            },
+            'issues': {
+                'before': {
+                    'opened': iso_before,
+                    'closed': isc_before,
+                },
+                'after': {
+                    'opened': iso_after,
+                    'closed': isc_after,
+                },
+                'change': {
+                    'opened': iso_change,
+                    'closed': isc_change,
+                }
+            },
+            'jira': {
+                'before': {
+                    'opened': jio_before,
+                    'closed': jic_before,
+                },
+                'after': {
+                    'opened': jio_after,
+                    'closed': jic_after,
+                },
+                'change': {
+                    'opened': jio_change,
+                    'closed': jic_change,
+                }
+            },
+            'commits' : {
+                'before': {
+                    'commits': cmt_before,
+                    'authors': cmtp_before,
+                },
+                'after': {
+                    'commits': cmt_after,
+                    'authors': cmtp_after,
+                },
+                'change': {
+                    'commits': cmt_change,
+                    'authors': cmtp_change,
+                }
+            },
+            'busiest': {
+                'email': topics['topN']['items'][:10],
+                'github': bissues['topN']['items'][:10],
+                'jira': bjiras['topN']['items'][:10] if bjiras else [],
+            },
+            'timeseries': {
+                'github': github_ts,
+                'jira': jira_ts,
+                'commits': commit_ts,
+                'devlist': dev_ts,
+                'userlist': user_ts,
+            }
+        }
+        output = json.dumps(js, indent = 2)
+        with open(cache_file, 'w') as f:
+            f.write(output)
+            f.close()
+    return js
\ No newline at end of file

Propchange: comdev/reporter.apache.org/trunk/scripts/rapp/kibble.py
------------------------------------------------------------------------------
    svn:executable = *



Mime
View raw message