community-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From s...@apache.org
Subject svn commit: r1689513 - in /comdev/reporter.apache.org/tags/test: data/parsepmcs.py mailglomper.py parseversions.py readjira.py scandist.py
Date Mon, 06 Jul 2015 22:12:29 GMT
Author: sebb
Date: Mon Jul  6 22:12:29 2015
New Revision: 1689513

URL: http://svn.apache.org/r1689513
Log:
Test

Added:
    comdev/reporter.apache.org/tags/test/data/parsepmcs.py   (with props)
    comdev/reporter.apache.org/tags/test/mailglomper.py   (with props)
    comdev/reporter.apache.org/tags/test/parseversions.py   (with props)
    comdev/reporter.apache.org/tags/test/readjira.py   (with props)
    comdev/reporter.apache.org/tags/test/scandist.py   (with props)

Added: comdev/reporter.apache.org/tags/test/data/parsepmcs.py
URL: http://svn.apache.org/viewvc/comdev/reporter.apache.org/tags/test/data/parsepmcs.py?rev=1689513&view=auto
==============================================================================
--- comdev/reporter.apache.org/tags/test/data/parsepmcs.py (added)
+++ comdev/reporter.apache.org/tags/test/data/parsepmcs.py Mon Jul  6 22:12:29 2015
@@ -0,0 +1,91 @@
+import re, urllib.request
+import csv
+import json
+import os
+import datetime
+import time
+pmcs = {}
+try:
+    with open("pmcs.json", "r") as f:
+        pmcs = json.loads(f.read())
+        f.close()
+except:
+    pass
+
+projects = {}
+try:
+    with open("projects.json", "r") as f:
+        projects = json.loads(f.read())
+        f.close()
+except:
+    pass
+
+
+people = {}
+newgroups = []
+
+data = urllib.request.urlopen("http://people.apache.org/committer-index.html").read().decode('utf-8')
+x = 0
+for committer in re.findall(r"<tr>([\S\s]+?)</tr>", data, re.MULTILINE | re.UNICODE):
+    x += 1
+##    print(committer)
+    m = re.search(r"<a id='(.+?)'>[\s\S]+?<td.+?>\s*(.+?)</td>[\s\S]+?>(.+)</td>",
committer, re.MULTILINE | re.UNICODE)
+    if m:
+        cid = m.group(1)
+        cname = re.sub(r"<.+?>", "", m.group(2), 4)
+        cproj = m.group(3)
+        isMember = False
+        if re.search(r"<b", committer, re.MULTILINE | re.UNICODE):
+            isMember = True
+        for project in re.findall(r"#([-a-z0-9._]+)-pmc", cproj):
+            now = time.time()
+            if not project in pmcs:
+                pmcs[project] = {}
+                newgroups.append(project)
+            if project in newgroups:
+                now = 0
+            if not cid in pmcs[project]:
+                pmcs[project][cid] = [cname, now, time.time()]
+            else:
+                pmcs[project][cid] = [pmcs[project][cid][0], pmcs[project][cid][1], time.time()]
+                
+        for project in re.findall(r"#([-a-z0-9._]+)(?!-pmc)", cproj):
+            now = time.time()
+            if not project in projects:
+                projects[project] = {}
+                newgroups.append(project)
+            elif project in newgroups:
+                now = 0
+            if not cid in projects[project]:
+                projects[project][cid] = [cname, now, time.time()]
+            else:
+                projects[project][cid] = [projects[project][cid][0], projects[project][cid][1],
time.time()]
+    
+# Delete retired members
+ret = 0
+for project in projects:
+    for cid in projects[project]:
+        if len(projects[project][cid]) < 3 or projects[project][cid][2] < (time.time()
- (86400*3)):
+            projects[project][cid] = "!"
+    projects[project] =  {i:projects[project][i] for i in projects[project] if projects[project][i]!="!"}
+
+for project in pmcs:
+    for cid in pmcs[project]:
+        if len(pmcs[project][cid]) < 3 or pmcs[project][cid][2] < (time.time() - (86400*3)):
+            pmcs[project][cid] = "!"
+    pmcs[project] =  {i:pmcs[project][i] for i in pmcs[project] if pmcs[project][i]!="!"}
+
+    
+print("Writing pmcs.json")
+with open("pmcs.json", "w") as f:
+    f.write(json.dumps(pmcs))
+    f.close()
+
+print("Writing projects.json")
+with open("projects.json", "w") as f:
+    f.write(json.dumps(projects))
+    f.close()
+    
+    
+print("All done! removed %u retired entries" % ret)
+

Propchange: comdev/reporter.apache.org/tags/test/data/parsepmcs.py
------------------------------------------------------------------------------
    svn:eol-style = native

Added: comdev/reporter.apache.org/tags/test/mailglomper.py
URL: http://svn.apache.org/viewvc/comdev/reporter.apache.org/tags/test/mailglomper.py?rev=1689513&view=auto
==============================================================================
--- comdev/reporter.apache.org/tags/test/mailglomper.py (added)
+++ comdev/reporter.apache.org/tags/test/mailglomper.py Mon Jul  6 22:12:29 2015
@@ -0,0 +1,67 @@
+import re, json, os, sys, urllib, time, email.utils
+
+from datetime import datetime
+
+
+mls = {}
+try:
+        with open("data/maildata_extended.json",'r') as f:
+                mls = json.loads(f.read())
+        print("Read JSON")
+except:
+        pass
+
+currentMonth = datetime.now().month
+currentYear = datetime.now().year
+after = time.time() - (86400*92)
+wayafter = time.time() - (86400*92*2)
+months = []
+for i in range(0,7):
+        date = "%04u%02u" % (currentYear, currentMonth)
+        currentMonth -= 1
+        if currentMonth == 0:
+                currentMonth = 12
+                currentYear -= 1
+        months.append(date)
+
+        now = int(time.time())
+
+data = urllib.urlopen("http://mail-archives.us.apache.org/mod_mbox/").read()
+print("Fetched %u bytes of main data" % len(data))
+y = 0
+for mlist in re.finditer(r"<a href='([-a-z0-9]+)/'", data):
+        ml = mlist.group(1)
+        y += 1
+        mls[ml] = {}
+        mls[ml]['quarterly'] = [0, 0];
+        mls[ml]['weekly'] = {}
+        for date in months:
+                        
+                try:
+                        mldata = urllib.urlopen("http://mail-archives.us.apache.org/mod_mbox/%s/%s.mbox"
% (ml, date)).read()
+                        if mldata:
+                                for c in re.finditer(r"Date: (.+)", mldata):
+                                        try:
+                                                d = email.utils.parsedate(c.group(1))
+                                                timestamp = int(time.mktime(d))
+                                                rounded = timestamp - (timestamp % 604800)
+ 604800
+                                                mls[ml]['weekly'][rounded] = (mls[ml]['weekly'][rounded]
if rounded in mls[ml]['weekly'] else 0) + 1
+                                                if timestamp >= after:
+                                                        mls[ml]['quarterly'][0] += 1
+                                                elif timestamp >= wayafter:
+                                                        mls[ml]['quarterly'][1] += 1
+                                        except:
+                                                pass
+                                                
+                except Exception as err:
+                        print(err)
+        print("%s: %u" % (ml, mls[ml]['quarterly'][0]))
+        if y == 50:
+                y = 0
+                with open("data/maildata_extended.json",'w+') as f:
+                        f.write(json.dumps(mls))
+with open("data/maildata_extended.json",'w+') as f:
+        f.write(json.dumps(mls))
+print("Dumped JSON")
+
+

Propchange: comdev/reporter.apache.org/tags/test/mailglomper.py
------------------------------------------------------------------------------
    svn:eol-style = native

Added: comdev/reporter.apache.org/tags/test/parseversions.py
URL: http://svn.apache.org/viewvc/comdev/reporter.apache.org/tags/test/parseversions.py?rev=1689513&view=auto
==============================================================================
--- comdev/reporter.apache.org/tags/test/parseversions.py (added)
+++ comdev/reporter.apache.org/tags/test/parseversions.py Mon Jul  6 22:12:29 2015
@@ -0,0 +1,41 @@
+import os, sys, json, urllib2, re, time, base64
+
+def getReleaseData(project):
+    try:
+        with open("/var/www/reporter.apache.org/data/releases/%s.json" % project, "r") as
f:
+            x = json.loads(f.read())
+            f.close()
+        return x;
+    except:
+        return {}
+       
+projects = {
+       'trafficserver': 'TS',
+       'accumulo': 'ACCUMULO'
+}
+
+jirapass = ""
+with open("/var/www/reporter.apache.org/data/jirapass.txt", "r") as f:
+    jirapass = f.read().strip()
+    f.close()
+
+for project in projects:
+       jiraname = projects[project]
+       base64string = base64.encodestring('%s:%s' % ('githubbot', jirapass))[:-1]
+       rdata = getReleaseData(project)
+       try:
+           req = req = urllib2.Request("https://issues.apache.org/jira/rest/api/2/project/%s/versions"
% jiraname)
+           req.add_header("Authorization", "Basic %s" % base64string)
+           cdata = json.loads(urllib2.urlopen(req).read())
+           for entry in cdata:
+              if ('name' in entry and 'releaseDate' in entry and 'released' in entry and
entry['released']):
+                     
+                     date = time.mktime(time.strptime(entry['releaseDate'], "%Y-%m-%d"))
+                     print("Updating version for %s - %s: %u" % (project, entry['name'],
date))
+                     rdata[entry['name']] = date
+       except Exception as err:
+           print(err)
+       
+       with open("/var/www/reporter.apache.org/data/releases/%s.json" % project, "w") as
f:
+              f.write(json.dumps(rdata))
+              f.close()
\ No newline at end of file

Propchange: comdev/reporter.apache.org/tags/test/parseversions.py
------------------------------------------------------------------------------
    svn:eol-style = native

Added: comdev/reporter.apache.org/tags/test/readjira.py
URL: http://svn.apache.org/viewvc/comdev/reporter.apache.org/tags/test/readjira.py?rev=1689513&view=auto
==============================================================================
--- comdev/reporter.apache.org/tags/test/readjira.py (added)
+++ comdev/reporter.apache.org/tags/test/readjira.py Mon Jul  6 22:12:29 2015
@@ -0,0 +1,43 @@
+#!/usr/bin/env python
+
+import re, os, json, urllib2, base64, time
+from os import listdir
+from os.path import isfile, join
+
+mypath = "/var/www/reporter.apache.org/data/JIRA"
+myfiles = [ f for f in listdir(mypath) if isfile(join(mypath,f)) ]
+
+jirapass = ""
+with open("/var/www/reporter.apache.org/data/jirapass.txt", "r") as f:
+    jirapass = f.read().strip()
+    f.close()
+    
+def getJIRAS(project):
+    refresh = True
+    if refresh:
+        base64string = base64.encodestring('%s:%s' % ('githubbot', jirapass))[:-1]
+
+        try:
+            req = req = urllib2.Request("""https://issues.apache.org/jira/rest/api/2/search?jql=project%20=%20"""
+ project + """%20AND%20created%20%3E=%20-91d""")
+            req.add_header("Authorization", "Basic %s" % base64string)
+            cdata = json.loads(urllib2.urlopen(req).read())
+            req = req = urllib2.Request("""https://issues.apache.org/jira/rest/api/2/search?jql=project%20=%20"""
+ project + """%20AND%20resolved%20%3E=%20-91d""")
+            req.add_header("Authorization", "Basic %s" % base64string)
+            rdata = json.loads(urllib2.urlopen(req).read())
+            with open("/var/www/reporter.apache.org/data/JIRA/%s.json" % project, "w") as
f:
+                f.write(json.dumps([cdata['total'], rdata['total'], project]))
+                f.close()
+            return cdata['total'], rdata['total'], project
+        except Exception as err:
+            with open("/var/www/reporter.apache.org/data/JIRA/%s.json" % project, "w") as
f:
+                f.write(json.dumps([0,0,None]))
+                f.close()
+            return 0,0, None
+
+for project in myfiles:
+    jiraname = project.replace(".json", "")
+    if jiraname != "projects":
+        print("Refreshing JIRA stats for " + jiraname)
+        getJIRAS(jiraname)
+        time.sleep(2)
+    
\ No newline at end of file

Propchange: comdev/reporter.apache.org/tags/test/readjira.py
------------------------------------------------------------------------------
    svn:eol-style = native

Added: comdev/reporter.apache.org/tags/test/scandist.py
URL: http://svn.apache.org/viewvc/comdev/reporter.apache.org/tags/test/scandist.py?rev=1689513&view=auto
==============================================================================
--- comdev/reporter.apache.org/tags/test/scandist.py (added)
+++ comdev/reporter.apache.org/tags/test/scandist.py Mon Jul  6 22:12:29 2015
@@ -0,0 +1,321 @@
+#!/usr/bin/env python
+
+#################################################
+# scandist.py: a minimalistic svnwcsub program  #
+#################################################
+
+from threading import Thread
+from datetime import datetime
+import os
+import sys
+
+
+# SMTP Lib
+import smtplib
+from email.mime.text import MIMEText
+from email.mime.multipart import MIMEMultipart
+
+version = 2
+if sys.hexversion < 0x03000000:
+    print("Using Python 2...")
+    import json, httplib, urllib, urllib2, ConfigParser as configparser, re, base64, sys,
os, time, atexit, signal, logging, socket, subprocess
+    socket._fileobject.default_bufsize = 0
+else:
+    print("Using Python 3")
+    version = 3
+    import json, http.client, urllib.request, urllib.parse, configparser, re, base64, sys,
os, time, atexit, signal, logging, subprocess
+
+targets = {}
+###########################################################
+# Daemon class, curtesy of an anonymous good-hearted soul #
+###########################################################
+class daemon:
+        """A generic daemon class.
+
+        Usage: subclass the daemon class and override the run() method."""
+
+        def __init__(self, pidfile): self.pidfile = pidfile
+        
+        def daemonize(self):
+                """Deamonize class. UNIX double fork mechanism."""
+
+                try: 
+                        pid = os.fork() 
+                        if pid > 0:
+                                # exit first parent
+                                sys.exit(0) 
+                except OSError as err: 
+                        sys.stderr.write('fork #1 failed: {0}\n'.format(err))
+                        sys.exit(1)
+        
+                # decouple from parent environment
+                os.chdir('/') 
+                os.setsid() 
+                os.umask(0) 
+        
+                # do second fork
+                try: 
+                        pid = os.fork() 
+                        if pid > 0:
+
+                                # exit from second parent
+                                sys.exit(0) 
+                except OSError as err: 
+                        sys.stderr.write('fork #2 failed: {0}\n'.format(err))
+                        sys.exit(1) 
+        
+                # redirect standard file descriptors
+                sys.stdout.flush()
+                sys.stderr.flush()
+                si = open(os.devnull, 'r')
+                so = open(os.devnull, 'a+')
+                se = open(os.devnull, 'a+')
+
+                os.dup2(si.fileno(), sys.stdin.fileno())
+                os.dup2(so.fileno(), sys.stdout.fileno())
+                os.dup2(se.fileno(), sys.stderr.fileno())
+        
+                # write pidfile
+                atexit.register(self.delpid)
+
+                pid = str(os.getpid())
+                with open(self.pidfile,'w+') as f:
+                        f.write(pid + '\n')
+        
+        def delpid(self):
+                os.remove(self.pidfile)
+
+        def start(self):
+                """Start the daemon."""
+
+                # Check for a pidfile to see if the daemon already runs
+                try:
+                        with open(self.pidfile,'r') as pf:
+
+                                pid = int(pf.read().strip())
+                except IOError:
+                        pid = None
+        
+                if pid:
+                        message = "pidfile {0} already exist. " + \
+                                        "Daemon already running?\n"
+                        sys.stderr.write(message.format(self.pidfile))
+                        sys.exit(1)
+                
+                # Start the daemon
+                self.daemonize()
+                self.run()
+
+        def stop(self):
+                """Stop the daemon."""
+
+                # Get the pid from the pidfile
+                try:
+                        with open(self.pidfile,'r') as pf:
+                                pid = int(pf.read().strip())
+                except IOError:
+                        pid = None
+        
+                if not pid:
+                        message = "pidfile {0} does not exist. " + \
+                                        "Daemon not running?\n"
+                        sys.stderr.write(message.format(self.pidfile))
+                        return # not an error in a restart
+
+                # Try killing the daemon process        
+                try:
+                        while 1:
+                                os.kill(pid, signal.SIGTERM)
+                                time.sleep(0.1)
+                except OSError as err:
+                        e = str(err.args)
+                        if e.find("No such process") > 0:
+                                if os.path.exists(self.pidfile):
+                                        os.remove(self.pidfile)
+                        else:
+                                print (str(err.args))
+                                sys.exit(1)
+
+        def restart(self):
+                """Restart the daemon."""
+                self.stop()
+                self.start()
+
+        def run(self):
+                """You should override this method when you subclass Daemon.
+                
+                It will be called after the process has been daemonized by 
+                start() or restart()."""
+
+
+
+####################
+# Helper functions #
+####################
+
+
+# read_chunk: iterator for reading chunks from the stream
+# since this is all handled via urllib now, this is quite rudimentary
+def read_chunk(req):
+    while True:
+        try:
+            line = req.readline().strip()
+            if line:
+                yield line
+            else:
+                print("No more lines?")
+                break
+        except Exception as info:
+            
+            break
+    return
+
+ 
+#########################
+# Main listener program #
+#########################
+
+
+
+# PubSub class: handles connecting to a pubsub service and checking commits
+class PubSubClient(Thread):
+    def run(self):
+        global targets
+        while True:
+            
+            self.req = None
+            while not self.req:
+                try:
+                    if version == 3:
+                        self.req = urllib.request.urlopen(self.url, None, 30)
+                    else:
+                        self.req = urllib2.urlopen(self.url, None, 30)
+                    
+                except:
+                    
+                    time.sleep(30)
+                    continue
+                
+            for line in read_chunk(self.req):
+                if version == 3:
+                    line = str( line, encoding='ascii' ).rstrip('\r\n,').replace('\x00','')
# strip away any old pre-0.9 commas from gitpubsub chunks and \0 in svnpubsub chunks
+                else:
+                    line = str( line ).rstrip('\r\n,').replace('\x00','') # strip away any
old pre-0.9 commas from gitpubsub chunks and \0 in svnpubsub chunks
+                try:
+                    obj = json.loads(line)
+                    if "commit" in obj and "repository" in obj['commit']:
+                        obj['commit']['whence'] = time.time()
+                    
+                            
+                        # If it's our public svn repo, then...
+                        if 'changed' in obj['commit']:
+                        
+                            #Grab some vars
+                            commit = obj['commit']
+                            body = commit['log']
+                            svnuser = commit['committer']
+                            path, action = commit['changed'].popitem()
+                            revision = commit['id']
+                            email = svnuser + "@apache.org"
+                            
+                            # Figure out the root thingermajig
+                            match = re.match(r"^release/([a-z0-9]+)", path)
+                            if match:
+                                if match.group(1) != "incubator":
+                                    targets[svnuser] = match.group(1)
+                            
+                    
+
+                except ValueError as detail:
+                    continue
+            
+
+
+
+
+################         
+# Main program #
+################
+def main():
+    global targets
+    if debug:
+        print("Foreground test mode enabled, no updates will be made")
+        
+  
+    
+    # Start the svn thread
+    svn_thread = PubSubClient()
+    svn_thread.url = "http://hades.apache.org:2069/commits/*"
+    svn_thread.start()
+    
+    while True:
+        now = time.time()
+        then = now - 86400
+       
+        time.sleep(600)
+        targetstwo = targets
+        targets = {}
+        for user in targetstwo:
+            email = user + "@apache.org"
+            project = targetstwo[user]
+            sender = 'no-reply@reporter.apache.org'
+            receivers = [email, 'humbedooh@apache.org']
+            print("Notifying %s of new data pushed to %s" % (email, project))
+            message = """From: Apache Reporter Service <no-reply@reporter.apache.org>
+To: %s <%s>
+Reply-To: dev@community.apache.org
+Subject: Please add your release data for '%s'
+
+Hi,
+This is an automated email from reporter.apache.org.
+I see that you just pushed something to our release repository for the %s project.
+
+If you are a PMC member of this project, we ask that you log on to:
+https://reporter.apache.org/addrelease.html?%s
+and add your release data (version and date) to the database.
+
+If you are not a PMC member, please have a PMC member add this information.
+
+While this is not a requirement, we ask that you still add this data to the
+reporter database, so that people using the Apache Reporter Service will be
+able to see the latest release data for this project.
+
+With regards,
+The Apache Reporter Service.
+            """ % (user, email, project, project, project)
+            
+            try:
+               smtpObj = smtplib.SMTP('localhost')
+               smtpObj.sendmail(sender, receivers, message)         
+               print "Successfully sent email"
+            except SMTPException:
+               print "Error: unable to send email"
+               
+
+##############
+# Daemonizer #
+##############
+class MyDaemon(daemon):
+    def run(self):
+        main()
+ 
+if __name__ == "__main__":
+        daemon = MyDaemon('/tmp/scandist.pid')
+        if len(sys.argv) >= 2:
+                if 'start' == sys.argv[1]:
+                    daemon.start()
+                elif 'stop' == sys.argv[1]:
+                    daemon.stop()
+                elif 'restart' == sys.argv[1]:
+                    daemon.restart()
+                elif 'foreground' == sys.argv[1]:
+                    debug = True
+                    main()
+                else:
+                    print("Unknown command")
+                    sys.exit(2)
+                sys.exit(0)
+        else:
+                print("usage: %s start|stop|restart|foreground" % sys.argv[0])
+                sys.exit(2)
+

Propchange: comdev/reporter.apache.org/tags/test/scandist.py
------------------------------------------------------------------------------
    svn:eol-style = native



Mime
View raw message