community-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From humbed...@apache.org
Subject svn commit: r1864689 - in /comdev/reporter.apache.org/trunk/scripts: pdata.py rapp/ rapp/drafts.py rapp/overview.py rapp/whimsy.py wsgi.py
Date Thu, 08 Aug 2019 11:07:48 GMT
Author: humbedooh
Date: Thu Aug  8 11:07:48 2019
New Revision: 1864689

URL: http://svn.apache.org/viewvc?rev=1864689&view=rev
Log:
Add in new API endpoints for WSGI app.

Added:
    comdev/reporter.apache.org/trunk/scripts/rapp/
    comdev/reporter.apache.org/trunk/scripts/rapp/drafts.py
    comdev/reporter.apache.org/trunk/scripts/rapp/overview.py
    comdev/reporter.apache.org/trunk/scripts/rapp/whimsy.py
Modified:
    comdev/reporter.apache.org/trunk/scripts/pdata.py
    comdev/reporter.apache.org/trunk/scripts/wsgi.py

Modified: comdev/reporter.apache.org/trunk/scripts/pdata.py
URL: http://svn.apache.org/viewvc/comdev/reporter.apache.org/trunk/scripts/pdata.py?rev=1864689&r1=1864688&r2=1864689&view=diff
==============================================================================
--- comdev/reporter.apache.org/trunk/scripts/pdata.py (original)
+++ comdev/reporter.apache.org/trunk/scripts/pdata.py Thu Aug  8 11:07:48 2019
@@ -41,6 +41,9 @@ MEMBER_INFO = 'https://whimsy.apache.org
 PROJECTS = 'https://whimsy.apache.org/public/public_ldap_projects.json'
 DESCRIPTIONS = 'https://projects.apache.org/json/foundation/committees.json'
 
+def has_cache(filename, ttl = 14400):
+    return (os.path.exists(filename) and os.path.getmtime(filename) > (time.time() - ttl))
+
 jmap = {
     'trafficserver': ['TS'],
     'cordova': ['CB'],

Added: comdev/reporter.apache.org/trunk/scripts/rapp/drafts.py
URL: http://svn.apache.org/viewvc/comdev/reporter.apache.org/trunk/scripts/rapp/drafts.py?rev=1864689&view=auto
==============================================================================
--- comdev/reporter.apache.org/trunk/scripts/rapp/drafts.py (added)
+++ comdev/reporter.apache.org/trunk/scripts/rapp/drafts.py Thu Aug  8 11:07:48 2019
@@ -0,0 +1,114 @@
+#!/usr/bin/env python3
+# -*- coding: UTF-8 -*-
+""" script for working with drafts """
+import os
+import sys
+import time
+import json
+import re
+import pdata
+import committee_info
+
+DRAFTS_DIR = '/tmp/rapp-drafts'
+EDITOR_TYPE = 'unified'
+
+if not os.path.isdir(DRAFTS_DIR):
+    os.makedirs(DRAFTS_DIR, exist_ok = True)
+
+def has_access(user, project):
+    member = pdata.isASFMember(user)
+    pmc = project in pdata.getPMCs(user)
+    return (member or pmc)
+
+def index(environ, user):
+    """ Listy currently existing drafts for a project """
+    project = environ.get('QUERY_STRING')
+    drafts = {}
+    
+    if has_access(user, project):
+        whence = int(time.time() - (60*86400)) # Max 2 months ago!
+        for filename in [x for x in os.listdir(DRAFTS_DIR) if x.startswith(EDITOR_TYPE) and
x.endswith('.draft')]:
+            e, p, t, u = filename.split('-', 3)
+            t = int(t)
+            # If a file is way old, try deleting it.
+            if t < whence:
+                try:
+                    os.unlink("%s/%s" % (DRAFTS_DIR, filename))
+                except:
+                    pass
+            elif p == project and t >= whence:
+                u = u.replace('.draft', '')
+                drafts[t] = {'filename': filename, 'creator': u, 'yours': user == u}
+    
+    return { 'drafts': drafts }
+
+def fetch(environ, user):
+    """ Fetch a draft if access is right... """
+    filename = environ.get('QUERY_STRING')
+    m =  re.match(r"[^-./]+-([^-./]+)-\d+-[^-./]+\.draft$", filename)
+    if not m:
+        return {'error': "Invalid filename!"}
+    if os.path.exists(os.path.join(DRAFTS_DIR, filename)):
+        project = m.group(1)
+        if has_access(user, project):
+            report = open(os.path.join(DRAFTS_DIR, filename), "r").read()
+            return {'report': report}
+    return {}
+
+def delete(environ, user):
+    """ Delete a draft if access is right... """
+    filename = environ.get('QUERY_STRING')
+    m =  re.match(r"[^-./]+-[^-./]+-\d+-([^-/]+)\.draft$", filename)
+    if not m:
+        return {'error': "Invalid filename!"}
+    if os.path.exists(os.path.join(DRAFTS_DIR, filename)):
+        u = m.group(1)
+        if user == u:
+            try:
+                os.unlink(os.path.join(DRAFTS_DIR, filename))
+                return {'message': 'Draft deleted'}
+            except:
+                pass
+    return {'error': "Could not delete draft!"}
+
+def save(environ, user):
+    """ Save a draft """
+    try:
+        request_body_size = int(environ.get('CONTENT_LENGTH', 0))
+    except (ValueError):
+        request_body_size = 0
+    if request_body_size:
+        request_body = environ['wsgi.input'].read(request_body_size)
+        try:
+            js = json.loads(request_body.decode('utf-8'))
+        except:
+            js = {}
+    if js:
+        project = js.get('project')
+        if has_access(user, project):
+            report = js.get('report')
+            now = int(time.time())
+            filename = '%s-%s-%s-%s.draft' % (EDITOR_TYPE, project, now, user)
+            try:
+                with open(os.path.join(DRAFTS_DIR, filename), "w") as f:
+                    f.write(report)
+                    f.close()
+                return {
+                    'okay': True,
+                    'filename': filename,
+                }
+            except:
+                return {
+                    'okay': False,
+                    'error': 'Could not save draft (permission issue?? disk full?)',
+                }
+        else:
+            return {
+                'okay': False,
+                'error': 'You do not have access to this project',
+            }
+    else:
+        return {
+            'okay': False,
+            'error': "Invalid data!",
+        }

Added: comdev/reporter.apache.org/trunk/scripts/rapp/overview.py
URL: http://svn.apache.org/viewvc/comdev/reporter.apache.org/trunk/scripts/rapp/overview.py?rev=1864689&view=auto
==============================================================================
--- comdev/reporter.apache.org/trunk/scripts/rapp/overview.py (added)
+++ comdev/reporter.apache.org/trunk/scripts/rapp/overview.py Thu Aug  8 11:07:48 2019
@@ -0,0 +1,57 @@
+#!/usr/bin/env python3
+import os
+import cgi
+import json
+import pdata
+import time
+import re
+import committee_info
+
+CACHE_TIMEOUT = 14400
+
+def run(environ, user):
+    committers = pdata.loadJson(pdata.COMMITTER_INFO)['people']
+    pmcSummary = committee_info.PMCsummary()
+    project = environ.get('QUERY_STRING')
+    output = {'okay': False, 'error': 'Unknown user ID provided!'}
+    
+    dumps = {}
+    groups = []
+    if user:
+        groups = pdata.getPMCs(user)
+    if project and user and re.match(r"[-a-z0-9]+", project):
+        groups = [project]
+    
+    for xproject in groups:
+        
+         # Try cache first? (max 6 hours old)
+        wanted_file = "/tmp/pdata-%s.json" % xproject
+        if xproject == project:
+            wanted_file = "/tmp/pdata-kibbled-%s.json" % xproject
+        if (os.path.exists(wanted_file) and os.path.getmtime(wanted_file) > (time.time()
- CACHE_TIMEOUT)):
+            mpdata = json.load(open(wanted_file, "r"))
+        # If cache failed, generate fom scratch
+        else:
+            mpdata = pdata.generate(user, xproject, xproject == project)
+            if not mpdata:
+                break
+            open(wanted_file, "w").write(json.dumps(mpdata))
+        # Weave results into combined object, mindful of kibble data
+        for k, v in mpdata.items():
+            if k not in dumps:
+                dumps[k] = {}
+            if (k != 'kibble'):
+                dumps[k][xproject] = v
+            if k == 'kibble' and v:
+                dumps['kibble'] =v
+    
+    # Set personalized vars, dump
+    if dumps and user:
+        ddata, allpmcs, health = pdata.getProjectData()
+        dumps['you'] = committers[user]
+        dumps['all'] = sorted(allpmcs)
+        dumps['pmcs'] = sorted(groups)
+        dumps['pmcsummary'] = pmcSummary
+        output = dumps
+    
+    return output

Added: comdev/reporter.apache.org/trunk/scripts/rapp/whimsy.py
URL: http://svn.apache.org/viewvc/comdev/reporter.apache.org/trunk/scripts/rapp/whimsy.py?rev=1864689&view=auto
==============================================================================
--- comdev/reporter.apache.org/trunk/scripts/rapp/whimsy.py (added)
+++ comdev/reporter.apache.org/trunk/scripts/rapp/whimsy.py Thu Aug  8 11:07:48 2019
@@ -0,0 +1,164 @@
+#!/usr/bin/env python3
+# -*- coding: UTF-8 -*-
+""" script for publishing a report to whimsy """
+import os
+import sys
+import time
+import json
+import requests
+import re
+import pdata
+import committee_info
+
+WHIMSY_SUBMIT = 'https://whimsy.apache.org/board/agenda/json/post'
+WHIMSY_AGENDA = 'https://whimsy.apache.org/board/agenda/latest.json'
+WHIMSY_COMMENTS = 'https://whimsy.apache.org/board/agenda/json/historical-comments'
+
+def get_whimsy(url, env, ttl = 14400):
+    cached = True
+    xurl = re.sub(r"[^a-z0-9]+", "-", url.replace('.json', ''))
+    wanted_file = '/tmp/%s.json' % xurl
+    if pdata.has_cache(wanted_file, ttl = ttl):
+        js = json.load(open(wanted_file))
+    else:
+        try:
+            print("Fetching %s => %s..." % (url, wanted_file))
+            js = requests.get(url, headers = {'Authorization': env.get('HTTP_AUTHORIZATION')},
timeout = 5).json()
+            with open(wanted_file, "w") as f:
+                json.dump(js, f)
+                f.close()
+                cached = False
+        except: # fall back to cache on failure!
+            js = json.load(open(wanted_file))
+    
+    return js, cached
+
+def has_access(user, project):
+    member = pdata.isASFMember(user)
+    pmc = project in pdata.getPMCs(user)
+    return (member or pmc)
+
+def guess_title(project):
+    """ Guess the whimsy name of a project """
+    pmcSummary = committee_info.PMCsummary()
+    
+    # Figure out the name as written in whimsy..
+    pname = project
+    if project in pmcSummary:
+        pname = pmcSummary[project]['name'].replace('Apache ', '')
+    
+    return pname
+
+def agenda_forced(environ, user):
+    """ Force whimsy agenda refresh... """
+    get_whimsy(WHIMSY_AGENDA, environ, ttl = 0)
+    return agenda(environ, user)
+
+def agenda(environ, user):
+    """ Returns data on the board report for a project, IF present and/or filed in the current
agenda """
+    project = environ.get('QUERY_STRING')
+    report = None
+    if has_access(user, project):
+        agenda, cached = get_whimsy(WHIMSY_AGENDA, environ, ttl = 3600)
+        for entry in agenda:
+            ml = entry.get('mail_list') # mailing list id, usually correct
+            rid = entry.get('roster', '').replace('https://whimsy.apache.org/roster/committee/',
'') # ldap id per roster
+            if ml and (ml == project or rid == project):
+                report = entry
+                break
+        
+        comments, cached = get_whimsy(WHIMSY_COMMENTS, environ)
+        title = report and report.get('title') or guess_title(project)
+        if title in comments:
+            comments = comments[title]
+        else:
+            comments = {}
+        
+        return {
+            'can_access': True,
+            'found': report and True or False,
+            'filed': report and report.get('report') and True or False,
+            'report': report,
+            'comments': comments,
+            }
+    
+    return {
+        'can_access': False,
+        'found': False,
+        }
+    
+    
+    comments, cached = get_whimsy(WHIMSY_COMMENTS, environ)
+
+def comments(environ, user):
+    """ Display board feedback from previous reports ... """
+    project = environ.get('QUERY_STRING')
+    comments, cached = get_whimsy(WHIMSY_COMMENTS, environ)
+    
+    pmcSummary = committee_info.PMCsummary()
+    
+    # Figure out the name as written in whimsy..
+    pname = project
+    if project in pmcSummary:
+        pname = pmcSummary[project]['name'].replace('Apache ', '')
+    cmt = {}
+    
+    # If we can access, fetch comments
+    if comments and pname in comments and has_access(user, project):
+        comments = comments[pname]
+    else:
+        comments = {}
+
+    js = {
+        "pid": project,
+        "pname": pname,
+        "comments": comments,
+        "used_cache": cached,
+    }
+    return js
+
+
+def publish(environ, user):
+    try:
+        request_body_size = int(environ.get('CONTENT_LENGTH', 0))
+    except (ValueError):
+        request_body_size = 0
+    if request_body_size:
+        request_body = environ['wsgi.input'].read(request_body_size)
+        try:
+            js = json.loads(request_body.decode('utf-8'))
+        except:
+            js = {}
+    if js:
+        agenda = js.get('agenda')
+        project = js.get('project')
+        report = js.get('report')
+        digest = js.get('digest')
+        attach = js.get('attach')
+        print(project, agenda)
+        if agenda and project and report:
+            message = "Publishing report for %s via Reporter" % project
+            payload = {
+             'agenda': agenda,
+             'project': project,
+             'report': report,
+             'message': message,
+            }
+            if digest and attach:
+                del payload['project']
+                payload['attach'] = attach
+                payload['message'] = "Updating report for %s via Reporter." % project
+                payload['digest'] = digest
+            try:
+                rv = requests.post(WHIMSY_SUBMIT, headers = {
+                    'Authorization': environ.get('HTTP_AUTHORIZATION'),
+                    "Content-Type": "application/json"
+                    }, json = payload, timeout = 10)
+                rv.raise_for_status()
+                print(rv.text)
+                return {'okay': True, 'message': "Posted to board agenda!"}
+            except:
+                pass
+    return {}
+    
+    
\ No newline at end of file

Modified: comdev/reporter.apache.org/trunk/scripts/wsgi.py
URL: http://svn.apache.org/viewvc/comdev/reporter.apache.org/trunk/scripts/wsgi.py?rev=1864689&r1=1864688&r2=1864689&view=diff
==============================================================================
--- comdev/reporter.apache.org/trunk/scripts/wsgi.py (original)
+++ comdev/reporter.apache.org/trunk/scripts/wsgi.py Thu Aug  8 11:07:48 2019
@@ -1,62 +1,42 @@
 #!/usr/bin/env python2.7
-import os
-import cgi
 import json
-import pdata
 import time
+import base64
 import re
-import committee_info
 
 CACHE_TIMEOUT = 14400
 
+import rapp.overview
+import rapp.whimsy
+import rapp.drafts
+
+webmap = {
+    '/api/overview': rapp.overview.run,
+    '/api/whimsy/comments': rapp.whimsy.comments,
+    '/api/whimsy/agenda': rapp.whimsy.agenda,
+    '/api/whimsy/agenda/refresh': rapp.whimsy.agenda_forced,
+    '/api/whimsy/publish': rapp.whimsy.publish,
+    '/api/drafts/index': rapp.drafts.index,
+    '/api/drafts/save': rapp.drafts.save,
+    '/api/drafts/fetch': rapp.drafts.fetch,
+    '/api/drafts/delete': rapp.drafts.delete,
+}
 
 def app(environ, start_fn):
-    committers = pdata.loadJson(pdata.COMMITTER_INFO)['people']
-    pmcSummary = committee_info.PMCsummary()
-    project = environ.get('QUERY_STRING')
-    user = environ.get('HTTP_X_AUTHENTICATED_USER', 'humbedooh')
-    
-    output = {'okay': False, 'error': 'Unknown user ID provided!'}
-    
-    dumps = {}
-    groups = []
-    if user:
-        groups = pdata.getPMCs(user)
-    if project and user and re.match(r"[-a-z0-9]+", project):
-        groups = [project]
-    
-    for xproject in groups:
-        
-         # Try cache first? (max 6 hours old)
-        wanted_file = "/tmp/pdata-%s.json" % xproject
-        if xproject == project:
-            wanted_file = "/tmp/pdata-kibbled-%s.json" % xproject
-        if (os.path.exists(wanted_file) and os.path.getmtime(wanted_file) > (time.time()
- CACHE_TIMEOUT)):
-            mpdata = json.load(open(wanted_file, "r"))
-        # If cache failed, generate fom scratch
-        else:
-            mpdata = pdata.generate(user, xproject, xproject == project)
-            if not mpdata:
-                break
-            open(wanted_file, "w").write(json.dumps(mpdata))
-        # Weave results into combined object, mindful of kibble data
-        for k, v in mpdata.items():
-            if k not in dumps:
-                dumps[k] = {}
-            if (k != 'kibble'):
-                dumps[k][xproject] = v
-            if k == 'kibble' and v:
-                dumps['kibble'] =v
-    
-    # Set personalized vars, dump
-    if dumps and user:
-        ddata, allpmcs, health = pdata.getProjectData()
-        dumps['you'] = committers[user]
-        dumps['all'] = sorted(allpmcs)
-        dumps['pmcs'] = sorted(groups)
-        dumps['pmcsummary'] = pmcSummary
-        output = dumps
+    now = time.time()
+    bauth = re.match(r"Basic (.+)", environ.get('HTTP_AUTHORIZATION', 'foo'))
+    if bauth:
+        bdec = base64.b64decode(bauth.group(1)).decode('utf-8')
+        m = re.match("^(.+?):(.+)$", bdec)
+        if m:
+            user = m.group(1)
+    uri = environ.get('PATH_INFO', '/')
+    if uri in webmap:
+        output = webmap[uri](environ, user)
+    else:
+        output = {'okay': False, 'message': 'Unknown URI %s' % uri}
     
+    output['took'] = int((time.time() - now) * 1000)
     out = json.dumps(output, indent = 2, sort_keys = True).encode('ascii')
     start_fn('200 OK', [('Content-Type', 'application/json'), ('Content-Length', str(len(out)))])
     return [out]



Mime
View raw message