community-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From rlenfer...@apache.org
Subject svn commit: r1881148 - /comdev/projects.apache.org/trunk/scripts/cronjobs/generaterepos.py
Date Mon, 24 Aug 2020 16:21:16 GMT
Author: rlenferink
Date: Mon Aug 24 16:21:16 2020
New Revision: 1881148

URL: http://svn.apache.org/viewvc?rev=1881148&view=rev
Log:
Added python script for generating a current repo list

Added:
    comdev/projects.apache.org/trunk/scripts/cronjobs/generaterepos.py

Added: comdev/projects.apache.org/trunk/scripts/cronjobs/generaterepos.py
URL: http://svn.apache.org/viewvc/comdev/projects.apache.org/trunk/scripts/cronjobs/generaterepos.py?rev=1881148&view=auto
==============================================================================
--- comdev/projects.apache.org/trunk/scripts/cronjobs/generaterepos.py (added)
+++ comdev/projects.apache.org/trunk/scripts/cronjobs/generaterepos.py Mon Aug 24 16:21:16
2020
@@ -0,0 +1,65 @@
+#!/usr/bin/python3
+
+"""
+
+Reads:
+https://svn.apache.org/repos/asf/
+https://gitbox.apache.org/repositories.json
+
+Updates:
+../../site/json/foundation/repositories.json
+
+"""
+
+import json
+import requests
+from html.parser import HTMLParser
+
+repos = {}
+
+class SVNRepoParser(HTMLParser):
+    handleProjectData = False
+
+    def handle_starttag(self, tag, attrs):
+        if tag == 'li':
+            self.handleProjectData = True
+
+    def handle_endtag(self, tag):
+        self.handleProjectData = False
+
+    def handle_data(self, data):
+        if self.handleProjectData:
+            committee = data.rstrip('/')
+            repos[committee + '-svn'] = 'https://svn.apache.org/repos/asf/' + committee +
'/'
+
+
+# Parse svn repos
+try:
+    svnResponse = requests.get("https://svn.apache.org/repos/asf/")
+    svnResponse.raise_for_status()
+
+    parser = SVNRepoParser()
+    parser.feed(svnResponse.content.decode("utf-8"))
+except requests.exceptions.RequestException as e:  # This is the correct syntax
+    print("ERROR: Unable to retrieve svn repos: %s", e)
+
+
+# Parse git repos
+try:
+    gitResponse = requests.get("https://gitbox.apache.org/repositories.json")
+    gitResponse.raise_for_status()
+    gitData = json.loads(gitResponse.content.decode("utf-8"))
+    
+    for committee in gitData['projects']:
+        for repo in gitData['projects'][committee]['repositories']:
+            repos[repo] = 'https://gitbox.apache.org/repos/asf/' + repo + '.git'
+except requests.exceptions.RequestException as e:  # This is the correct syntax
+    print("ERROR: Unable to retrieve git repos: %s", e)
+
+print("Writing json/foundation/repositories.json...")
+with open("../../site/json/foundation/repositories.json", "w") as f:
+    json.dump(repos, f, sort_keys=True, indent=0)
+    f.close()
+
+print("All done!")
+



Mime
View raw message