--- /dev/null Thu Jan 01 00:00:00 1970 +0000
+++ b/rl_ci_tools.py Wed Jun 14 16:36:46 2017 +0100
@@ -0,0 +1,196 @@
+VERSION='0.0.0'
+import os, sys, glob, time, json
+PROG=os.path.basename(sys.argv[0])
+debug=verbosity=0
+
+class PyPiRequestor():
+ scheme = 'https'
+ host = os.environ.get('CITOOLS_SERVER','www.reportlab.com')
+ root = '%s://%s' % (scheme,host)
+ loginurl = "%s/accounts/login/" % root
+ def __init__(self,debug=0):
+ self.debug = debug
+ import requests
+ self.session = requests.session()
+
+ def login(self,u,p,nxt='/test-7491/'):
+ s = self.session
+ resp = s.get(self.loginurl)
+ loginpage = resp.text
+ if self.debug>1:
+ if self.debug>2: print('=============loginpage\n%r' % loginpage)
+ resp = s.post(self.loginurl,
+ data=dict(
+ csrfmiddlewaretoken=s.cookies['csrftoken'],
+ username=u,
+ password=p,
+ next=nxt,
+ ),
+ headers=dict(
+ Referer=self.loginurl,
+ ),
+ )
+ text = resp.text
+ status_code = resp.status_code
+ if debug>2:
+ print('!!!!!\n%s\n!!!!!'% text)
+ print('%s: test-7491 csrftoken=%r' % (PROG,resp.cookies.get('csrftoken','???')))
+ if text!='I am alive!' or status_code!=200:
+ raise ValueError('%s: login at %r failed with status_code=%r' % (PROG,self.loginurl,status_code))
+ elif verbosity>=2:
+ print('%s: logged in OK' % PROG)
+ return status_code
+
+ def _download(self,u,p, kind, fn, dst):
+ self.login(u,p)
+ base = '%s/pypi/%s/' % (self.root,kind)
+ url = base + fn + '/'
+ resp = self.session.get(url,
+ data=dict(csrfmiddlewaretoken=self.session.cookies['csrftoken']),
+ headers = dict(Referer=self.loginurl),
+ )
+ status_code = resp.status_code
+ b = resp.content
+ if debug>2:
+ print('!!!!!\n%r\n!!!!!'% b)
+ if status_code!=200:
+ raise ValueError('%s: download %r failed with status_code=%r!\n%r' % (PROG,url,status_code,b))
+ if dst:
+ fn = os.path.join(dst,fn)
+ with open(fn,'wb') as f:
+ f.write(b)
+ if verbosity:
+ print('%s: %r(%d bytes) downloaded from %r.' % (PROG, fn, len(b),base))
+ return resp.status_code
+
+ def download(self,u,p,kind, fn, dst):
+ for i in self.info(u,p,kind[:-1],fn):
+ self._download(u,p,kind,i[0],dst)
+
+ def info(self,u,p,kind,pat):
+ #self.login(u,p)
+ url = '%s/pypi/%s-info/%s/?json=1' % (self.root,kind,pat)
+ resp = self.session.get(url,
+ #data=dict(csrfmiddlewaretoken=self.session.cookies['csrftoken']),
+ headers = dict(Referer=self.loginurl),
+ )
+ status_code = resp.status_code
+ b = resp.content
+ if debug>2:
+ print('!!!!!\n%r\n!!!!!'% b)
+ if status_code!=200:
+ raise ValueError('%s: request %r failed with status_code=%r!\n%r' % (PROG,url,status_code,b))
+ I = json.loads(b)
+ if verbosity>1:
+ print('%s: %r --> %d rows' % (PROG, url, len(I)))
+ return I
+
+ def upload(self,u,p,kind,fn):
+ self.login(u,p)
+ url = '%s/pypi/upload-%s/' % (self.root,kind)
+ files= dict(file=(os.path.basename(fn),open(fn,'rb'),'application/octet-stream'))
+ resp = self.session.post(url,
+ data=dict(csrfmiddlewaretoken=self.session.cookies['csrftoken']),
+ files=files,
+ headers = dict(Referer=self.loginurl),
+ )
+ status_code = resp.status_code
+ text = resp.text
+ if text!='OK' or status_code!=200:
+ raise ValueError('%s: upload %r failed with status_code=%r!\n%r' % (PROG,url,status_code,text))
+ if verbosity:
+ print('%s: uploaded %r to %r.' % (PROG,fn,url))
+ return resp.status_code
+
+def getoption(key,default=0,cnv=int):
+ key = '--%s=' % key
+ v = [x for x in sys.argv if x.startswith(key)]
+ if v:
+ for x in v:
+ sys.argv.remove(x)
+ v = cnv(v[-1][len(key):])
+ else:
+ v = default
+ return v
+
+def _file_info(fn):
+ st = os.stat(fn)
+ return (fn,st.st_size,st.st_mtime)
+
+def tabulate(I):
+ if I:
+ rows = [['Name','Length',(5*' ')+'Modified']]
+ for nm,sz,mt in I:
+ rows.append([nm,str(sz),time.strftime('%Y%m%d %H:%M:%S',time.localtime(mt))])
+ W = [max(map(len,col)) for col in [list(i) for i in zip(*rows)]]
+ if debug>3:
+ print('tabluate: rows=%s' % repr(rows))
+ print('tabluate: W=%s' % repr(W))
+ fmt = '{:<%d} {:>%d} {:<%d}' % tuple(W)
+ print('\n'.join(fmt.format(*i) for i in rows))
+
+def main():
+ u = os.environ.get('CITOOLS_USER','beta')
+ p = os.environ.get('CITOOLS_PASSWORD','???')
+ global debug, verbosity
+ debug = getoption('debug')
+ verbosity = getoption('verbosity')
+
+ if debug>3:
+ import logging
+
+ # These two lines enable debugging at httplib level (requests->urllib3->http.client)
+ # You will see the REQUEST, including HEADERS and DATA, and RESPONSE with HEADERS but without DATA.
+ # The only thing missing will be the response.body which is not logged.
+ try:
+ import http.client as http_client
+ except ImportError:
+ # Python 2
+ import httplib as http_client
+ http_client.HTTPConnection.debuglevel = 1
+
+ # You must initialize logging, otherwise you'll not see debug output.
+ logging.basicConfig()
+ logging.getLogger().setLevel(logging.DEBUG)
+ requests_log = logging.getLogger("requests.packages.urllib3")
+ requests_log.setLevel(logging.DEBUG)
+ requests_log.propagate = True
+
+ dst = getoption('dst',None,cnv=str)
+ try:
+ cmd = sys.argv[1]
+ except:
+ cmd = 'help'
+ pypi = PyPiRequestor(debug=debug)
+ if cmd=='test':
+ status_code = pypi.login(u,p)
+ if debug:
+ print('status=%s' % status_code)
+ elif cmd.startswith('download-'):
+ kind = cmd.split('-')[1]
+ if not kind in ('resources','packages'):
+ raise ValueError('%s: invalid download kind: %r' % (PROG,kind))
+ if dst and not os.path.isdir(dst):
+ raise ValueError('%s: %r is not a directory!' % (PROG,dst))
+ for fn in sys.argv[2:]:
+ pypi.download(u,p,kind,fn,dst)
+ elif cmd.endswith('-info'):
+ kind = cmd.split('-')[0]
+ if not kind in ('resource','package'):
+ raise ValueError('%s: invalid info kind: %r' % (PROG,kind))
+ tabulate([i for fn in sys.argv[2:] for i in pypi.info(u,p,kind,fn)])
+ elif cmd.startswith('upload-'):
+ kind = cmd.split('-')[1]
+ if not kind in ('resources','packages'):
+ raise ValueError('%s: invalid upload kind: %r' % (PROG,kind))
+ for pat in sys.argv[2:]:
+ for fn in glob.glob(pat):
+ pypi.upload(u,p,kind[:-1],fn)
+ elif cmd=='info':
+ tabulate([_file_info(i) for fn in sys.argv[2:] for i in glob.glob(fn)])
+ elif cmd=='help':
+ print('Usage %s [test|info|download-[resources|packages]|upload-[resources|packages]|[package|resource]-info] path....' % PROG)
+ else:
+ raise ValueError('%s: nknown command %r' % (PROG,cmd))
+if __name__=='__main__':
+ main()