rl_ci_tools.py
author robin <robin@reportlab.com>
Wed, 14 Jun 2017 16:54:38 +0100
changeset 2 e30f155ae0db
parent 0 664ae0993bd1
child 3 dd3ccb935508
permissions -rw-r--r--
bump version to 0.0.1

VERSION='0.0.1'
import os, sys, glob, time, json
PROG=os.path.basename(sys.argv[0])
debug=verbosity=0

class PyPiRequestor():
    scheme = 'https'
    host = os.environ.get('CITOOLS_SERVER','www.reportlab.com')
    root = '%s://%s' % (scheme,host)
    loginurl = "%s/accounts/login/" % root
    def __init__(self,debug=0):
        self.debug = debug
        import requests
        self.session = requests.session()

    def login(self,u,p,nxt='/test-7491/'):
        s = self.session
        resp = s.get(self.loginurl)
        loginpage = resp.text
        if self.debug>1:
            if self.debug>2: print('=============loginpage\n%r' % loginpage)
        resp = s.post(self.loginurl,
                        data=dict(
                                csrfmiddlewaretoken=s.cookies['csrftoken'],
                                username=u,
                                password=p,
                                next=nxt,
                                ),
                        headers=dict(
                            Referer=self.loginurl,
                            ),
                        )
        text = resp.text
        status_code = resp.status_code
        if debug>2:
            print('!!!!!\n%s\n!!!!!'% text)
            print('%s: test-7491 csrftoken=%r' % (PROG,resp.cookies.get('csrftoken','???')))
        if text!='I am alive!' or status_code!=200:
            raise ValueError('%s: login at %r failed with status_code=%r' % (PROG,self.loginurl,status_code))
        elif verbosity>=2:
            print('%s: logged in OK' % PROG)
        return status_code

    def _download(self,u,p, kind, fn, dst):
        self.login(u,p)
        base = '%s/pypi/%s/' % (self.root,kind)
        url = base + fn + '/'
        resp = self.session.get(url,
                data=dict(csrfmiddlewaretoken=self.session.cookies['csrftoken']),
                headers = dict(Referer=self.loginurl),
                )
        status_code = resp.status_code
        b = resp.content
        if debug>2:
            print('!!!!!\n%r\n!!!!!'% b)
        if status_code!=200:
            raise ValueError('%s: download %r failed with status_code=%r!\n%r' % (PROG,url,status_code,b))
        if dst:
            fn = os.path.join(dst,fn)
        with open(fn,'wb') as f:
            f.write(b)
        if verbosity:
            print('%s: %r(%d bytes) downloaded from %r.' % (PROG, fn, len(b),base))
        return resp.status_code

    def download(self,u,p,kind, fn, dst):
        for i in self.info(u,p,kind[:-1],fn):
            self._download(u,p,kind,i[0],dst)

    def info(self,u,p,kind,pat):
        #self.login(u,p)
        url = '%s/pypi/%s-info/%s/?json=1' % (self.root,kind,pat)
        resp = self.session.get(url,
                #data=dict(csrfmiddlewaretoken=self.session.cookies['csrftoken']),
                headers = dict(Referer=self.loginurl),
                )
        status_code = resp.status_code
        b = resp.content
        if debug>2:
            print('!!!!!\n%r\n!!!!!'% b)
        if status_code!=200:
            raise ValueError('%s: request %r failed with status_code=%r!\n%r' % (PROG,url,status_code,b))
        I = json.loads(b)
        if verbosity>1:
            print('%s: %r --> %d rows' % (PROG, url, len(I)))
        return I

    def upload(self,u,p,kind,fn):
        self.login(u,p)
        url = '%s/pypi/upload-%s/' % (self.root,kind)
        files= dict(file=(os.path.basename(fn),open(fn,'rb'),'application/octet-stream'))
        resp = self.session.post(url,
                data=dict(csrfmiddlewaretoken=self.session.cookies['csrftoken']),
                files=files,
                headers = dict(Referer=self.loginurl),
                )
        status_code = resp.status_code
        text = resp.text
        if text!='OK' or status_code!=200:
            raise ValueError('%s: upload %r failed with status_code=%r!\n%r' % (PROG,url,status_code,text))
        if verbosity:
            print('%s: uploaded %r to %r.' % (PROG,fn,url))
        return resp.status_code

def getoption(key,default=0,cnv=int):
    key = '--%s=' % key
    v = [x for x in sys.argv if x.startswith(key)]
    if v:
        for x in v:
            sys.argv.remove(x)
        v = cnv(v[-1][len(key):])
    else:
        v = default
    return v

def _file_info(fn):
    st = os.stat(fn)
    return (fn,st.st_size,st.st_mtime)

def tabulate(I):
    if I:
        rows = [['Name','Length',(5*' ')+'Modified']]
        for nm,sz,mt in I:
            rows.append([nm,str(sz),time.strftime('%Y%m%d %H:%M:%S',time.localtime(mt))])
        W = [max(map(len,col)) for col in [list(i) for i in zip(*rows)]]
        if debug>3:
            print('tabluate: rows=%s' % repr(rows))
            print('tabluate: W=%s' % repr(W))
        fmt = '{:<%d} {:>%d}    {:<%d}' % tuple(W)
        print('\n'.join(fmt.format(*i) for i in rows))

def main():
    u = os.environ.get('CITOOLS_USER','beta')
    p = os.environ.get('CITOOLS_PASSWORD','???')
    global debug, verbosity
    debug = getoption('debug')
    verbosity = getoption('verbosity')

    if debug>3:
        import logging

        # These two lines enable debugging at httplib level (requests->urllib3->http.client)
        # You will see the REQUEST, including HEADERS and DATA, and RESPONSE with HEADERS but without DATA.
        # The only thing missing will be the response.body which is not logged.
        try:
            import http.client as http_client
        except ImportError:
            # Python 2
            import httplib as http_client
        http_client.HTTPConnection.debuglevel = 1

        # You must initialize logging, otherwise you'll not see debug output.
        logging.basicConfig()
        logging.getLogger().setLevel(logging.DEBUG)
        requests_log = logging.getLogger("requests.packages.urllib3")
        requests_log.setLevel(logging.DEBUG)
        requests_log.propagate = True

    dst = getoption('dst',None,cnv=str)
    try:
        cmd = sys.argv[1]
    except:
        cmd = 'help'
    pypi = PyPiRequestor(debug=debug)
    if cmd=='test':
        status_code = pypi.login(u,p)
        if debug:
            print('status=%s' % status_code)
    elif cmd.startswith('download-'):
        kind = cmd.split('-')[1]
        if not kind in ('resources','packages'):
            raise ValueError('%s: invalid download kind: %r' % (PROG,kind))
        if dst and not os.path.isdir(dst):
            raise ValueError('%s: %r is not a directory!' % (PROG,dst))
        for fn in sys.argv[2:]:
            pypi.download(u,p,kind,fn,dst)
    elif cmd.endswith('-info'):
        kind = cmd.split('-')[0]
        if not kind in ('resource','package'):
            raise ValueError('%s: invalid info kind: %r' % (PROG,kind))
        tabulate([i for fn in sys.argv[2:] for i in pypi.info(u,p,kind,fn)])
    elif cmd.startswith('upload-'):
        kind = cmd.split('-')[1]
        if not kind in ('resources','packages'):
            raise ValueError('%s: invalid upload kind: %r' % (PROG,kind))
        for pat in sys.argv[2:]:
            for fn in glob.glob(pat):
                pypi.upload(u,p,kind[:-1],fn)
    elif cmd=='info':
        tabulate([_file_info(i) for fn in sys.argv[2:] for i in glob.glob(fn)])
    elif cmd=='help':
        print('Usage %s [test|info|download-[resources|packages]|upload-[resources|packages]|[package|resource]-info] path....' % PROG)
    else:
        raise ValueError('%s: nknown command %r' % (PROG,cmd))
if __name__=='__main__':
    main()