| author | robin |
| Fri, 23 Jun 2017 12:15:17 +0100 | |
| changeset 10 | 863dc27e003d |
| parent 9 | 55f05d532826 |
| child 11 | 138feabe2772 |
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/.hgeol Fri Jun 23 12:15:17 2017 +0100 @@ -0,0 +1,44 @@ +[repository] +native = LF + +[patterns] +**.cgi=native +**.pas=native +**.ini=native +**.in=native +**.css=native +**.htm=native +**.html=native +**.xml=native +**.rml=native +**.prep=native +**.js=native +**.tcl=native +**.py=native +**.pyw=native +**.pyx=native +**.tac=native +**.tap=native +**.c=native +**.cpp=native +**.h=native +**.sh=native +**.java=native +**.pl=native +**.asp=native +**.jsp=native +**.cat=native +**.bat=native +**.dtd=native +**/Makefile=native +**.txt=native +**.patch=native +**.dsp=CRLF +**.dsw=CRLF +**.png=BIN +**.jpg=BIN +**.jpeg=BIN +**.gif=BIN +**.bmp=BIN +**.pdf=BIN +**.eps=BIN
--- a/manylinux/build-wheels Fri Jun 23 11:50:00 2017 +0100 +++ b/manylinux/build-wheels Fri Jun 23 12:15:17 2017 +0100 @@ -7,6 +7,7 @@ REPO=${REPO:-https://bitbucket.org/rptlab/reportlab} REQUIREMENT=${REPO:-$(basename ${REPO})} DOCKER_SCRIPT=${DOCKER_SCRIPT:-container-build-wheels} +DOCKER_PATH=${DOCKER_PATH:-/io/rl_ci_tools/manylinux/${DOCKER_SCRIPT}} #IMAGESRC=rl sudo rm -rf wheels wheelsu mkdir wheels wheelsu @@ -16,10 +17,15 @@ DOCKER_IMAGE=${IMAGESRC}/manylinux1_${arch} docker pull $DOCKER_IMAGE docker run --rm \ + ${DOCKER_ARGS} \ -e PYTHON_VERSIONS="$PYTHON_VERSIONS" \ -e UNICODE_WIDTHS="$UNICODE_WIDTHS" \ -e REPO="$REPO" \ - -e REEQUIREMENT="$REQUIREMENT" \ + -e REQUIREMENT="$REQUIREMENT" \ + -e DOCKER_PATH="$DOCKER_PATH" \ -e ARCH="$arch" \ - -v $(pwd):/io ${DOCKER_IMAGE} /io/rl_ci_tools/manylinux/${DOCKER_SCRIPT} + -v $(pwd):/io ${DOCKER_IMAGE} ${DOCKER_PATH} + cp wheelhouse/* wheels/ + cp wheels_unfixed/* wheelsu/ done +sudo rm -rf wheels_unfixed wheelhouse
--- a/manylinux/common_utils.sh Fri Jun 23 11:50:00 2017 +0100 +++ b/manylinux/common_utils.sh Fri Jun 23 12:15:17 2017 +0100 @@ -1,200 +1,200 @@ -#!/bin/bash -# Utilities for both OSX and Docker Linux -# Python should be on the PATH -set -e - -MULTIBUILD_DIR=$(dirname "${BASH_SOURCE[0]}") -if [ $(uname) == "Darwin" ]; then IS_OSX=1; fi - -function abspath { - python -c "import os.path; print(os.path.abspath('$1'))" -} - -function relpath { - # Path of first input relative to second (or $PWD if not specified) - python -c "import os.path; print(os.path.relpath('$1','${2:-$PWD}'))" -} - -function realpath { - python -c "import os; print(os.path.realpath('$1'))" -} - -function lex_ver { - # Echoes dot-separated version string padded with zeros - # Thus: - # 3.2.1 -> 003002001 - # 3 -> 003000000 - echo $1 | awk -F "." '{printf "%03d%03d%03d", $1, $2, $3}' -} - -function unlex_ver { - # Reverses lex_ver to produce major.minor.micro - # Thus: - # 003002001 -> 3.2.1 - # 003000000 -> 3.0.0 - echo "$((10#${1:0:3}+0)).$((10#${1:3:3}+0)).$((10#${1:6:3}+0))" -} - -function strip_ver_suffix { - echo $(unlex_ver $(lex_ver $1)) -} - -function is_function { - # Echo "true" if input argument string is a function - # Allow errors during "set -e" blocks. - (set +e; echo $($(declare -Ff "$1") > /dev/null && echo true)) -} - -function gh-clone { - git clone https://github.com/$1 -} - -function rm_mkdir { - # Remove directory if present, then make directory - local path=$1 - if [ -z "$path" ]; then echo "Need not-empty path"; exit 1; fi - if [ -d "$path" ]; then rm -rf $path; fi - mkdir $path -} - -function untar { - local in_fname=$1 - if [ -z "$in_fname" ];then echo "in_fname not defined"; exit 1; fi - local extension=${in_fname##*.} - case $extension in - tar) tar xf $in_fname ;; - gz|tgz) tar zxf $in_fname ;; - bz2) tar jxf $in_fname ;; - zip) unzip $in_fname ;; - xz) unxz -c $in_fname | tar xf ;; - *) echo Did not recognize extension $extension; exit 1 ;; - esac -} - -function fetch_unpack { - # Fetch input archive name from input URL - # Parameters - # url - URL from which to fetch archive - # archive_fname (optional) archive name - # - # If `archive_fname` not specified then use basename from `url` - # If `archive_fname` already present at download location, use that instead. - local url=$1 - if [ -z "$url" ];then echo "url not defined"; exit 1; fi - local archive_fname=${2:-$(basename $url)} - local arch_sdir="${ARCHIVE_SDIR:-archives}" - # Make the archive directory in case it doesn't exist - mkdir -p $arch_sdir - local out_archive="${arch_sdir}/${archive_fname}" - # Fetch the archive if it does not exist - if [ ! -f "$out_archive" ]; then - curl -L $url > $out_archive - fi - # Unpack archive, refreshing contents - rm_mkdir arch_tmp - (cd arch_tmp && untar ../$out_archive && rsync --delete -avh * ..) -} - -function clean_code { - local repo_dir=${1:-$REPO_DIR} - local build_commit=${2:-$BUILD_COMMIT} - [ -z "$repo_dir" ] && echo "repo_dir not defined" && exit 1 - [ -z "$build_commit" ] && echo "build_commit not defined" && exit 1 - (cd $repo_dir \ - && git fetch origin \ - && git checkout $build_commit \ - && git clean -fxd \ - && git reset --hard \ - && git submodule update --init --recursive) -} - -function build_wheel_cmd { - # Builds wheel with named command, puts into $WHEEL_SDIR - # - # Parameters: - # cmd (optional, default "pip_wheel_cmd" - # Name of command for builing wheel - # repo_dir (optional, default $REPO_DIR) - # - # Depends on - # REPO_DIR (or via input argument) - # WHEEL_SDIR (optional, default "wheelhouse") - # BUILD_DEPENDS (optional, default "") - # MANYLINUX_URL (optional, default "") (via pip_opts function) - local cmd=${1:-pip_wheel_cmd} - local repo_dir=${2:-$REPO_DIR} - [ -z "$repo_dir" ] && echo "repo_dir not defined" && exit 1 - local wheelhouse=$(abspath ${WHEEL_SDIR:-wheelhouse}) - if [ -n "$(is_function "pre_build")" ]; then pre_build; fi - if [ -n "$BUILD_DEPENDS" ]; then - pip install $(pip_opts) $BUILD_DEPENDS - fi - (cd $repo_dir && $cmd $wheelhouse) - repair_wheelhouse $wheelhouse -} - -function pip_wheel_cmd { - local abs_wheelhouse=$1 - pip wheel $(pip_opts) -w $abs_wheelhouse --no-deps . -} - -function bdist_wheel_cmd { - # Builds wheel with bdist_wheel, puts into wheelhouse - # - # It may sometimes be useful to use bdist_wheel for the wheel building - # process. For example, versioneer has problems with versions which are - # fixed with bdist_wheel: - # https://github.com/warner/python-versioneer/issues/121 - local abs_wheelhouse=$1 - python setup.py bdist_wheel - cp dist/*.whl $abs_wheelhouse -} - -function build_pip_wheel { - # Standard wheel building command with pip wheel - build_wheel_cmd "pip_wheel_cmd" $@ -} - -function build_bdist_wheel { - # Wheel building with bdist_wheel. See bdist_wheel_cmd - build_wheel_cmd "bdist_wheel_cmd" $@ -} - -function build_wheel { - # Set default building method to pip - build_pip_wheel $@ -} - -function pip_opts { - [ -n "$MANYLINUX_URL" ] && echo "--find-links $MANYLINUX_URL" -} - -function get_platform { - # Report platform as given by uname - python -c 'import platform; print(platform.uname()[4])' -} - -function install_wheel { - # Install test dependencies and built wheel - # - # Pass any input flags to pip install steps - # - # Depends on: - # WHEEL_SDIR (optional, default "wheelhouse") - # TEST_DEPENDS (optional, default "") - # MANYLINUX_URL (optional, default "") (via pip_opts function) - local wheelhouse=$(abspath ${WHEEL_SDIR:-wheelhouse}) - if [ -n "$TEST_DEPENDS" ]; then - pip install $(pip_opts) $@ $TEST_DEPENDS - fi - # Install compatible wheel - pip install $(pip_opts) $@ \ - $(python $MULTIBUILD_DIR/supported_wheels.py $wheelhouse/*.whl) -} - -function install_run { - # Depends on function `run_tests` defined in `config.sh` - install_wheel - mkdir tmp_for_test - (cd tmp_for_test && run_tests) -} +#!/bin/bash +# Utilities for both OSX and Docker Linux +# Python should be on the PATH +set -e + +MULTIBUILD_DIR=$(dirname "${BASH_SOURCE[0]}") +if [ $(uname) == "Darwin" ]; then IS_OSX=1; fi + +function abspath { + python -c "import os.path; print(os.path.abspath('$1'))" +} + +function relpath { + # Path of first input relative to second (or $PWD if not specified) + python -c "import os.path; print(os.path.relpath('$1','${2:-$PWD}'))" +} + +function realpath { + python -c "import os; print(os.path.realpath('$1'))" +} + +function lex_ver { + # Echoes dot-separated version string padded with zeros + # Thus: + # 3.2.1 -> 003002001 + # 3 -> 003000000 + echo $1 | awk -F "." '{printf "%03d%03d%03d", $1, $2, $3}' +} + +function unlex_ver { + # Reverses lex_ver to produce major.minor.micro + # Thus: + # 003002001 -> 3.2.1 + # 003000000 -> 3.0.0 + echo "$((10#${1:0:3}+0)).$((10#${1:3:3}+0)).$((10#${1:6:3}+0))" +} + +function strip_ver_suffix { + echo $(unlex_ver $(lex_ver $1)) +} + +function is_function { + # Echo "true" if input argument string is a function + # Allow errors during "set -e" blocks. + (set +e; echo $($(declare -Ff "$1") > /dev/null && echo true)) +} + +function gh-clone { + git clone https://github.com/$1 +} + +function rm_mkdir { + # Remove directory if present, then make directory + local path=$1 + if [ -z "$path" ]; then echo "Need not-empty path"; exit 1; fi + if [ -d "$path" ]; then rm -rf $path; fi + mkdir $path +} + +function untar { + local in_fname=$1 + if [ -z "$in_fname" ];then echo "in_fname not defined"; exit 1; fi + local extension=${in_fname##*.} + case $extension in + tar) tar xf $in_fname ;; + gz|tgz) tar zxf $in_fname ;; + bz2) tar jxf $in_fname ;; + zip) unzip $in_fname ;; + xz) unxz -c $in_fname | tar xf ;; + *) echo Did not recognize extension $extension; exit 1 ;; + esac +} + +function fetch_unpack { + # Fetch input archive name from input URL + # Parameters + # url - URL from which to fetch archive + # archive_fname (optional) archive name + # + # If `archive_fname` not specified then use basename from `url` + # If `archive_fname` already present at download location, use that instead. + local url=$1 + if [ -z "$url" ];then echo "url not defined"; exit 1; fi + local archive_fname=${2:-$(basename $url)} + local arch_sdir="${ARCHIVE_SDIR:-archives}" + # Make the archive directory in case it doesn't exist + mkdir -p $arch_sdir + local out_archive="${arch_sdir}/${archive_fname}" + # Fetch the archive if it does not exist + if [ ! -f "$out_archive" ]; then + curl -L $url > $out_archive + fi + # Unpack archive, refreshing contents + rm_mkdir arch_tmp + (cd arch_tmp && untar ../$out_archive && rsync --delete -avh * ..) +} + +function clean_code { + local repo_dir=${1:-$REPO_DIR} + local build_commit=${2:-$BUILD_COMMIT} + [ -z "$repo_dir" ] && echo "repo_dir not defined" && exit 1 + [ -z "$build_commit" ] && echo "build_commit not defined" && exit 1 + (cd $repo_dir \ + && git fetch origin \ + && git checkout $build_commit \ + && git clean -fxd \ + && git reset --hard \ + && git submodule update --init --recursive) +} + +function build_wheel_cmd { + # Builds wheel with named command, puts into $WHEEL_SDIR + # + # Parameters: + # cmd (optional, default "pip_wheel_cmd" + # Name of command for builing wheel + # repo_dir (optional, default $REPO_DIR) + # + # Depends on + # REPO_DIR (or via input argument) + # WHEEL_SDIR (optional, default "wheelhouse") + # BUILD_DEPENDS (optional, default "") + # MANYLINUX_URL (optional, default "") (via pip_opts function) + local cmd=${1:-pip_wheel_cmd} + local repo_dir=${2:-$REPO_DIR} + [ -z "$repo_dir" ] && echo "repo_dir not defined" && exit 1 + local wheelhouse=$(abspath ${WHEEL_SDIR:-wheelhouse}) + if [ -n "$(is_function "pre_build")" ]; then pre_build; fi + if [ -n "$BUILD_DEPENDS" ]; then + pip install $(pip_opts) $BUILD_DEPENDS + fi + (cd $repo_dir && $cmd $wheelhouse) + repair_wheelhouse $wheelhouse +} + +function pip_wheel_cmd { + local abs_wheelhouse=$1 + pip wheel $(pip_opts) -w $abs_wheelhouse --no-deps . +} + +function bdist_wheel_cmd { + # Builds wheel with bdist_wheel, puts into wheelhouse + # + # It may sometimes be useful to use bdist_wheel for the wheel building + # process. For example, versioneer has problems with versions which are + # fixed with bdist_wheel: + # https://github.com/warner/python-versioneer/issues/121 + local abs_wheelhouse=$1 + python setup.py bdist_wheel + cp dist/*.whl $abs_wheelhouse +} + +function build_pip_wheel { + # Standard wheel building command with pip wheel + build_wheel_cmd "pip_wheel_cmd" $@ +} + +function build_bdist_wheel { + # Wheel building with bdist_wheel. See bdist_wheel_cmd + build_wheel_cmd "bdist_wheel_cmd" $@ +} + +function build_wheel { + # Set default building method to pip + build_pip_wheel $@ +} + +function pip_opts { + [ -n "$MANYLINUX_URL" ] && echo "--find-links $MANYLINUX_URL" +} + +function get_platform { + # Report platform as given by uname + python -c 'import platform; print(platform.uname()[4])' +} + +function install_wheel { + # Install test dependencies and built wheel + # + # Pass any input flags to pip install steps + # + # Depends on: + # WHEEL_SDIR (optional, default "wheelhouse") + # TEST_DEPENDS (optional, default "") + # MANYLINUX_URL (optional, default "") (via pip_opts function) + local wheelhouse=$(abspath ${WHEEL_SDIR:-wheelhouse}) + if [ -n "$TEST_DEPENDS" ]; then + pip install $(pip_opts) $@ $TEST_DEPENDS + fi + # Install compatible wheel + pip install $(pip_opts) $@ \ + $(python $MULTIBUILD_DIR/supported_wheels.py $wheelhouse/*.whl) +} + +function install_run { + # Depends on function `run_tests` defined in `config.sh` + install_wheel + mkdir tmp_for_test + (cd tmp_for_test && run_tests) +}
--- a/manylinux/container-build-wheels Fri Jun 23 11:50:00 2017 +0100 +++ b/manylinux/container-build-wheels Fri Jun 23 12:15:17 2017 +0100 @@ -1,72 +1,72 @@ -#!/bin/bash -# Run with: -# docker run --rm -v $PWD:/io quay.io/pypa/manylinux1_x86_64 /io/rl_ci_tools/manylinux/reportlab-wheels.sh -# or something like: -# docker run --rm -e PYTHON_VERSIONS=2.7 -v $PWD:/io quay.io/pypa/manylinux1_x86_64 /io/rl_ci_tools/manylinux/reportlab-wheels.sh -# or: -# docker run --rm -e PYTHON_VERSIONS=2.7 -v $PWD:/io quay.io/pypa/manylinux1_x86_64 /io/rl_ci_tools/manylinux/reportlab-wheels.sh -set -e - -UNICODE_WIDTHS=${UNICODE_WIDTHS:-16 32} -WHEELHOUSE=${WHEELHOUSE:-/io/rl_ci_tools/manylinux/wheelhouse} -WHEELS_UNFIXED=${WHEELS_UNFIXED:-/io/rl_ci_tools/manylinux/wheels_unfixed} - -mark(){ - echo "######################################################################################" - [ "$#" -gt 0 ] && echo "$@" && echo "######################################################################################" - } - -# Manylinux, openblas version, lex_ver etc etc -mark source /io/rl_ci_tools/manylinux/manylinux_utils.sh -source /io/rl_ci_tools/manylinux/manylinux_utils.sh -mark source /io/rl_ci_tools/manylinux/library_builders.sh -source /io/rl_ci_tools/manylinux/library_builders.sh -mark "$(env)" - -#mark build_jpeg -#build_jpeg -#mark build_tiff -#build_tiff -#mark build_openjpeg -#build_openjpeg -#mark build_lcsm2 -#build_lcms2 -#mark build_libwebp -#build_libwebp -mark build_freetype -build_freetype -#yum install -y tk-devel -#yum install mercurial - -PYLO=${PYLO:-2.7} -PYHI=${PYHI:-3.7} - -# Directory to store wheels -rm_mkdir $(basename "${WHEELS_UNFIXED}") - -OPATH="$PATH" -export RL_MANYLINUX=1 - -# Compile wheels -for PYTHON in ${PYTHON_VERSIONS}; do - [ $(lex_ver $PYTHON) -lt $(lex_ver $PYLO) ] && continue - [ $(lex_ver $PYTHON) -ge $(lex_ver $PYHI) ] && continue - for uw in ${UNICODE_WIDTHS}; do - [ $(lex_ver $PYTHON) -ge $(lex_ver 3.3) ] && [ "$uw" != 32 ] && continue - ( - export UNICODE_WIDTH="$uw" - mark "building reportlab wheel PYTHON=$PYTHON UNICODE_WIDTH=$UNICODE_WIDTH" - PP="$(cpython_path $PYTHON $UNICODE_WIDTH)" - PY=$PP/bin/python - mark "platform=$($PY -mplatform) sys.platform=$($PY -c'import sys;print(sys.platform)') os.name=$($PY -c'import os;print(os.name)')" - export PATH="$PP/bin:$OPATH" - PIP="$PP/bin/pip" - $PIP install wheel setuptools -U - echo "Building reportlab for Python $PYTHON" - $PIP wheel --no-binary=:all: --no-deps -w "${WHEELS_UNFIXED}" -e "${REPO}#egg=${REQUIREMENT}" - ) - done -done - -# Bundle external shared libraries into the wheels -repair_wheelhouse "${WHEELS_UNFIXED}" "${WHEELHOUSE}" +#!/bin/bash +# Run with: +# docker run --rm -v $PWD:/io quay.io/pypa/manylinux1_x86_64 /io/rl_ci_tools/manylinux/reportlab-wheels.sh +# or something like: +# docker run --rm -e PYTHON_VERSIONS=2.7 -v $PWD:/io quay.io/pypa/manylinux1_x86_64 /io/rl_ci_tools/manylinux/reportlab-wheels.sh +# or: +# docker run --rm -e PYTHON_VERSIONS=2.7 -v $PWD:/io quay.io/pypa/manylinux1_x86_64 /io/rl_ci_tools/manylinux/reportlab-wheels.sh +set -e + +UNICODE_WIDTHS=${UNICODE_WIDTHS:-16 32} +WHEELHOUSE=${WHEELHOUSE:-/io/rl_ci_tools/manylinux/wheelhouse} +WHEELS_UNFIXED=${WHEELS_UNFIXED:-/io/rl_ci_tools/manylinux/wheels_unfixed} + +mark(){ + echo "######################################################################################" + [ "$#" -gt 0 ] && echo "$@" && echo "######################################################################################" + } + +# Manylinux, openblas version, lex_ver etc etc +mark source /io/rl_ci_tools/manylinux/manylinux_utils.sh +source /io/rl_ci_tools/manylinux/manylinux_utils.sh +mark source /io/rl_ci_tools/manylinux/library_builders.sh +source /io/rl_ci_tools/manylinux/library_builders.sh +mark "$(env)" + +#mark build_jpeg +#build_jpeg +#mark build_tiff +#build_tiff +#mark build_openjpeg +#build_openjpeg +#mark build_lcsm2 +#build_lcms2 +#mark build_libwebp +#build_libwebp +mark build_freetype +build_freetype +#yum install -y tk-devel +#yum install mercurial + +PYLO=${PYLO:-2.7} +PYHI=${PYHI:-3.7} + +# Directory to store wheels +rm_mkdir $(basename "${WHEELS_UNFIXED}") + +OPATH="$PATH" +export RL_MANYLINUX=1 + +# Compile wheels +for PYTHON in ${PYTHON_VERSIONS}; do + [ $(lex_ver $PYTHON) -lt $(lex_ver $PYLO) ] && continue + [ $(lex_ver $PYTHON) -ge $(lex_ver $PYHI) ] && continue + for uw in ${UNICODE_WIDTHS}; do + [ $(lex_ver $PYTHON) -ge $(lex_ver 3.3) ] && [ "$uw" != 32 ] && continue + ( + export UNICODE_WIDTH="$uw" + mark "building reportlab wheel PYTHON=$PYTHON UNICODE_WIDTH=$UNICODE_WIDTH" + PP="$(cpython_path $PYTHON $UNICODE_WIDTH)" + PY=$PP/bin/python + mark "platform=$($PY -mplatform) sys.platform=$($PY -c'import sys;print(sys.platform)') os.name=$($PY -c'import os;print(os.name)')" + export PATH="$PP/bin:$OPATH" + PIP="$PP/bin/pip" + $PIP install wheel setuptools -U + echo "Building reportlab for Python $PYTHON" + $PIP wheel --no-binary=:all: --no-deps -w "${WHEELS_UNFIXED}" -e "${REPO}#egg=${REQUIREMENT}" + ) + done +done + +# Bundle external shared libraries into the wheels +repair_wheelhouse "${WHEELS_UNFIXED}" "${WHEELHOUSE}"
--- a/manylinux/container-test-wheels Fri Jun 23 11:50:00 2017 +0100 +++ b/manylinux/container-test-wheels Fri Jun 23 12:15:17 2017 +0100 @@ -1,81 +1,81 @@ -#!/bin/bash -# Run with: -# docker run --rm -v $PWD:/io quay.io/pypa/manylinux1_x86_64 /io/build_reportlab.sh -# or something like: -# docker run --rm -e PYTHON_VERSIONS=2.7 -v $PWD:/io quay.io/pypa/manylinux1_x86_64 /io/build_reportlab.sh -# or: -# docker run --rm -e PYTHON_VERSIONS=2.7 -v $PWD:/io quay.io/pypa/manylinux1_x86_64 /io/build_reportlab.sh -set -e - -UNICODE_WIDTHS=${UNICODE_WIDTHS:-16 32} -WHEELHOUSE=${WHEELHOUSE:-/io/wheelhouse} -WHEELS_UNFIXED=${WHEELS_UNFIXED:-/io/wheels_unfixed} - -mark(){ - echo "######################################################################################" - [ "$#" -gt 0 ] && echo "$@" && echo "######################################################################################" - } - -# Manylinux, openblas version, lex_ver etc etc -mark source /io/rl_ci_utils/manylinux/manylinux_utils.sh -source /io/rl_ci_utils/manylinux/manylinux_utils.sh -mark source /io/rl_ci_utils/manylinux/library_builders.sh -source /io/rl_ci_utils/manylinux/library_builders.sh -mark "$(env)" - -#mark build_jpeg -#build_jpeg -#mark build_tiff -#build_tiff -#mark build_openjpeg -#build_openjpeg -#mark build_lcsm2 -#build_lcms2 -#mark build_libwebp -#build_libwebp -#mark build_freetype -#build_freetype -#yum install -y tk-devel - -PYLO=${PYLO:-2.7} -PYHI=${PYHI:-3.7} - -OPATH="$PATH" -export RL_MANYLINUX=1 -ENV=/io/test-env -SRC=/io/${REQUIREMENT}-src - -# Compile wheels -for PYTHON in ${PYTHON_VERSIONS}; do - [ $(lex_ver $PYTHON) -lt $(lex_ver $PYLO) ] && continue - [ $(lex_ver $PYTHON) -ge $(lex_ver $PYHI) ] && continue - for uw in ${UNICODE_WIDTHS}; do - [ $(lex_ver $PYTHON) -ge $(lex_ver 3.3) ] && [ "$uw" != 32 ] && continue - ( - export UNICODE_WIDTH="$uw" - mark "building reportlab wheel PYTHON=$PYTHON UNICODE_WIDTH=$UNICODE_WIDTH" - PP="$(cpython_path $PYTHON $UNICODE_WIDTH)" - PY=$PP/bin/python - mark "platform=$($PY -mplatform) sys.platform=$($PY -c'import sys;print(sys.platform)') os.name=$($PY -c'import os;print(os.name)')" - export PATH="$PP/bin:$OPATH" - PIP="$PP/bin/pip" - echo "Testing ${REQUIREMENT} for Python $PYTHON" - $PIP install virtualenv - rm -rf $ENV - $PY -mvirtualenv $ENV - ( - cd $ENV - . $ENV/bin/activate - EPY=$ENV/bin/python - EPIP=$ENV/bin/pip - [ "${REQUIREMENT}" = "reportlab" ] && $EPIP install --find-links=/io/TEST-WHEELS pillow - $EPIP install --find-links=/io/wheels ${REQUIREMENT} - rm -f /tmp/eee - $EPY setup.py test | tee /tmp/eee - grep -q 'OK' /tmp/eee && R="##### OK " || R="!!!!! FAIL" - echo "${R} PYTHON=${PYTHON} UNICODE_WIDTH=${UNICODE_WIDTH} ARCH=${ARCH}" >> /io/test-results.txt - deactivate - ) - ) - done -done +#!/bin/bash +# Run with: +# docker run --rm -v $PWD:/io quay.io/pypa/manylinux1_x86_64 /io/build_reportlab.sh +# or something like: +# docker run --rm -e PYTHON_VERSIONS=2.7 -v $PWD:/io quay.io/pypa/manylinux1_x86_64 /io/build_reportlab.sh +# or: +# docker run --rm -e PYTHON_VERSIONS=2.7 -v $PWD:/io quay.io/pypa/manylinux1_x86_64 /io/build_reportlab.sh +set -e + +UNICODE_WIDTHS=${UNICODE_WIDTHS:-16 32} +WHEELHOUSE=${WHEELHOUSE:-/io/wheelhouse} +WHEELS_UNFIXED=${WHEELS_UNFIXED:-/io/wheels_unfixed} + +mark(){ + echo "######################################################################################" + [ "$#" -gt 0 ] && echo "$@" && echo "######################################################################################" + } + +# Manylinux, openblas version, lex_ver etc etc +mark source /io/rl_ci_utils/manylinux/manylinux_utils.sh +source /io/rl_ci_utils/manylinux/manylinux_utils.sh +mark source /io/rl_ci_utils/manylinux/library_builders.sh +source /io/rl_ci_utils/manylinux/library_builders.sh +mark "$(env)" + +#mark build_jpeg +#build_jpeg +#mark build_tiff +#build_tiff +#mark build_openjpeg +#build_openjpeg +#mark build_lcsm2 +#build_lcms2 +#mark build_libwebp +#build_libwebp +#mark build_freetype +#build_freetype +#yum install -y tk-devel + +PYLO=${PYLO:-2.7} +PYHI=${PYHI:-3.7} + +OPATH="$PATH" +export RL_MANYLINUX=1 +ENV=/io/test-env +SRC=/io/${REQUIREMENT}-src + +# Compile wheels +for PYTHON in ${PYTHON_VERSIONS}; do + [ $(lex_ver $PYTHON) -lt $(lex_ver $PYLO) ] && continue + [ $(lex_ver $PYTHON) -ge $(lex_ver $PYHI) ] && continue + for uw in ${UNICODE_WIDTHS}; do + [ $(lex_ver $PYTHON) -ge $(lex_ver 3.3) ] && [ "$uw" != 32 ] && continue + ( + export UNICODE_WIDTH="$uw" + mark "building reportlab wheel PYTHON=$PYTHON UNICODE_WIDTH=$UNICODE_WIDTH" + PP="$(cpython_path $PYTHON $UNICODE_WIDTH)" + PY=$PP/bin/python + mark "platform=$($PY -mplatform) sys.platform=$($PY -c'import sys;print(sys.platform)') os.name=$($PY -c'import os;print(os.name)')" + export PATH="$PP/bin:$OPATH" + PIP="$PP/bin/pip" + echo "Testing ${REQUIREMENT} for Python $PYTHON" + $PIP install virtualenv + rm -rf $ENV + $PY -mvirtualenv $ENV + ( + cd $ENV + . $ENV/bin/activate + EPY=$ENV/bin/python + EPIP=$ENV/bin/pip + [ "${REQUIREMENT}" = "reportlab" ] && $EPIP install --find-links=/io/TEST-WHEELS pillow + $EPIP install --find-links=/io/wheels ${REQUIREMENT} + rm -f /tmp/eee + $EPY setup.py test | tee /tmp/eee + grep -q 'OK' /tmp/eee && R="##### OK " || R="!!!!! FAIL" + echo "${R} PYTHON=${PYTHON} UNICODE_WIDTH=${UNICODE_WIDTH} ARCH=${ARCH}" >> /io/test-results.txt + deactivate + ) + ) + done +done
--- a/manylinux/library_builders.sh Fri Jun 23 11:50:00 2017 +0100 +++ b/manylinux/library_builders.sh Fri Jun 23 12:15:17 2017 +0100 @@ -1,199 +1,199 @@ -# Recipes for building some libaries -OPENBLAS_VERSION="${OPENBLAS_VERSION:-0.2.18}" -# We use system zlib by default - see build_new_zlib -ZLIB_VERSION="${ZLIB_VERSION:-1.2.8}" -LIBPNG_VERSION="${LIBPNG_VERSION:-1.6.21}" -BZIP2_VERSION="${BZIP2_VERSION:-1.0.6}" -FREETYPE_VERSION="${FREETYPE_VERSION:-2.6.3}" -TIFF_VERSION="${TIFF_VERSION:-4.0.6}" -OPENJPEG_VERSION="${OPENJPEG_VERSION:-2.1}" -LCMS2_VERSION="${LCMS2_VERSION:-2.7}" -GIFLIB_VERSION="${GIFLIB_VERSION:-5.1.3}" -LIBWEBP_VERSION="${LIBWEBP_VERSION:-0.5.0}" -XZ_VERSION="${XZ_VERSION:-5.2.2}" -LIBYAML_VERSION="${LIBYAML_VERSION:-0.1.5}" -SZIP_VERSION="${SZIP_VERSION:-2.1}" -HDF5_VERSION="${HDF5_VERSION:-1.8.17}" -LIBAEC_VERSION="${LIBAEC_VERSION:-0.3.3}" -BUILD_PREFIX="${BUILD_PREFIX:-/usr/local}" -ARCHIVE_SDIR=${ARCHIVE_DIR:-archives} -ENABLE_STATIC=${ENABLE_STATIC:-} - -# Set default compilation flags and OSX flag variable -if [ $(uname) == "Darwin" ]; then - # Dual arch build by default - ARCH_FLAGS=${ARCH_FLAGS:-"-arch i386 -arch x86_64"} - # Only set CFLAGS, FFLAGS if they are not already defined. Build functions - # can override the arch flags by setting CFLAGS, FFLAGS - export CFLAGS="${CFLAGS:-$ARCH_FLAGS}" - export FFLAGS="${FFLAGS:-$ARCH_FLAGS}" - IS_OSX=1 -fi - -function build_simple { - local name=$1 - local version=$2 - local url=$3 - if [ -e "${name}-stamp" ]; then - return - fi - local name_version="${name}-${version}" - local targz=${name_version}.tar.gz - fetch_unpack $url/$targz - (cd $name_version \ - && ./configure --prefix=$BUILD_PREFIX ${ENABLE_STATIC} \ - && make \ - && make install) - touch "${name}-stamp" -} - -function build_openblas { - if [ -e openblas-stamp ]; then return; fi - if [ -d "OpenBLAS" ]; then - (cd OpenBLAS && git clean -fxd && git reset --hard) - else - git clone https://github.com/xianyi/OpenBLAS - fi - (cd OpenBLAS \ - && git checkout "v${OPENBLAS_VERSION}" \ - && make DYNAMIC_ARCH=1 USE_OPENMP=0 NUM_THREADS=64 > /dev/null \ - && make PREFIX=$BUILD_PREFIX install) - touch openblas-stamp -} - -function build_zlib { - # Gives an old but safe version - if [ -e zlib-stamp ]; then return; fi - # OSX has zlib already - if [ -z "$IS_OSX" ]; then yum install -y zlib-devel; fi - touch zlib-stamp -} - -function build_new_zlib { - # Careful, this one may cause yum to segfault - build_simple zlib $ZLIB_VERSION http://zlib.net -} - -function build_jpeg { - if [ -e jpeg-stamp ]; then return; fi - fetch_unpack http://ijg.org/files/jpegsrc.v9b.tar.gz - (cd jpeg-9b \ - && ./configure --prefix=$BUILD_PREFIX \ - && make \ - && make install) - touch jpeg-stamp -} - -function build_libpng { - build_zlib - build_simple libpng $LIBPNG_VERSION http://download.sourceforge.net/libpng -} - -function build_bzip2 { - if [ -e bzip2-stamp ]; then return; fi - fetch_unpack http://bzip.org/${BZIP2_VERSION}/bzip2-${BZIP2_VERSION}.tar.gz - (cd bzip2-${BZIP2_VERSION} \ - && make -f Makefile-libbz2_so \ - && make install PREFIX=$BUILD_PREFIX) - touch bzip2-stamp -} - -function build_tiff { - build_zlib - build_jpeg - build_openjpeg - build_xz - build_simple tiff $TIFF_VERSION http://download.osgeo.org/libtiff -} - -function build_openjpeg { - if [ -e openjpeg-stamp ]; then return; fi - local cmake=cmake - if [ -n "$IS_OSX" ]; then - brew install cmake - else - yum install -y cmake28 - cmake=cmake28 - fi - fetch_unpack https://github.com/uclouvain/openjpeg/archive/version.${OPENJPEG_VERSION}.tar.gz - (cd openjpeg-version.${OPENJPEG_VERSION} \ - && $cmake -DCMAKE_INSTALL_PREFIX=$BUILD_PREFIX . \ - && make install) - touch openjpeg-stamp -} - -function build_lcms2 { - build_tiff - build_simple lcms2 $LCMS2_VERSION http://downloads.sourceforge.net/project/lcms/lcms/$LCMS2_VERSION -} - -function build_giflib { - build_simple giflib $GIFLIB_VERSION http://downloads.sourceforge.net/project/giflib -} - -function build_xz { - build_simple xz $XZ_VERSION http://tukaani.org/xz -} - -function build_libwebp { - if [ -e libwebp-stamp ]; then return; fi - build_libpng - build_tiff - build_giflib - fetch_unpack https://storage.googleapis.com/downloads.webmproject.org/releases/webp/libwebp-${LIBWEBP_VERSION}.tar.gz - (cd libwebp-${LIBWEBP_VERSION} && \ - ./configure --enable-libwebpmux --enable-libwebpdemux --prefix=$BUILD_PREFIX \ - && make \ - && make install) - touch libwebp-stamp -} - -function build_freetype { - build_libpng - build_bzip2 - build_simple freetype $FREETYPE_VERSION http://download.savannah.gnu.org/releases/freetype -} - -function build_libyaml { - build_simple yaml $LIBYAML_VERSION http://pyyaml.org/download/libyaml -} - -function build_szip { - # Build szip without encoding (patent restrictions) - if [ -e szip-stamp ]; then return; fi - build_zlib - local szip_url=https://www.hdfgroup.org/ftp/lib-external/szip/ - fetch_unpack ${szip_url}/$SZIP_VERSION/src/szip-$SZIP_VERSION.tar.gz - (cd szip-$SZIP_VERSION \ - && ./configure --enable-encoding=no --prefix=$BUILD_PREFIX \ - && make \ - && make install) - touch szip-stamp -} - -function build_hdf5 { - if [ -e hdf5-stamp ]; then return; fi - build_zlib - # libaec is a drop-in replacement for szip - build_libaec - local hdf5_url=https://www.hdfgroup.org/ftp/HDF5/releases - fetch_unpack $hdf5_url/hdf5-$HDF5_VERSION/src/hdf5-$HDF5_VERSION.tar.gz - (cd hdf5-$HDF5_VERSION \ - && ./configure --with-szlib=$BUILD_PREFIX --prefix=$BUILD_PREFIX \ - && make \ - && make install) - touch hdf5-stamp -} - -function build_libaec { - if [ -e libaec-stamp ]; then return; fi - local root_name=libaec-0.3.3 - local tar_name=${root_name}.tar.gz - # Note URL will change for each version - fetch_unpack https://gitlab.dkrz.de/k202009/libaec/uploads/48398bd5b7bc05a3edb3325abfeac864/${tar_name} - (cd $root_name \ - && ./configure --prefix=$BUILD_PREFIX \ - && make \ - && make install) - touch libaec-stamp -} +# Recipes for building some libaries +OPENBLAS_VERSION="${OPENBLAS_VERSION:-0.2.18}" +# We use system zlib by default - see build_new_zlib +ZLIB_VERSION="${ZLIB_VERSION:-1.2.8}" +LIBPNG_VERSION="${LIBPNG_VERSION:-1.6.21}" +BZIP2_VERSION="${BZIP2_VERSION:-1.0.6}" +FREETYPE_VERSION="${FREETYPE_VERSION:-2.6.3}" +TIFF_VERSION="${TIFF_VERSION:-4.0.6}" +OPENJPEG_VERSION="${OPENJPEG_VERSION:-2.1}" +LCMS2_VERSION="${LCMS2_VERSION:-2.7}" +GIFLIB_VERSION="${GIFLIB_VERSION:-5.1.3}" +LIBWEBP_VERSION="${LIBWEBP_VERSION:-0.5.0}" +XZ_VERSION="${XZ_VERSION:-5.2.2}" +LIBYAML_VERSION="${LIBYAML_VERSION:-0.1.5}" +SZIP_VERSION="${SZIP_VERSION:-2.1}" +HDF5_VERSION="${HDF5_VERSION:-1.8.17}" +LIBAEC_VERSION="${LIBAEC_VERSION:-0.3.3}" +BUILD_PREFIX="${BUILD_PREFIX:-/usr/local}" +ARCHIVE_SDIR=${ARCHIVE_DIR:-archives} +ENABLE_STATIC=${ENABLE_STATIC:-} + +# Set default compilation flags and OSX flag variable +if [ $(uname) == "Darwin" ]; then + # Dual arch build by default + ARCH_FLAGS=${ARCH_FLAGS:-"-arch i386 -arch x86_64"} + # Only set CFLAGS, FFLAGS if they are not already defined. Build functions + # can override the arch flags by setting CFLAGS, FFLAGS + export CFLAGS="${CFLAGS:-$ARCH_FLAGS}" + export FFLAGS="${FFLAGS:-$ARCH_FLAGS}" + IS_OSX=1 +fi + +function build_simple { + local name=$1 + local version=$2 + local url=$3 + if [ -e "${name}-stamp" ]; then + return + fi + local name_version="${name}-${version}" + local targz=${name_version}.tar.gz + fetch_unpack $url/$targz + (cd $name_version \ + && ./configure --prefix=$BUILD_PREFIX ${ENABLE_STATIC} \ + && make \ + && make install) + touch "${name}-stamp" +} + +function build_openblas { + if [ -e openblas-stamp ]; then return; fi + if [ -d "OpenBLAS" ]; then + (cd OpenBLAS && git clean -fxd && git reset --hard) + else + git clone https://github.com/xianyi/OpenBLAS + fi + (cd OpenBLAS \ + && git checkout "v${OPENBLAS_VERSION}" \ + && make DYNAMIC_ARCH=1 USE_OPENMP=0 NUM_THREADS=64 > /dev/null \ + && make PREFIX=$BUILD_PREFIX install) + touch openblas-stamp +} + +function build_zlib { + # Gives an old but safe version + if [ -e zlib-stamp ]; then return; fi + # OSX has zlib already + if [ -z "$IS_OSX" ]; then yum install -y zlib-devel; fi + touch zlib-stamp +} + +function build_new_zlib { + # Careful, this one may cause yum to segfault + build_simple zlib $ZLIB_VERSION http://zlib.net +} + +function build_jpeg { + if [ -e jpeg-stamp ]; then return; fi + fetch_unpack http://ijg.org/files/jpegsrc.v9b.tar.gz + (cd jpeg-9b \ + && ./configure --prefix=$BUILD_PREFIX \ + && make \ + && make install) + touch jpeg-stamp +} + +function build_libpng { + build_zlib + build_simple libpng $LIBPNG_VERSION http://download.sourceforge.net/libpng +} + +function build_bzip2 { + if [ -e bzip2-stamp ]; then return; fi + fetch_unpack http://bzip.org/${BZIP2_VERSION}/bzip2-${BZIP2_VERSION}.tar.gz + (cd bzip2-${BZIP2_VERSION} \ + && make -f Makefile-libbz2_so \ + && make install PREFIX=$BUILD_PREFIX) + touch bzip2-stamp +} + +function build_tiff { + build_zlib + build_jpeg + build_openjpeg + build_xz + build_simple tiff $TIFF_VERSION http://download.osgeo.org/libtiff +} + +function build_openjpeg { + if [ -e openjpeg-stamp ]; then return; fi + local cmake=cmake + if [ -n "$IS_OSX" ]; then + brew install cmake + else + yum install -y cmake28 + cmake=cmake28 + fi + fetch_unpack https://github.com/uclouvain/openjpeg/archive/version.${OPENJPEG_VERSION}.tar.gz + (cd openjpeg-version.${OPENJPEG_VERSION} \ + && $cmake -DCMAKE_INSTALL_PREFIX=$BUILD_PREFIX . \ + && make install) + touch openjpeg-stamp +} + +function build_lcms2 { + build_tiff + build_simple lcms2 $LCMS2_VERSION http://downloads.sourceforge.net/project/lcms/lcms/$LCMS2_VERSION +} + +function build_giflib { + build_simple giflib $GIFLIB_VERSION http://downloads.sourceforge.net/project/giflib +} + +function build_xz { + build_simple xz $XZ_VERSION http://tukaani.org/xz +} + +function build_libwebp { + if [ -e libwebp-stamp ]; then return; fi + build_libpng + build_tiff + build_giflib + fetch_unpack https://storage.googleapis.com/downloads.webmproject.org/releases/webp/libwebp-${LIBWEBP_VERSION}.tar.gz + (cd libwebp-${LIBWEBP_VERSION} && \ + ./configure --enable-libwebpmux --enable-libwebpdemux --prefix=$BUILD_PREFIX \ + && make \ + && make install) + touch libwebp-stamp +} + +function build_freetype { + build_libpng + build_bzip2 + build_simple freetype $FREETYPE_VERSION http://download.savannah.gnu.org/releases/freetype +} + +function build_libyaml { + build_simple yaml $LIBYAML_VERSION http://pyyaml.org/download/libyaml +} + +function build_szip { + # Build szip without encoding (patent restrictions) + if [ -e szip-stamp ]; then return; fi + build_zlib + local szip_url=https://www.hdfgroup.org/ftp/lib-external/szip/ + fetch_unpack ${szip_url}/$SZIP_VERSION/src/szip-$SZIP_VERSION.tar.gz + (cd szip-$SZIP_VERSION \ + && ./configure --enable-encoding=no --prefix=$BUILD_PREFIX \ + && make \ + && make install) + touch szip-stamp +} + +function build_hdf5 { + if [ -e hdf5-stamp ]; then return; fi + build_zlib + # libaec is a drop-in replacement for szip + build_libaec + local hdf5_url=https://www.hdfgroup.org/ftp/HDF5/releases + fetch_unpack $hdf5_url/hdf5-$HDF5_VERSION/src/hdf5-$HDF5_VERSION.tar.gz + (cd hdf5-$HDF5_VERSION \ + && ./configure --with-szlib=$BUILD_PREFIX --prefix=$BUILD_PREFIX \ + && make \ + && make install) + touch hdf5-stamp +} + +function build_libaec { + if [ -e libaec-stamp ]; then return; fi + local root_name=libaec-0.3.3 + local tar_name=${root_name}.tar.gz + # Note URL will change for each version + fetch_unpack https://gitlab.dkrz.de/k202009/libaec/uploads/48398bd5b7bc05a3edb3325abfeac864/${tar_name} + (cd $root_name \ + && ./configure --prefix=$BUILD_PREFIX \ + && make \ + && make install) + touch libaec-stamp +}
--- a/manylinux/manylinux_utils.sh Fri Jun 23 11:50:00 2017 +0100 +++ b/manylinux/manylinux_utils.sh Fri Jun 23 12:15:17 2017 +0100 @@ -1,51 +1,51 @@ -#!/bin/bash -# Useful utilities common across manylinux1 builds - -MULTIBUILD_DIR=$(dirname "${BASH_SOURCE[0]}") -source $MULTIBUILD_DIR/common_utils.sh - -# UNICODE_WIDTH selects "32"=wide (UCS4) or "16"=narrow (UCS2/UTF16) builds -UNICODE_WIDTH="${UNICODE_WIDTH:-32}" - -function get_platform { - # Report platform as given by uname - python -c 'import platform; print(platform.uname()[4])' -} - -function cpython_path { - # Return path to cpython given - # * version (of form "2.7") - # * u_width ("16" or "32" default "32") - # - # For back-compatibility "u" as u_width also means "32" - local py_ver="${1:-2.7}" - local u_width="${2:-${UNICODE_WIDTH}}" - local u_suff=u - # Back-compatibility - if [ "$u_width" == "u" ]; then u_width=32; fi - # For Python >= 3.3, "u" suffix not meaningful - if [ $(lex_ver $py_ver) -ge $(lex_ver 3.3) ] || - [ "$u_width" == "16" ]; then - u_suff="" - elif [ "$u_width" != "32" ]; then - echo "Incorrect u_width value $u_width" - exit 1 - fi - local no_dots=$(echo $py_ver | tr -d .) - echo "/opt/python/cp${no_dots}-cp${no_dots}m${u_suff}" -} - -function repair_wheelhouse { - local in_dir=$1 - local out_dir=${2:-$in_dir} - for whl in $in_dir/*.whl; do - if [[ $whl == *none-any.whl ]]; then # Pure Python wheel - if [ "$in_dir" != "$out_dir" ]; then cp $whl $out_dir; fi - else - auditwheel repair $whl -w $out_dir/ - # Remove unfixed if writing into same directory - if [ "$in_dir" == "$out_dir" ]; then rm $whl; fi - fi - done - chmod -R a+rwX $out_dir -} +#!/bin/bash +# Useful utilities common across manylinux1 builds + +MULTIBUILD_DIR=$(dirname "${BASH_SOURCE[0]}") +source $MULTIBUILD_DIR/common_utils.sh + +# UNICODE_WIDTH selects "32"=wide (UCS4) or "16"=narrow (UCS2/UTF16) builds +UNICODE_WIDTH="${UNICODE_WIDTH:-32}" + +function get_platform { + # Report platform as given by uname + python -c 'import platform; print(platform.uname()[4])' +} + +function cpython_path { + # Return path to cpython given + # * version (of form "2.7") + # * u_width ("16" or "32" default "32") + # + # For back-compatibility "u" as u_width also means "32" + local py_ver="${1:-2.7}" + local u_width="${2:-${UNICODE_WIDTH}}" + local u_suff=u + # Back-compatibility + if [ "$u_width" == "u" ]; then u_width=32; fi + # For Python >= 3.3, "u" suffix not meaningful + if [ $(lex_ver $py_ver) -ge $(lex_ver 3.3) ] || + [ "$u_width" == "16" ]; then + u_suff="" + elif [ "$u_width" != "32" ]; then + echo "Incorrect u_width value $u_width" + exit 1 + fi + local no_dots=$(echo $py_ver | tr -d .) + echo "/opt/python/cp${no_dots}-cp${no_dots}m${u_suff}" +} + +function repair_wheelhouse { + local in_dir=$1 + local out_dir=${2:-$in_dir} + for whl in $in_dir/*.whl; do + if [[ $whl == *none-any.whl ]]; then # Pure Python wheel + if [ "$in_dir" != "$out_dir" ]; then cp $whl $out_dir; fi + else + auditwheel repair $whl -w $out_dir/ + # Remove unfixed if writing into same directory + if [ "$in_dir" == "$out_dir" ]; then rm $whl; fi + fi + done + chmod -R a+rwX $out_dir +}
--- a/rl_ci_tools.py Fri Jun 23 11:50:00 2017 +0100 +++ b/rl_ci_tools.py Fri Jun 23 12:15:17 2017 +0100 @@ -1,290 +1,290 @@ -VERSION='0.0.3' -import os, sys, glob, time, json -PROG=os.path.basename(sys.argv[0]) -debug=verbosity=0 - -class PyPiRequestor(): - scheme = 'https' - host = os.environ.get('CITOOLS_SERVER','www.reportlab.com') - root = '%s://%s' % (scheme,host) - loginurl = "%s/accounts/login/" % root - def __init__(self,debug=0): - self.debug = debug - import requests - self.session = requests.session() - - def login(self,u,p,nxt='/test-7491/'): - s = self.session - resp = s.get(self.loginurl) - loginpage = resp.text - if self.debug>1: - if self.debug>2: print('=============loginpage\n%r' % loginpage) - resp = s.post(self.loginurl, - data=dict( - csrfmiddlewaretoken=s.cookies['csrftoken'], - username=u, - password=p, - next=nxt, - ), - headers=dict( - Referer=self.loginurl, - ), - ) - text = resp.text - status_code = resp.status_code - if debug>2: - print('!!!!!\n%s\n!!!!!'% text) - print('%s: test-7491 csrftoken=%r' % (PROG,resp.cookies.get('csrftoken','???'))) - if text!='I am alive!' or status_code!=200: - raise ValueError('%s: login at %r failed with status_code=%r' % (PROG,self.loginurl,status_code)) - elif verbosity>=2: - print('%s: logged in OK' % PROG) - return status_code - - def _download(self,u,p, kind, fn, dst): - self.login(u,p) - base = '%s/pypi/%s/' % (self.root,kind) - url = base + fn + '/' - resp = self.session.get(url, - data=dict(csrfmiddlewaretoken=self.session.cookies['csrftoken']), - headers = dict(Referer=self.loginurl), - ) - status_code = resp.status_code - b = resp.content - if debug>2: - print('!!!!!\n%r\n!!!!!'% b) - if status_code!=200: - raise ValueError('%s: download %r failed with status_code=%r!\n%r' % (PROG,url,status_code,b)) - if dst: - fn = os.path.join(dst,fn) - with open(fn,'wb') as f: - f.write(b) - if verbosity: - print('%s: %r(%d bytes) downloaded from %r.' % (PROG, fn, len(b),base)) - return resp.status_code - - def download(self,u,p,kind, fn, dst): - for i in self.info(u,p,kind[:-1],fn): - self._download(u,p,kind,i[0],dst) - - def info(self,u,p,kind,pat,subdir=''): - #self.login(u,p) - if subdir: - subdir += '/' - url = '%s/pypi/%s-info/%s%s/?json=1' % (self.root,kind,subdir,pat) - resp = self.session.get(url, - #data=dict(csrfmiddlewaretoken=self.session.cookies['csrftoken']), - headers = dict(Referer=self.loginurl), - ) - status_code = resp.status_code - b = resp.content - if debug>2: - print('!!!!!\n%r\n!!!!!'% b) - if status_code!=200: - raise ValueError('%s: request %r failed with status_code=%r!\n%r' % (PROG,url,status_code,b)) - I = json.loads(b) - if verbosity>1: - print('%s: %r --> %d rows' % (PROG, url, len(I))) - return I - - def upload(self,u,p,kind,fn,subdir=''): - self.login(u,p) - if subdir: - subdir = '/' + subdir - url = '%s/pypi/upload-%s%s/' % (self.root,kind,subdir) - files= dict(file=(os.path.basename(fn),open(fn,'rb'),'application/octet-stream')) - resp = self.session.post(url, - data=dict(csrfmiddlewaretoken=self.session.cookies['csrftoken']), - files=files, - headers = dict(Referer=self.loginurl), - ) - status_code = resp.status_code - text = resp.text - if text!='OK' or status_code!=200: - raise ValueError('%s: upload %r failed with status_code=%r!\n%r' % (PROG,url,status_code,text)) - if verbosity: - print('%s: uploaded %r to %r.' % (PROG,fn,url)) - return resp.status_code - - def clear_cache(self,u,p,fn): - self.login(u,p) - url = '%s/pypi/clear-cache/%s/' % (self.root,fn) - resp = self.session.post(url, - data=dict(csrfmiddlewaretoken=self.session.cookies['csrftoken']), - headers = dict(Referer=self.loginurl), - ) - status_code = resp.status_code - text = resp.text - if not text.endswith('OK') or status_code!=200: - raise ValueError('%s: clear-cache %r failed with status_code=%r!\n%r' % (PROG,url,status_code,text)) - if verbosity: - print('%s: cleared cache %r.' % (PROG,fn)) - return resp.status_code - - def email(self,u,p,subject,faddr,taddr,body): - self.login(u,p) - url = '%s/pypi/email/' % self.root - resp = self.session.post(url, - data=dict(csrfmiddlewaretoken=self.session.cookies['csrftoken'], - subject = subject, - faddr = faddr, - taddr = taddr, - body = body.replace('\\n','\n')), - headers = dict(Referer=self.loginurl), - ) - status_code = resp.status_code - text = resp.text - if not text.endswith('OK') or status_code!=200: - raise ValueError('%s: %r failed with status_code=%r!\n%r' % (PROG,url,status_code,text)) - if verbosity: - print('%s: email to %r(%r) sent.' % (PROG,taddr,subject)) - return resp.status_code - - def package_version(self,u,p,pkg): - I = self.info(u,p,'package','%s-*' % pkg) - if not I: - v = 'unknown' - else: - v = '.'.join(map(str,list(sorted([tuple([int(x) for x in i[0].split('-',2)[1].split('.') if x and x[0] in '0123456789']) for i in I]))[-1])) - return (pkg,v) - -def getoption(key,default=0,cnv=int): - key = '--%s=' % key - v = [x for x in sys.argv if x.startswith(key)] - if v: - for x in v: - sys.argv.remove(x) - v = cnv(v[-1][len(key):]) - else: - v = default - return v - -def _file_info(fn): - st = os.stat(fn) - return (fn,st.st_size,st.st_mtime) - -def _list_fs(patterns,recur=False): - for pat in patterns: - for fn in glob.glob(pat): - if not recur: - yield fn - elif os.path.isdir(fn): - for r,s,F in os.walk(fn): - yield r - for f in F: - yield os.path.join(r,f) - else: - yield fn - -def tabulate(I, - hdrs=['Name','Length',(5*' ')+'Modified'], - fmtmpl8='{:<%d}\x20{:>%d}\x20\x20{:<%d}', - cnvf=(str,str,lambda t: time.strftime('%Y%m%d %H:%M:%S',time.localtime(t))), - ): - if I: - rows = [hdrs] - for row in I: - rows.append([c(r) for c,r in zip(cnvf,row)]) - W = [max(map(len,col)) for col in [list(i) for i in zip(*rows)]] - if debug>3: - print('tabluate: rows=%s' % repr(rows)) - print('tabluate: W=%s' % repr(W)) - fmt = fmtmpl8 % tuple(W) - print('\n'.join(fmt.format(*i) for i in rows)) - -def main(): - u = os.environ.get('CITOOLS_USER','beta') - p = os.environ.get('CITOOLS_PASSWORD','???') - global debug, verbosity - debug = getoption('debug') - verbosity = getoption('verbosity') - - if debug>3: - import logging - - # These two lines enable debugging at httplib level (requests->urllib3->http.client) - # You will see the REQUEST, including HEADERS and DATA, and RESPONSE with HEADERS but without DATA. - # The only thing missing will be the response.body which is not logged. - try: - import http.client as http_client - except ImportError: - # Python 2 - import httplib as http_client - http_client.HTTPConnection.debuglevel = 1 - - # You must initialize logging, otherwise you'll not see debug output. - logging.basicConfig() - logging.getLogger().setLevel(logging.DEBUG) - requests_log = logging.getLogger("requests.packages.urllib3") - requests_log.setLevel(logging.DEBUG) - requests_log.propagate = True - - dst = getoption('dst',None,cnv=str) - try: - cmd = sys.argv[1] - except: - cmd = 'help' - if cmd=='env': - print('Environment') - print('===========') - I = list(sorted(os.environ.iteritems())) - i = max([len(i[0]) for i in I]) - print(('{:<%d} {}' % i).format('Key','Value')) - fmt = '{:<%d} = {}' % i - for i in I: - print(fmt.format(*i)) - elif cmd=='info': - recur = [fn for fn in sys.argv[2:] if fn=='--recur'] - if recur: - map(sys.argv.remove,recur) - recur = True - tabulate([_file_info(i) for i in _list_fs(sys.argv[2:],recur)]) - elif cmd=='help': - print('Usage %s [test|info|env|download-[resources|packages]|upload-[resources|packages]|[packages|resources]-info] path....' % PROG) - else: - pypi = PyPiRequestor(debug=debug) - if cmd=='test': - status_code = pypi.login(u,p) - if debug: - print('status=%s' % status_code) - elif cmd.startswith('download-'): - kind = cmd.split('-')[1] - if not kind in ('resources','packages', 'caches'): - raise ValueError('%s: invalid download kind: %r' % (PROG,kind)) - if dst and not os.path.isdir(dst): - raise ValueError('%s: %r is not a directory!' % (PROG,dst)) - for fn in sys.argv[2:]: - pypi.download(u,p,kind,fn,dst) - elif cmd.endswith('-info'): - kind = cmd.split('-')[0] - if not kind in ('resource','package', 'cache'): - raise ValueError('%s: invalid info kind: %r' % (PROG,kind)) - subdir = getoption('subdir','',str) if kind=='cache' else '' - tabulate([i for fn in sys.argv[2:] for i in pypi.info(u,p,kind,fn,subdir)]) - elif cmd.startswith('upload-'): - kind = cmd.split('-')[1] - if not kind in ('resources','packages', 'caches'): - raise ValueError('%s: invalid upload kind: %r' % (PROG,kind)) - subdir = getoption('subdir','',str) if kind=='caches' else '' - for pat in sys.argv[2:]: - for fn in glob.glob(pat): - pypi.upload(u,p,kind[:-1],fn,subdir) - elif cmd=='clear-cache': - for fn in sys.argv[2:]: - pypi.clear_cache(u,p,fn) - elif cmd=='email': - subject = getoption('subject','unknown',str) - faddr = getoption('faddr','bitbucket@reportlab.com',str) - taddr = getoption('taddr','live_errors@reportlab.com',str) - body = getoption('body','live_errors@reportlab.com',str) - pypi.email(u,p,subject,faddr,taddr,body) - elif cmd=='package-version': - tabulate([pypi.package_version(u,p,fn) for fn in sys.argv[2:]], - hdrs = ['Package','Version'], - fmtmpl8 = '{:<%d} {:>%d}', - cnvf = (str,str), - ) - else: - raise ValueError('%s: unknown command %r' % (PROG,cmd)) -if __name__=='__main__': - main() +VERSION='0.0.3' +import os, sys, glob, time, json +PROG=os.path.basename(sys.argv[0]) +debug=verbosity=0 + +class PyPiRequestor(): + scheme = 'https' + host = os.environ.get('CITOOLS_SERVER','www.reportlab.com') + root = '%s://%s' % (scheme,host) + loginurl = "%s/accounts/login/" % root + def __init__(self,debug=0): + self.debug = debug + import requests + self.session = requests.session() + + def login(self,u,p,nxt='/test-7491/'): + s = self.session + resp = s.get(self.loginurl) + loginpage = resp.text + if self.debug>1: + if self.debug>2: print('=============loginpage\n%r' % loginpage) + resp = s.post(self.loginurl, + data=dict( + csrfmiddlewaretoken=s.cookies['csrftoken'], + username=u, + password=p, + next=nxt, + ), + headers=dict( + Referer=self.loginurl, + ), + ) + text = resp.text + status_code = resp.status_code + if debug>2: + print('!!!!!\n%s\n!!!!!'% text) + print('%s: test-7491 csrftoken=%r' % (PROG,resp.cookies.get('csrftoken','???'))) + if text!='I am alive!' or status_code!=200: + raise ValueError('%s: login at %r failed with status_code=%r' % (PROG,self.loginurl,status_code)) + elif verbosity>=2: + print('%s: logged in OK' % PROG) + return status_code + + def _download(self,u,p, kind, fn, dst): + self.login(u,p) + base = '%s/pypi/%s/' % (self.root,kind) + url = base + fn + '/' + resp = self.session.get(url, + data=dict(csrfmiddlewaretoken=self.session.cookies['csrftoken']), + headers = dict(Referer=self.loginurl), + ) + status_code = resp.status_code + b = resp.content + if debug>2: + print('!!!!!\n%r\n!!!!!'% b) + if status_code!=200: + raise ValueError('%s: download %r failed with status_code=%r!\n%r' % (PROG,url,status_code,b)) + if dst: + fn = os.path.join(dst,fn) + with open(fn,'wb') as f: + f.write(b) + if verbosity: + print('%s: %r(%d bytes) downloaded from %r.' % (PROG, fn, len(b),base)) + return resp.status_code + + def download(self,u,p,kind, fn, dst): + for i in self.info(u,p,kind[:-1],fn): + self._download(u,p,kind,i[0],dst) + + def info(self,u,p,kind,pat,subdir=''): + #self.login(u,p) + if subdir: + subdir += '/' + url = '%s/pypi/%s-info/%s%s/?json=1' % (self.root,kind,subdir,pat) + resp = self.session.get(url, + #data=dict(csrfmiddlewaretoken=self.session.cookies['csrftoken']), + headers = dict(Referer=self.loginurl), + ) + status_code = resp.status_code + b = resp.content + if debug>2: + print('!!!!!\n%r\n!!!!!'% b) + if status_code!=200: + raise ValueError('%s: request %r failed with status_code=%r!\n%r' % (PROG,url,status_code,b)) + I = json.loads(b) + if verbosity>1: + print('%s: %r --> %d rows' % (PROG, url, len(I))) + return I + + def upload(self,u,p,kind,fn,subdir=''): + self.login(u,p) + if subdir: + subdir = '/' + subdir + url = '%s/pypi/upload-%s%s/' % (self.root,kind,subdir) + files= dict(file=(os.path.basename(fn),open(fn,'rb'),'application/octet-stream')) + resp = self.session.post(url, + data=dict(csrfmiddlewaretoken=self.session.cookies['csrftoken']), + files=files, + headers = dict(Referer=self.loginurl), + ) + status_code = resp.status_code + text = resp.text + if text!='OK' or status_code!=200: + raise ValueError('%s: upload %r failed with status_code=%r!\n%r' % (PROG,url,status_code,text)) + if verbosity: + print('%s: uploaded %r to %r.' % (PROG,fn,url)) + return resp.status_code + + def clear_cache(self,u,p,fn): + self.login(u,p) + url = '%s/pypi/clear-cache/%s/' % (self.root,fn) + resp = self.session.post(url, + data=dict(csrfmiddlewaretoken=self.session.cookies['csrftoken']), + headers = dict(Referer=self.loginurl), + ) + status_code = resp.status_code + text = resp.text + if not text.endswith('OK') or status_code!=200: + raise ValueError('%s: clear-cache %r failed with status_code=%r!\n%r' % (PROG,url,status_code,text)) + if verbosity: + print('%s: cleared cache %r.' % (PROG,fn)) + return resp.status_code + + def email(self,u,p,subject,faddr,taddr,body): + self.login(u,p) + url = '%s/pypi/email/' % self.root + resp = self.session.post(url, + data=dict(csrfmiddlewaretoken=self.session.cookies['csrftoken'], + subject = subject, + faddr = faddr, + taddr = taddr, + body = body.replace('\\n','\n')), + headers = dict(Referer=self.loginurl), + ) + status_code = resp.status_code + text = resp.text + if not text.endswith('OK') or status_code!=200: + raise ValueError('%s: %r failed with status_code=%r!\n%r' % (PROG,url,status_code,text)) + if verbosity: + print('%s: email to %r(%r) sent.' % (PROG,taddr,subject)) + return resp.status_code + + def package_version(self,u,p,pkg): + I = self.info(u,p,'package','%s-*' % pkg) + if not I: + v = 'unknown' + else: + v = '.'.join(map(str,list(sorted([tuple([int(x) for x in i[0].split('-',2)[1].split('.') if x and x[0] in '0123456789']) for i in I]))[-1])) + return (pkg,v) + +def getoption(key,default=0,cnv=int): + key = '--%s=' % key + v = [x for x in sys.argv if x.startswith(key)] + if v: + for x in v: + sys.argv.remove(x) + v = cnv(v[-1][len(key):]) + else: + v = default + return v + +def _file_info(fn): + st = os.stat(fn) + return (fn,st.st_size,st.st_mtime) + +def _list_fs(patterns,recur=False): + for pat in patterns: + for fn in glob.glob(pat): + if not recur: + yield fn + elif os.path.isdir(fn): + for r,s,F in os.walk(fn): + yield r + for f in F: + yield os.path.join(r,f) + else: + yield fn + +def tabulate(I, + hdrs=['Name','Length',(5*' ')+'Modified'], + fmtmpl8='{:<%d}\x20{:>%d}\x20\x20{:<%d}', + cnvf=(str,str,lambda t: time.strftime('%Y%m%d %H:%M:%S',time.localtime(t))), + ): + if I: + rows = [hdrs] + for row in I: + rows.append([c(r) for c,r in zip(cnvf,row)]) + W = [max(map(len,col)) for col in [list(i) for i in zip(*rows)]] + if debug>3: + print('tabluate: rows=%s' % repr(rows)) + print('tabluate: W=%s' % repr(W)) + fmt = fmtmpl8 % tuple(W) + print('\n'.join(fmt.format(*i) for i in rows)) + +def main(): + u = os.environ.get('CITOOLS_USER','beta') + p = os.environ.get('CITOOLS_PASSWORD','???') + global debug, verbosity + debug = getoption('debug') + verbosity = getoption('verbosity') + + if debug>3: + import logging + + # These two lines enable debugging at httplib level (requests->urllib3->http.client) + # You will see the REQUEST, including HEADERS and DATA, and RESPONSE with HEADERS but without DATA. + # The only thing missing will be the response.body which is not logged. + try: + import http.client as http_client + except ImportError: + # Python 2 + import httplib as http_client + http_client.HTTPConnection.debuglevel = 1 + + # You must initialize logging, otherwise you'll not see debug output. + logging.basicConfig() + logging.getLogger().setLevel(logging.DEBUG) + requests_log = logging.getLogger("requests.packages.urllib3") + requests_log.setLevel(logging.DEBUG) + requests_log.propagate = True + + dst = getoption('dst',None,cnv=str) + try: + cmd = sys.argv[1] + except: + cmd = 'help' + if cmd=='env': + print('Environment') + print('===========') + I = list(sorted(os.environ.iteritems())) + i = max([len(i[0]) for i in I]) + print(('{:<%d} {}' % i).format('Key','Value')) + fmt = '{:<%d} = {}' % i + for i in I: + print(fmt.format(*i)) + elif cmd=='info': + recur = [fn for fn in sys.argv[2:] if fn=='--recur'] + if recur: + map(sys.argv.remove,recur) + recur = True + tabulate([_file_info(i) for i in _list_fs(sys.argv[2:],recur)]) + elif cmd=='help': + print('Usage %s [test|info|env|download-[resources|packages]|upload-[resources|packages]|[packages|resources]-info] path....' % PROG) + else: + pypi = PyPiRequestor(debug=debug) + if cmd=='test': + status_code = pypi.login(u,p) + if debug: + print('status=%s' % status_code) + elif cmd.startswith('download-'): + kind = cmd.split('-')[1] + if not kind in ('resources','packages', 'caches'): + raise ValueError('%s: invalid download kind: %r' % (PROG,kind)) + if dst and not os.path.isdir(dst): + raise ValueError('%s: %r is not a directory!' % (PROG,dst)) + for fn in sys.argv[2:]: + pypi.download(u,p,kind,fn,dst) + elif cmd.endswith('-info'): + kind = cmd.split('-')[0] + if not kind in ('resource','package', 'cache'): + raise ValueError('%s: invalid info kind: %r' % (PROG,kind)) + subdir = getoption('subdir','',str) if kind=='cache' else '' + tabulate([i for fn in sys.argv[2:] for i in pypi.info(u,p,kind,fn,subdir)]) + elif cmd.startswith('upload-'): + kind = cmd.split('-')[1] + if not kind in ('resources','packages', 'caches'): + raise ValueError('%s: invalid upload kind: %r' % (PROG,kind)) + subdir = getoption('subdir','',str) if kind=='caches' else '' + for pat in sys.argv[2:]: + for fn in glob.glob(pat): + pypi.upload(u,p,kind[:-1],fn,subdir) + elif cmd=='clear-cache': + for fn in sys.argv[2:]: + pypi.clear_cache(u,p,fn) + elif cmd=='email': + subject = getoption('subject','unknown',str) + faddr = getoption('faddr','bitbucket@reportlab.com',str) + taddr = getoption('taddr','live_errors@reportlab.com',str) + body = getoption('body','live_errors@reportlab.com',str) + pypi.email(u,p,subject,faddr,taddr,body) + elif cmd=='package-version': + tabulate([pypi.package_version(u,p,fn) for fn in sys.argv[2:]], + hdrs = ['Package','Version'], + fmtmpl8 = '{:<%d} {:>%d}', + cnvf = (str,str), + ) + else: + raise ValueError('%s: unknown command %r' % (PROG,cmd)) +if __name__=='__main__': + main()
--- a/setup.py Fri Jun 23 11:50:00 2017 +0100 +++ b/setup.py Fri Jun 23 12:15:17 2017 +0100 @@ -1,25 +1,25 @@ -#from distutils.core import setup -from setuptools import setup -import os, sys - -if __name__=='__main__': - pkgDir=os.path.dirname(sys.argv[0]) - if not pkgDir: - pkgDir=os.getcwd() - if not os.path.isabs(pkgDir): - pkgDir=os.path.abspath(pkgDir) - sys.path.insert(0,pkgDir) - os.chdir(pkgDir) - import rl_ci_tools - version = rl_ci_tools.VERSION - - setup(name='rl_ci_tools', - version=version, - description='Continuous Integration Support for ReportLab and friends', - author='Robin Becker', - author_email='andy@reportlab.com', - url='https://bitbucket.org/MrRLBitBucket/rl_ci_tools', - py_modules=['rl_ci_tools'], - requires=['requests'], - install_requires=['requests'], - ) +#from distutils.core import setup +from setuptools import setup +import os, sys + +if __name__=='__main__': + pkgDir=os.path.dirname(sys.argv[0]) + if not pkgDir: + pkgDir=os.getcwd() + if not os.path.isabs(pkgDir): + pkgDir=os.path.abspath(pkgDir) + sys.path.insert(0,pkgDir) + os.chdir(pkgDir) + import rl_ci_tools + version = rl_ci_tools.VERSION + + setup(name='rl_ci_tools', + version=version, + description='Continuous Integration Support for ReportLab and friends', + author='Robin Becker', + author_email='andy@reportlab.com', + url='https://bitbucket.org/MrRLBitBucket/rl_ci_tools', + py_modules=['rl_ci_tools'], + requires=['requests'], + install_requires=['requests'], + )